[v8,2/2] hash: unify crc32 selection for x86 and Arm
Checks
Commit Message
Merge crc32 hash calculation public API implementation for x86 and Arm.
Select the best available CRC32 algorithm when unsupported algorithm
on a given CPU architecture is requested by an application.
Previously, if an application directly includes `rte_crc_arm64.h`
without including `rte_hash_crc.h` it will fail to compile.
Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
Reviewed-by: Ruifeng Wang <ruifeng.wang@arm.com>
---
lib/hash/meson.build | 1 +
lib/hash/rte_crc_arm64.h | 69 +++---------------
lib/hash/rte_crc_generic.h | 72 ++++++++++++++++++
lib/hash/rte_crc_x86.h | 89 +++++++++++++++++++++++
lib/hash/rte_hash_crc.h | 145 ++++++++++---------------------------
5 files changed, 210 insertions(+), 166 deletions(-)
create mode 100644 lib/hash/rte_crc_generic.h
Comments
On Fri, Apr 29, 2022 at 6:17 PM Pavan Nikhilesh
<pbhagavatula@marvell.com> wrote:
>
> Merge crc32 hash calculation public API implementation for x86 and Arm.
> Select the best available CRC32 algorithm when unsupported algorithm
> on a given CPU architecture is requested by an application.
>
> Previously, if an application directly includes `rte_crc_arm64.h`
> without including `rte_hash_crc.h` it will fail to compile.
On this topic, a followup patch could add a check to those internal
headers, to ensure _RTE_HASH_CRC_H_ is defined and #error otherwise.
Like what is done in lib/eal/x86/include/rte_atomic_64.h, for example.
>
> Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
> Reviewed-by: Ruifeng Wang <ruifeng.wang@arm.com>
> ---
> lib/hash/meson.build | 1 +
> lib/hash/rte_crc_arm64.h | 69 +++---------------
> lib/hash/rte_crc_generic.h | 72 ++++++++++++++++++
> lib/hash/rte_crc_x86.h | 89 +++++++++++++++++++++++
> lib/hash/rte_hash_crc.h | 145 ++++++++++---------------------------
> 5 files changed, 210 insertions(+), 166 deletions(-)
> create mode 100644 lib/hash/rte_crc_generic.h
>
> diff --git a/lib/hash/meson.build b/lib/hash/meson.build
> index ff13e2c7f9..b36c8b0c01 100644
> --- a/lib/hash/meson.build
> +++ b/lib/hash/meson.build
> @@ -13,6 +13,7 @@ indirect_headers += files(
> 'rte_crc_sw.h',
> 'rte_crc_x86.h',
> 'rte_crc_arm64.h',
> + 'rte_crc_generic.h',
> 'rte_thash_x86_gfni.h',
> )
>
> diff --git a/lib/hash/rte_crc_arm64.h b/lib/hash/rte_crc_arm64.h
> index b4628cfc09..172894335f 100644
> --- a/lib/hash/rte_crc_arm64.h
> +++ b/lib/hash/rte_crc_arm64.h
> @@ -2,23 +2,8 @@
> * Copyright(c) 2015 Cavium, Inc
> */
>
> -#ifndef _RTE_CRC_ARM64_H_
> -#define _RTE_CRC_ARM64_H_
> -
> -/**
> - * @file
> - *
> - * RTE CRC arm64 Hash
> - */
> -
> -#ifdef __cplusplus
> -extern "C" {
> -#endif
> -
> -#include <stdint.h>
> -#include <rte_cpuflags.h>
> -#include <rte_branch_prediction.h>
> -#include <rte_common.h>
> +#ifndef _HASH_CRC_ARM64_H_
> +#define _HASH_CRC_ARM64_H_
>
> static inline uint32_t
> crc32c_arm64_u8(uint8_t data, uint32_t init_val)
> @@ -61,40 +46,8 @@ crc32c_arm64_u64(uint64_t data, uint32_t init_val)
> }
>
> /**
> - * Allow or disallow use of arm64 SIMD instrinsics for CRC32 hash
> - * calculation.
> - *
> - * @param alg
> - * An OR of following flags:
> - * - (CRC32_SW) Don't use arm64 crc intrinsics
> - * - (CRC32_ARM64) Use ARMv8 CRC intrinsic if available
> - *
> - */
> -static inline void
> -rte_hash_crc_set_alg(uint8_t alg)
> -{
> - switch (alg) {
> - case CRC32_ARM64:
> - if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_CRC32))
> - alg = CRC32_SW;
> - /* fall-through */
> - case CRC32_SW:
> - crc32_alg = alg;
> - /* fall-through */
> - default:
> - break;
> - }
> -}
> -
> -/* Setting the best available algorithm */
> -RTE_INIT(rte_hash_crc_init_alg)
> -{
> - rte_hash_crc_set_alg(CRC32_ARM64);
> -}
> -
> -/**
> - * Use single crc32 instruction to perform a hash on a 1 byte value.
> - * Fall back to software crc32 implementation in case arm64 crc intrinsics is
> + * Use single crc32 instruction to perform a hash on a byte value.
> + * Fall back to software crc32 implementation in case ARM CRC is
> * not supported
> *
> * @param data
> @@ -115,7 +68,7 @@ rte_hash_crc_1byte(uint8_t data, uint32_t init_val)
>
> /**
> * Use single crc32 instruction to perform a hash on a 2 bytes value.
> - * Fall back to software crc32 implementation in case arm64 crc intrinsics is
> + * Fall back to software crc32 implementation in case ARM CRC is
> * not supported
> *
> * @param data
> @@ -136,7 +89,7 @@ rte_hash_crc_2byte(uint16_t data, uint32_t init_val)
>
> /**
> * Use single crc32 instruction to perform a hash on a 4 byte value.
> - * Fall back to software crc32 implementation in case arm64 crc intrinsics is
> + * Fall back to software crc32 implementation in case ARM CRC is
> * not supported
> *
> * @param data
> @@ -157,7 +110,7 @@ rte_hash_crc_4byte(uint32_t data, uint32_t init_val)
>
> /**
> * Use single crc32 instruction to perform a hash on a 8 byte value.
> - * Fall back to software crc32 implementation in case arm64 crc intrinsics is
> + * Fall back to software crc32 implementation in case ARM CRC is
> * not supported
> *
> * @param data
> @@ -170,14 +123,10 @@ rte_hash_crc_4byte(uint32_t data, uint32_t init_val)
> static inline uint32_t
> rte_hash_crc_8byte(uint64_t data, uint32_t init_val)
> {
> - if (likely(crc32_alg == CRC32_ARM64))
> + if (likely(crc32_alg & CRC32_ARM64))
> return crc32c_arm64_u64(data, init_val);
>
> return crc32c_2words(data, init_val);
> }
>
> -#ifdef __cplusplus
> -}
> -#endif
> -
> -#endif /* _RTE_CRC_ARM64_H_ */
> +#endif /* _HASH_CRC_ARM64_H_ */
> diff --git a/lib/hash/rte_crc_generic.h b/lib/hash/rte_crc_generic.h
> new file mode 100644
> index 0000000000..0c55947896
> --- /dev/null
> +++ b/lib/hash/rte_crc_generic.h
> @@ -0,0 +1,72 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2022 Marvell.
> + */
> +
> +#ifndef _HASH_CRC_GENERIC_H_
> +#define _HASH_CRC_GENERIC_H_
> +
> +/**
> + * Software crc32 implementation for 1 byte value.
> + *
> + * @param data
> + * Data to perform hash on.
> + * @param init_val
> + * Value to initialise hash generator.
> + * @return
> + * 32bit calculated hash value.
> + */
> +static inline uint32_t
> +rte_hash_crc_1byte(uint8_t data, uint32_t init_val)
> +{
> + return crc32c_1byte(data, init_val);
> +}
> +
> +/**
> + * Software crc32 implementation for 2 byte value.
> + *
> + * @param data
> + * Data to perform hash on.
> + * @param init_val
> + * Value to initialise hash generator.
> + * @return
> + * 32bit calculated hash value.
> + */
> +static inline uint32_t
> +rte_hash_crc_2byte(uint16_t data, uint32_t init_val)
> +{
> + return crc32c_2bytes(data, init_val);
> +}
> +
> +/**
> + * Software crc32 implementation for 4 byte value.
> + *
> + * @param data
> + * Data to perform hash on.
> + * @param init_val
> + * Value to initialise hash generator.
> + * @return
> + * 32bit calculated hash value.
> + */
> +static inline uint32_t
> +rte_hash_crc_4byte(uint32_t data, uint32_t init_val)
> +{
> + return crc32c_1word(data, init_val);
> +}
> +
> +/**
> + * Software crc32 implementation for 8 byte value.
> + *
> + * @param data
> + * Data to perform hash on.
> + * @param init_val
> + * Value to initialise hash generator.
> + * @return
> + * 32bit calculated hash value.
> + */
> +static inline uint32_t
> +rte_hash_crc_8byte(uint64_t data, uint32_t init_val)
> +{
> + return crc32c_2words(data, init_val);
> +}
> +
> +#endif
> diff --git a/lib/hash/rte_crc_x86.h b/lib/hash/rte_crc_x86.h
> index b80a742afa..19eb3584e7 100644
> --- a/lib/hash/rte_crc_x86.h
> +++ b/lib/hash/rte_crc_x86.h
> @@ -59,4 +59,93 @@ crc32c_sse42_u64(uint64_t data, uint64_t init_val)
> return (uint32_t)init_val;
> }
>
> +/**
> + * Use single crc32 instruction to perform a hash on a byte value.
> + * Fall back to software crc32 implementation in case SSE4.2 is
> + * not supported
> + *
> + * @param data
> + * Data to perform hash on.
> + * @param init_val
> + * Value to initialise hash generator.
> + * @return
> + * 32bit calculated hash value.
> + */
> +static inline uint32_t
> +rte_hash_crc_1byte(uint8_t data, uint32_t init_val)
> +{
> + if (likely(crc32_alg & CRC32_SSE42))
> + return crc32c_sse42_u8(data, init_val);
> +
> + return crc32c_1byte(data, init_val);
> +}
> +
> +/**
> + * Use single crc32 instruction to perform a hash on a 2 bytes value.
> + * Fall back to software crc32 implementation in case SSE4.2 is
> + * not supported
> + *
> + * @param data
> + * Data to perform hash on.
> + * @param init_val
> + * Value to initialise hash generator.
> + * @return
> + * 32bit calculated hash value.
> + */
> +static inline uint32_t
> +rte_hash_crc_2byte(uint16_t data, uint32_t init_val)
> +{
> + if (likely(crc32_alg & CRC32_SSE42))
> + return crc32c_sse42_u16(data, init_val);
> +
> + return crc32c_2bytes(data, init_val);
> +}
> +
> +/**
> + * Use single crc32 instruction to perform a hash on a 4 byte value.
> + * Fall back to software crc32 implementation in case SSE4.2 is
> + * not supported
> + *
> + * @param data
> + * Data to perform hash on.
> + * @param init_val
> + * Value to initialise hash generator.
> + * @return
> + * 32bit calculated hash value.
> + */
> +static inline uint32_t
> +rte_hash_crc_4byte(uint32_t data, uint32_t init_val)
> +{
> + if (likely(crc32_alg & CRC32_SSE42))
> + return crc32c_sse42_u32(data, init_val);
> +
> + return crc32c_1word(data, init_val);
> +}
> +
> +/**
> + * Use single crc32 instruction to perform a hash on a 8 byte value.
> + * Fall back to software crc32 implementation in case SSE4.2 is
> + * not supported
> + *
> + * @param data
> + * Data to perform hash on.
> + * @param init_val
> + * Value to initialise hash generator.
> + * @return
> + * 32bit calculated hash value.
> + */
> +static inline uint32_t
> +rte_hash_crc_8byte(uint64_t data, uint32_t init_val)
> +{
> +#ifdef RTE_ARCH_X86_64
> + if (likely(crc32_alg == CRC32_SSE42_x64))
> + return crc32c_sse42_u64(data, init_val);
> +#endif
> +
> + if (likely(crc32_alg & CRC32_SSE42))
> + return crc32c_sse42_u64_mimic(data, init_val);
> +
> + return crc32c_2words(data, init_val);
> +}
> +
> #endif
> diff --git a/lib/hash/rte_hash_crc.h b/lib/hash/rte_hash_crc.h
> index 308fdde414..cd9ec17082 100644
> --- a/lib/hash/rte_hash_crc.h
> +++ b/lib/hash/rte_hash_crc.h
> @@ -16,10 +16,12 @@ extern "C" {
> #endif
>
> #include <stdint.h>
> -#include <rte_config.h>
> -#include <rte_cpuflags.h>
> +
> #include <rte_branch_prediction.h>
> #include <rte_common.h>
> +#include <rte_config.h>
> +#include <rte_cpuflags.h>
> +#include <rte_log.h>
>
> #include "rte_crc_sw.h"
>
> @@ -33,136 +35,67 @@ static uint8_t crc32_alg = CRC32_SW;
>
> #if defined(RTE_ARCH_ARM64) && defined(__ARM_FEATURE_CRC32)
> #include "rte_crc_arm64.h"
> -#else
> +#elif defined(RTE_ARCH_X86)
> #include "rte_crc_x86.h"
> +#else
> +#include "rte_crc_generic.h"
> +#endif
>
> /**
> - * Allow or disallow use of SSE4.2 instrinsics for CRC32 hash
> + * Allow or disallow use of SSE4.2/ARMv8 intrinsics for CRC32 hash
> * calculation.
> *
> * @param alg
> * An OR of following flags:
> - * - (CRC32_SW) Don't use SSE4.2 intrinsics
> + * - (CRC32_SW) Don't use SSE4.2/ARMv8 intrinsics (default non-[x86/ARMv8])
> * - (CRC32_SSE42) Use SSE4.2 intrinsics if available
> - * - (CRC32_SSE42_x64) Use 64-bit SSE4.2 intrinsic if available (default)
> + * - (CRC32_SSE42_x64) Use 64-bit SSE4.2 intrinsic if available (default x86)
> + * - (CRC32_ARM64) Use ARMv8 CRC intrinsic if available (default ARMv8)
> *
> */
> static inline void
> rte_hash_crc_set_alg(uint8_t alg)
> {
> -#if defined(RTE_ARCH_X86)
> - if (alg == CRC32_SSE42_x64 &&
> - !rte_cpu_get_flag_enabled(RTE_CPUFLAG_EM64T))
> - alg = CRC32_SSE42;
> -#endif
> - crc32_alg = alg;
> -}
> + crc32_alg = CRC32_SW;
>
> -/* Setting the best available algorithm */
> -RTE_INIT(rte_hash_crc_init_alg)
> -{
> - rte_hash_crc_set_alg(CRC32_SSE42_x64);
> -}
> + if (alg == CRC32_SW)
> + return;
>
> -/**
> - * Use single crc32 instruction to perform a hash on a byte value.
> - * Fall back to software crc32 implementation in case SSE4.2 is
> - * not supported
> - *
> - * @param data
> - * Data to perform hash on.
> - * @param init_val
> - * Value to initialise hash generator.
> - * @return
> - * 32bit calculated hash value.
> - */
> -static inline uint32_t
> -rte_hash_crc_1byte(uint8_t data, uint32_t init_val)
By moving all these sw helpers into a new file, the rendered
documentation loses their descriptions.
We should keep a declaration of each function of the API under a
#ifdef __DOXYGEN__ in rte_hash_crc.h.
If there is no objection, I'll fix by keeping only the generic
description of each function, like shown below:
diff --git a/lib/hash/rte_hash_crc.h b/lib/hash/rte_hash_crc.h
index cd9ec17082..86c341a7a4 100644
--- a/lib/hash/rte_hash_crc.h
+++ b/lib/hash/rte_hash_crc.h
@@ -96,6 +96,62 @@ RTE_INIT(rte_hash_crc_init_alg)
#endif
}
+#ifdef __DOXYGEN__
+
+/**
+ * Use single crc32 instruction to perform a hash on a byte value.
+ *
+ * @param data
+ * Data to perform hash on.
+ * @param init_val
+ * Value to initialise hash generator.
+ * @return
+ * 32bit calculated hash value.
+ */
+static inline uint32_t
+rte_hash_crc_1byte(uint8_t data, uint32_t init_val);
+
[snip]
+static inline uint32_t
+rte_hash_crc_8byte(uint64_t data, uint32_t init_val);
+
+#endif /* __DOXYGEN__ */
+
/**
* Calculate CRC32 hash on user-supplied byte array.
*
> -----Original Message-----
> From: Pavan Nikhilesh <pbhagavatula@marvell.com>
> Sent: Friday, April 29, 2022 9:17 AM
> To: ruifeng.wang@arm.com; Wang, Yipeng1 <yipeng1.wang@intel.com>;
> Gobriel, Sameh <sameh.gobriel@intel.com>; Richardson, Bruce
> <bruce.richardson@intel.com>; Medvedkin, Vladimir
> <vladimir.medvedkin@intel.com>
> Cc: jerinj@marvell.com; dev@dpdk.org; Pavan Nikhilesh
> <pbhagavatula@marvell.com>
> Subject: [PATCH v8 2/2] hash: unify crc32 selection for x86 and Arm
>
> Merge crc32 hash calculation public API implementation for x86 and Arm.
> Select the best available CRC32 algorithm when unsupported algorithm on a
> given CPU architecture is requested by an application.
>
> Previously, if an application directly includes `rte_crc_arm64.h` without
> including `rte_hash_crc.h` it will fail to compile.
>
> Signed-off-by: Pavan Nikhilesh <pbhagavatula@marvell.com>
> Reviewed-by: Ruifeng Wang <ruifeng.wang@arm.com>
> ---
> lib/hash/meson.build | 1 +
> lib/hash/rte_crc_arm64.h | 69 +++---------------
> lib/hash/rte_crc_generic.h | 72 ++++++++++++++++++
> lib/hash/rte_crc_x86.h | 89 +++++++++++++++++++++++
> lib/hash/rte_hash_crc.h | 145 ++++++++++---------------------------
> 5 files changed, 210 insertions(+), 166 deletions(-) create mode 100644
> lib/hash/rte_crc_generic.h
>
> diff --git a/lib/hash/meson.build b/lib/hash/meson.build index
> ff13e2c7f9..b36c8b0c01 100644
> --- a/lib/hash/meson.build
> +++ b/lib/hash/meson.build
> @@ -13,6 +13,7 @@ indirect_headers += files(
> 'rte_crc_sw.h',
> 'rte_crc_x86.h',
> 'rte_crc_arm64.h',
> + 'rte_crc_generic.h',
> 'rte_thash_x86_gfni.h',
> )
>
> diff --git a/lib/hash/rte_crc_arm64.h b/lib/hash/rte_crc_arm64.h index
> b4628cfc09..172894335f 100644
> --- a/lib/hash/rte_crc_arm64.h
> +++ b/lib/hash/rte_crc_arm64.h
> @@ -2,23 +2,8 @@
> * Copyright(c) 2015 Cavium, Inc
> */
>
> -#ifndef _RTE_CRC_ARM64_H_
> -#define _RTE_CRC_ARM64_H_
> -
> -/**
> - * @file
> - *
> - * RTE CRC arm64 Hash
> - */
> -
> -#ifdef __cplusplus
> -extern "C" {
> -#endif
> -
> -#include <stdint.h>
> -#include <rte_cpuflags.h>
> -#include <rte_branch_prediction.h>
> -#include <rte_common.h>
> +#ifndef _HASH_CRC_ARM64_H_
> +#define _HASH_CRC_ARM64_H_
>
> static inline uint32_t
> crc32c_arm64_u8(uint8_t data, uint32_t init_val) @@ -61,40 +46,8 @@
> crc32c_arm64_u64(uint64_t data, uint32_t init_val) }
>
> /**
> - * Allow or disallow use of arm64 SIMD instrinsics for CRC32 hash
> - * calculation.
> - *
> - * @param alg
> - * An OR of following flags:
> - * - (CRC32_SW) Don't use arm64 crc intrinsics
> - * - (CRC32_ARM64) Use ARMv8 CRC intrinsic if available
> - *
> - */
> -static inline void
> -rte_hash_crc_set_alg(uint8_t alg)
> -{
> - switch (alg) {
> - case CRC32_ARM64:
> - if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_CRC32))
> - alg = CRC32_SW;
> - /* fall-through */
> - case CRC32_SW:
> - crc32_alg = alg;
> - /* fall-through */
> - default:
> - break;
> - }
> -}
> -
> -/* Setting the best available algorithm */
> -RTE_INIT(rte_hash_crc_init_alg)
> -{
> - rte_hash_crc_set_alg(CRC32_ARM64);
> -}
> -
> -/**
> - * Use single crc32 instruction to perform a hash on a 1 byte value.
> - * Fall back to software crc32 implementation in case arm64 crc intrinsics is
> + * Use single crc32 instruction to perform a hash on a byte value.
> + * Fall back to software crc32 implementation in case ARM CRC is
> * not supported
> *
> * @param data
> @@ -115,7 +68,7 @@ rte_hash_crc_1byte(uint8_t data, uint32_t init_val)
>
> /**
> * Use single crc32 instruction to perform a hash on a 2 bytes value.
> - * Fall back to software crc32 implementation in case arm64 crc intrinsics is
> + * Fall back to software crc32 implementation in case ARM CRC is
> * not supported
> *
> * @param data
> @@ -136,7 +89,7 @@ rte_hash_crc_2byte(uint16_t data, uint32_t init_val)
>
> /**
> * Use single crc32 instruction to perform a hash on a 4 byte value.
> - * Fall back to software crc32 implementation in case arm64 crc intrinsics is
> + * Fall back to software crc32 implementation in case ARM CRC is
> * not supported
> *
> * @param data
> @@ -157,7 +110,7 @@ rte_hash_crc_4byte(uint32_t data, uint32_t init_val)
>
> /**
> * Use single crc32 instruction to perform a hash on a 8 byte value.
> - * Fall back to software crc32 implementation in case arm64 crc intrinsics is
> + * Fall back to software crc32 implementation in case ARM CRC is
> * not supported
> *
> * @param data
> @@ -170,14 +123,10 @@ rte_hash_crc_4byte(uint32_t data, uint32_t init_val)
> static inline uint32_t rte_hash_crc_8byte(uint64_t data, uint32_t init_val) {
> - if (likely(crc32_alg == CRC32_ARM64))
> + if (likely(crc32_alg & CRC32_ARM64))
> return crc32c_arm64_u64(data, init_val);
>
> return crc32c_2words(data, init_val);
> }
>
> -#ifdef __cplusplus
> -}
> -#endif
> -
> -#endif /* _RTE_CRC_ARM64_H_ */
> +#endif /* _HASH_CRC_ARM64_H_ */
> diff --git a/lib/hash/rte_crc_generic.h b/lib/hash/rte_crc_generic.h new file
> mode 100644 index 0000000000..0c55947896
> --- /dev/null
> +++ b/lib/hash/rte_crc_generic.h
> @@ -0,0 +1,72 @@
> +/* SPDX-License-Identifier: BSD-3-Clause
> + * Copyright(c) 2022 Marvell.
> + */
> +
> +#ifndef _HASH_CRC_GENERIC_H_
> +#define _HASH_CRC_GENERIC_H_
> +
> +/**
> + * Software crc32 implementation for 1 byte value.
> + *
> + * @param data
> + * Data to perform hash on.
> + * @param init_val
> + * Value to initialise hash generator.
> + * @return
> + * 32bit calculated hash value.
> + */
> +static inline uint32_t
> +rte_hash_crc_1byte(uint8_t data, uint32_t init_val) {
> + return crc32c_1byte(data, init_val);
> +}
> +
> +/**
> + * Software crc32 implementation for 2 byte value.
> + *
> + * @param data
> + * Data to perform hash on.
> + * @param init_val
> + * Value to initialise hash generator.
> + * @return
> + * 32bit calculated hash value.
> + */
> +static inline uint32_t
> +rte_hash_crc_2byte(uint16_t data, uint32_t init_val) {
> + return crc32c_2bytes(data, init_val);
> +}
> +
> +/**
> + * Software crc32 implementation for 4 byte value.
> + *
> + * @param data
> + * Data to perform hash on.
> + * @param init_val
> + * Value to initialise hash generator.
> + * @return
> + * 32bit calculated hash value.
> + */
> +static inline uint32_t
> +rte_hash_crc_4byte(uint32_t data, uint32_t init_val) {
> + return crc32c_1word(data, init_val);
> +}
> +
> +/**
> + * Software crc32 implementation for 8 byte value.
> + *
> + * @param data
> + * Data to perform hash on.
> + * @param init_val
> + * Value to initialise hash generator.
> + * @return
> + * 32bit calculated hash value.
> + */
> +static inline uint32_t
> +rte_hash_crc_8byte(uint64_t data, uint32_t init_val) {
> + return crc32c_2words(data, init_val);
> +}
> +
> +#endif
> diff --git a/lib/hash/rte_crc_x86.h b/lib/hash/rte_crc_x86.h index
> b80a742afa..19eb3584e7 100644
> --- a/lib/hash/rte_crc_x86.h
> +++ b/lib/hash/rte_crc_x86.h
> @@ -59,4 +59,93 @@ crc32c_sse42_u64(uint64_t data, uint64_t init_val)
> return (uint32_t)init_val;
> }
>
> +/**
> + * Use single crc32 instruction to perform a hash on a byte value.
> + * Fall back to software crc32 implementation in case SSE4.2 is
> + * not supported
> + *
> + * @param data
> + * Data to perform hash on.
> + * @param init_val
> + * Value to initialise hash generator.
> + * @return
> + * 32bit calculated hash value.
> + */
> +static inline uint32_t
> +rte_hash_crc_1byte(uint8_t data, uint32_t init_val) {
> + if (likely(crc32_alg & CRC32_SSE42))
> + return crc32c_sse42_u8(data, init_val);
> +
> + return crc32c_1byte(data, init_val);
> +}
> +
> +/**
> + * Use single crc32 instruction to perform a hash on a 2 bytes value.
> + * Fall back to software crc32 implementation in case SSE4.2 is
> + * not supported
> + *
> + * @param data
> + * Data to perform hash on.
> + * @param init_val
> + * Value to initialise hash generator.
> + * @return
> + * 32bit calculated hash value.
> + */
> +static inline uint32_t
> +rte_hash_crc_2byte(uint16_t data, uint32_t init_val) {
> + if (likely(crc32_alg & CRC32_SSE42))
> + return crc32c_sse42_u16(data, init_val);
> +
> + return crc32c_2bytes(data, init_val);
> +}
> +
> +/**
> + * Use single crc32 instruction to perform a hash on a 4 byte value.
> + * Fall back to software crc32 implementation in case SSE4.2 is
> + * not supported
> + *
> + * @param data
> + * Data to perform hash on.
> + * @param init_val
> + * Value to initialise hash generator.
> + * @return
> + * 32bit calculated hash value.
> + */
> +static inline uint32_t
> +rte_hash_crc_4byte(uint32_t data, uint32_t init_val) {
> + if (likely(crc32_alg & CRC32_SSE42))
> + return crc32c_sse42_u32(data, init_val);
> +
> + return crc32c_1word(data, init_val);
> +}
> +
> +/**
> + * Use single crc32 instruction to perform a hash on a 8 byte value.
> + * Fall back to software crc32 implementation in case SSE4.2 is
> + * not supported
> + *
> + * @param data
> + * Data to perform hash on.
> + * @param init_val
> + * Value to initialise hash generator.
> + * @return
> + * 32bit calculated hash value.
> + */
> +static inline uint32_t
> +rte_hash_crc_8byte(uint64_t data, uint32_t init_val) { #ifdef
> +RTE_ARCH_X86_64
> + if (likely(crc32_alg == CRC32_SSE42_x64))
> + return crc32c_sse42_u64(data, init_val); #endif
> +
> + if (likely(crc32_alg & CRC32_SSE42))
> + return crc32c_sse42_u64_mimic(data, init_val);
> +
> + return crc32c_2words(data, init_val);
> +}
> +
> #endif
> diff --git a/lib/hash/rte_hash_crc.h b/lib/hash/rte_hash_crc.h index
> 308fdde414..cd9ec17082 100644
> --- a/lib/hash/rte_hash_crc.h
> +++ b/lib/hash/rte_hash_crc.h
> @@ -16,10 +16,12 @@ extern "C" {
> #endif
>
> #include <stdint.h>
> -#include <rte_config.h>
> -#include <rte_cpuflags.h>
> +
> #include <rte_branch_prediction.h>
> #include <rte_common.h>
> +#include <rte_config.h>
> +#include <rte_cpuflags.h>
> +#include <rte_log.h>
>
> #include "rte_crc_sw.h"
>
> @@ -33,136 +35,67 @@ static uint8_t crc32_alg = CRC32_SW;
>
> #if defined(RTE_ARCH_ARM64) && defined(__ARM_FEATURE_CRC32)
> #include "rte_crc_arm64.h"
> -#else
> +#elif defined(RTE_ARCH_X86)
> #include "rte_crc_x86.h"
> +#else
> +#include "rte_crc_generic.h"
> +#endif
>
> /**
> - * Allow or disallow use of SSE4.2 instrinsics for CRC32 hash
> + * Allow or disallow use of SSE4.2/ARMv8 intrinsics for CRC32 hash
> * calculation.
> *
> * @param alg
> * An OR of following flags:
> - * - (CRC32_SW) Don't use SSE4.2 intrinsics
> + * - (CRC32_SW) Don't use SSE4.2/ARMv8 intrinsics (default non-
> [x86/ARMv8])
> * - (CRC32_SSE42) Use SSE4.2 intrinsics if available
> - * - (CRC32_SSE42_x64) Use 64-bit SSE4.2 intrinsic if available (default)
> + * - (CRC32_SSE42_x64) Use 64-bit SSE4.2 intrinsic if available (default x86)
> + * - (CRC32_ARM64) Use ARMv8 CRC intrinsic if available (default ARMv8)
> *
> */
> static inline void
> rte_hash_crc_set_alg(uint8_t alg)
> {
> -#if defined(RTE_ARCH_X86)
> - if (alg == CRC32_SSE42_x64 &&
> - !rte_cpu_get_flag_enabled(RTE_CPUFLAG_EM64T))
> - alg = CRC32_SSE42;
> -#endif
> - crc32_alg = alg;
> -}
> + crc32_alg = CRC32_SW;
>
> -/* Setting the best available algorithm */
> -RTE_INIT(rte_hash_crc_init_alg)
> -{
> - rte_hash_crc_set_alg(CRC32_SSE42_x64);
> -}
> + if (alg == CRC32_SW)
> + return;
>
> -/**
> - * Use single crc32 instruction to perform a hash on a byte value.
> - * Fall back to software crc32 implementation in case SSE4.2 is
> - * not supported
> - *
> - * @param data
> - * Data to perform hash on.
> - * @param init_val
> - * Value to initialise hash generator.
> - * @return
> - * 32bit calculated hash value.
> - */
> -static inline uint32_t
> -rte_hash_crc_1byte(uint8_t data, uint32_t init_val) -{ -#if defined
> RTE_ARCH_X86
> - if (likely(crc32_alg & CRC32_SSE42))
> - return crc32c_sse42_u8(data, init_val);
> -#endif
> -
> - return crc32c_1byte(data, init_val);
> -}
> -
> -/**
> - * Use single crc32 instruction to perform a hash on a 2 bytes value.
> - * Fall back to software crc32 implementation in case SSE4.2 is
> - * not supported
> - *
> - * @param data
> - * Data to perform hash on.
> - * @param init_val
> - * Value to initialise hash generator.
> - * @return
> - * 32bit calculated hash value.
> - */
> -static inline uint32_t
> -rte_hash_crc_2byte(uint16_t data, uint32_t init_val) -{ #if defined
> RTE_ARCH_X86
> - if (likely(crc32_alg & CRC32_SSE42))
> - return crc32c_sse42_u16(data, init_val);
> + if (!(alg & CRC32_SSE42_x64))
> + RTE_LOG(WARNING, HASH,
> + "Unsupported CRC32 algorithm requested using
> CRC32_x64/CRC32_SSE42\n");
[Wang, Yipeng]
I have a question regarding this logic.
For the set_alg API, how about if user specify to use sse42 (not the 64bit version) algorithm on a em64 CPU, does it also warn "unsupported algorithm" and force user to use the x64 version?
It seems behaves differently than the current API definition.
> + if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_EM64T))
> + crc32_alg = CRC32_SSE42;
> + else
> + crc32_alg = CRC32_SSE42_x64;
> #endif
>
> - return crc32c_2bytes(data, init_val);
> -}
> -
On Wed, May 4, 2022 at 4:53 AM Wang, Yipeng1 <yipeng1.wang@intel.com> wrote:
> > static inline void
> > rte_hash_crc_set_alg(uint8_t alg)
> > {
> > + crc32_alg = CRC32_SW;
> >
> > + if (alg == CRC32_SW)
> > + return;
> >
> > + if (!(alg & CRC32_SSE42_x64))
> > + RTE_LOG(WARNING, HASH,
> > + "Unsupported CRC32 algorithm requested using
> > CRC32_x64/CRC32_SSE42\n");
> [Wang, Yipeng]
> I have a question regarding this logic.
> For the set_alg API, how about if user specify to use sse42 (not the 64bit version) algorithm on a em64 CPU, does it also warn "unsupported algorithm" and force user to use the x64 version?
> It seems behaves differently than the current API definition.
Can we conclude on this topic?
Thanks.
>
> > + if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_EM64T))
> > + crc32_alg = CRC32_SSE42;
> > + else
> > + crc32_alg = CRC32_SSE42_x64;
> > #endif
> >
@@ -13,6 +13,7 @@ indirect_headers += files(
'rte_crc_sw.h',
'rte_crc_x86.h',
'rte_crc_arm64.h',
+ 'rte_crc_generic.h',
'rte_thash_x86_gfni.h',
)
@@ -2,23 +2,8 @@
* Copyright(c) 2015 Cavium, Inc
*/
-#ifndef _RTE_CRC_ARM64_H_
-#define _RTE_CRC_ARM64_H_
-
-/**
- * @file
- *
- * RTE CRC arm64 Hash
- */
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdint.h>
-#include <rte_cpuflags.h>
-#include <rte_branch_prediction.h>
-#include <rte_common.h>
+#ifndef _HASH_CRC_ARM64_H_
+#define _HASH_CRC_ARM64_H_
static inline uint32_t
crc32c_arm64_u8(uint8_t data, uint32_t init_val)
@@ -61,40 +46,8 @@ crc32c_arm64_u64(uint64_t data, uint32_t init_val)
}
/**
- * Allow or disallow use of arm64 SIMD instrinsics for CRC32 hash
- * calculation.
- *
- * @param alg
- * An OR of following flags:
- * - (CRC32_SW) Don't use arm64 crc intrinsics
- * - (CRC32_ARM64) Use ARMv8 CRC intrinsic if available
- *
- */
-static inline void
-rte_hash_crc_set_alg(uint8_t alg)
-{
- switch (alg) {
- case CRC32_ARM64:
- if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_CRC32))
- alg = CRC32_SW;
- /* fall-through */
- case CRC32_SW:
- crc32_alg = alg;
- /* fall-through */
- default:
- break;
- }
-}
-
-/* Setting the best available algorithm */
-RTE_INIT(rte_hash_crc_init_alg)
-{
- rte_hash_crc_set_alg(CRC32_ARM64);
-}
-
-/**
- * Use single crc32 instruction to perform a hash on a 1 byte value.
- * Fall back to software crc32 implementation in case arm64 crc intrinsics is
+ * Use single crc32 instruction to perform a hash on a byte value.
+ * Fall back to software crc32 implementation in case ARM CRC is
* not supported
*
* @param data
@@ -115,7 +68,7 @@ rte_hash_crc_1byte(uint8_t data, uint32_t init_val)
/**
* Use single crc32 instruction to perform a hash on a 2 bytes value.
- * Fall back to software crc32 implementation in case arm64 crc intrinsics is
+ * Fall back to software crc32 implementation in case ARM CRC is
* not supported
*
* @param data
@@ -136,7 +89,7 @@ rte_hash_crc_2byte(uint16_t data, uint32_t init_val)
/**
* Use single crc32 instruction to perform a hash on a 4 byte value.
- * Fall back to software crc32 implementation in case arm64 crc intrinsics is
+ * Fall back to software crc32 implementation in case ARM CRC is
* not supported
*
* @param data
@@ -157,7 +110,7 @@ rte_hash_crc_4byte(uint32_t data, uint32_t init_val)
/**
* Use single crc32 instruction to perform a hash on a 8 byte value.
- * Fall back to software crc32 implementation in case arm64 crc intrinsics is
+ * Fall back to software crc32 implementation in case ARM CRC is
* not supported
*
* @param data
@@ -170,14 +123,10 @@ rte_hash_crc_4byte(uint32_t data, uint32_t init_val)
static inline uint32_t
rte_hash_crc_8byte(uint64_t data, uint32_t init_val)
{
- if (likely(crc32_alg == CRC32_ARM64))
+ if (likely(crc32_alg & CRC32_ARM64))
return crc32c_arm64_u64(data, init_val);
return crc32c_2words(data, init_val);
}
-#ifdef __cplusplus
-}
-#endif
-
-#endif /* _RTE_CRC_ARM64_H_ */
+#endif /* _HASH_CRC_ARM64_H_ */
new file mode 100644
@@ -0,0 +1,72 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2022 Marvell.
+ */
+
+#ifndef _HASH_CRC_GENERIC_H_
+#define _HASH_CRC_GENERIC_H_
+
+/**
+ * Software crc32 implementation for 1 byte value.
+ *
+ * @param data
+ * Data to perform hash on.
+ * @param init_val
+ * Value to initialise hash generator.
+ * @return
+ * 32bit calculated hash value.
+ */
+static inline uint32_t
+rte_hash_crc_1byte(uint8_t data, uint32_t init_val)
+{
+ return crc32c_1byte(data, init_val);
+}
+
+/**
+ * Software crc32 implementation for 2 byte value.
+ *
+ * @param data
+ * Data to perform hash on.
+ * @param init_val
+ * Value to initialise hash generator.
+ * @return
+ * 32bit calculated hash value.
+ */
+static inline uint32_t
+rte_hash_crc_2byte(uint16_t data, uint32_t init_val)
+{
+ return crc32c_2bytes(data, init_val);
+}
+
+/**
+ * Software crc32 implementation for 4 byte value.
+ *
+ * @param data
+ * Data to perform hash on.
+ * @param init_val
+ * Value to initialise hash generator.
+ * @return
+ * 32bit calculated hash value.
+ */
+static inline uint32_t
+rte_hash_crc_4byte(uint32_t data, uint32_t init_val)
+{
+ return crc32c_1word(data, init_val);
+}
+
+/**
+ * Software crc32 implementation for 8 byte value.
+ *
+ * @param data
+ * Data to perform hash on.
+ * @param init_val
+ * Value to initialise hash generator.
+ * @return
+ * 32bit calculated hash value.
+ */
+static inline uint32_t
+rte_hash_crc_8byte(uint64_t data, uint32_t init_val)
+{
+ return crc32c_2words(data, init_val);
+}
+
+#endif
@@ -59,4 +59,93 @@ crc32c_sse42_u64(uint64_t data, uint64_t init_val)
return (uint32_t)init_val;
}
+/**
+ * Use single crc32 instruction to perform a hash on a byte value.
+ * Fall back to software crc32 implementation in case SSE4.2 is
+ * not supported
+ *
+ * @param data
+ * Data to perform hash on.
+ * @param init_val
+ * Value to initialise hash generator.
+ * @return
+ * 32bit calculated hash value.
+ */
+static inline uint32_t
+rte_hash_crc_1byte(uint8_t data, uint32_t init_val)
+{
+ if (likely(crc32_alg & CRC32_SSE42))
+ return crc32c_sse42_u8(data, init_val);
+
+ return crc32c_1byte(data, init_val);
+}
+
+/**
+ * Use single crc32 instruction to perform a hash on a 2 bytes value.
+ * Fall back to software crc32 implementation in case SSE4.2 is
+ * not supported
+ *
+ * @param data
+ * Data to perform hash on.
+ * @param init_val
+ * Value to initialise hash generator.
+ * @return
+ * 32bit calculated hash value.
+ */
+static inline uint32_t
+rte_hash_crc_2byte(uint16_t data, uint32_t init_val)
+{
+ if (likely(crc32_alg & CRC32_SSE42))
+ return crc32c_sse42_u16(data, init_val);
+
+ return crc32c_2bytes(data, init_val);
+}
+
+/**
+ * Use single crc32 instruction to perform a hash on a 4 byte value.
+ * Fall back to software crc32 implementation in case SSE4.2 is
+ * not supported
+ *
+ * @param data
+ * Data to perform hash on.
+ * @param init_val
+ * Value to initialise hash generator.
+ * @return
+ * 32bit calculated hash value.
+ */
+static inline uint32_t
+rte_hash_crc_4byte(uint32_t data, uint32_t init_val)
+{
+ if (likely(crc32_alg & CRC32_SSE42))
+ return crc32c_sse42_u32(data, init_val);
+
+ return crc32c_1word(data, init_val);
+}
+
+/**
+ * Use single crc32 instruction to perform a hash on a 8 byte value.
+ * Fall back to software crc32 implementation in case SSE4.2 is
+ * not supported
+ *
+ * @param data
+ * Data to perform hash on.
+ * @param init_val
+ * Value to initialise hash generator.
+ * @return
+ * 32bit calculated hash value.
+ */
+static inline uint32_t
+rte_hash_crc_8byte(uint64_t data, uint32_t init_val)
+{
+#ifdef RTE_ARCH_X86_64
+ if (likely(crc32_alg == CRC32_SSE42_x64))
+ return crc32c_sse42_u64(data, init_val);
+#endif
+
+ if (likely(crc32_alg & CRC32_SSE42))
+ return crc32c_sse42_u64_mimic(data, init_val);
+
+ return crc32c_2words(data, init_val);
+}
+
#endif
@@ -16,10 +16,12 @@ extern "C" {
#endif
#include <stdint.h>
-#include <rte_config.h>
-#include <rte_cpuflags.h>
+
#include <rte_branch_prediction.h>
#include <rte_common.h>
+#include <rte_config.h>
+#include <rte_cpuflags.h>
+#include <rte_log.h>
#include "rte_crc_sw.h"
@@ -33,136 +35,67 @@ static uint8_t crc32_alg = CRC32_SW;
#if defined(RTE_ARCH_ARM64) && defined(__ARM_FEATURE_CRC32)
#include "rte_crc_arm64.h"
-#else
+#elif defined(RTE_ARCH_X86)
#include "rte_crc_x86.h"
+#else
+#include "rte_crc_generic.h"
+#endif
/**
- * Allow or disallow use of SSE4.2 instrinsics for CRC32 hash
+ * Allow or disallow use of SSE4.2/ARMv8 intrinsics for CRC32 hash
* calculation.
*
* @param alg
* An OR of following flags:
- * - (CRC32_SW) Don't use SSE4.2 intrinsics
+ * - (CRC32_SW) Don't use SSE4.2/ARMv8 intrinsics (default non-[x86/ARMv8])
* - (CRC32_SSE42) Use SSE4.2 intrinsics if available
- * - (CRC32_SSE42_x64) Use 64-bit SSE4.2 intrinsic if available (default)
+ * - (CRC32_SSE42_x64) Use 64-bit SSE4.2 intrinsic if available (default x86)
+ * - (CRC32_ARM64) Use ARMv8 CRC intrinsic if available (default ARMv8)
*
*/
static inline void
rte_hash_crc_set_alg(uint8_t alg)
{
-#if defined(RTE_ARCH_X86)
- if (alg == CRC32_SSE42_x64 &&
- !rte_cpu_get_flag_enabled(RTE_CPUFLAG_EM64T))
- alg = CRC32_SSE42;
-#endif
- crc32_alg = alg;
-}
+ crc32_alg = CRC32_SW;
-/* Setting the best available algorithm */
-RTE_INIT(rte_hash_crc_init_alg)
-{
- rte_hash_crc_set_alg(CRC32_SSE42_x64);
-}
+ if (alg == CRC32_SW)
+ return;
-/**
- * Use single crc32 instruction to perform a hash on a byte value.
- * Fall back to software crc32 implementation in case SSE4.2 is
- * not supported
- *
- * @param data
- * Data to perform hash on.
- * @param init_val
- * Value to initialise hash generator.
- * @return
- * 32bit calculated hash value.
- */
-static inline uint32_t
-rte_hash_crc_1byte(uint8_t data, uint32_t init_val)
-{
-#if defined RTE_ARCH_X86
- if (likely(crc32_alg & CRC32_SSE42))
- return crc32c_sse42_u8(data, init_val);
-#endif
-
- return crc32c_1byte(data, init_val);
-}
-
-/**
- * Use single crc32 instruction to perform a hash on a 2 bytes value.
- * Fall back to software crc32 implementation in case SSE4.2 is
- * not supported
- *
- * @param data
- * Data to perform hash on.
- * @param init_val
- * Value to initialise hash generator.
- * @return
- * 32bit calculated hash value.
- */
-static inline uint32_t
-rte_hash_crc_2byte(uint16_t data, uint32_t init_val)
-{
#if defined RTE_ARCH_X86
- if (likely(crc32_alg & CRC32_SSE42))
- return crc32c_sse42_u16(data, init_val);
+ if (!(alg & CRC32_SSE42_x64))
+ RTE_LOG(WARNING, HASH,
+ "Unsupported CRC32 algorithm requested using CRC32_x64/CRC32_SSE42\n");
+ if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_EM64T))
+ crc32_alg = CRC32_SSE42;
+ else
+ crc32_alg = CRC32_SSE42_x64;
#endif
- return crc32c_2bytes(data, init_val);
-}
-
-/**
- * Use single crc32 instruction to perform a hash on a 4 byte value.
- * Fall back to software crc32 implementation in case SSE4.2 is
- * not supported
- *
- * @param data
- * Data to perform hash on.
- * @param init_val
- * Value to initialise hash generator.
- * @return
- * 32bit calculated hash value.
- */
-static inline uint32_t
-rte_hash_crc_4byte(uint32_t data, uint32_t init_val)
-{
-#if defined RTE_ARCH_X86
- if (likely(crc32_alg & CRC32_SSE42))
- return crc32c_sse42_u32(data, init_val);
+#if defined RTE_ARCH_ARM64
+ if (!(alg & CRC32_ARM64))
+ RTE_LOG(WARNING, HASH,
+ "Unsupported CRC32 algorithm requested using CRC32_ARM64\n");
+ if (rte_cpu_get_flag_enabled(RTE_CPUFLAG_CRC32))
+ crc32_alg = CRC32_ARM64;
#endif
- return crc32c_1word(data, init_val);
+ if (crc32_alg == CRC32_SW)
+ RTE_LOG(WARNING, HASH,
+ "Unsupported CRC32 algorithm requested using CRC32_SW\n");
}
-/**
- * Use single crc32 instruction to perform a hash on a 8 byte value.
- * Fall back to software crc32 implementation in case SSE4.2 is
- * not supported
- *
- * @param data
- * Data to perform hash on.
- * @param init_val
- * Value to initialise hash generator.
- * @return
- * 32bit calculated hash value.
- */
-static inline uint32_t
-rte_hash_crc_8byte(uint64_t data, uint32_t init_val)
+/* Setting the best available algorithm */
+RTE_INIT(rte_hash_crc_init_alg)
{
-#ifdef RTE_ARCH_X86_64
- if (likely(crc32_alg == CRC32_SSE42_x64))
- return crc32c_sse42_u64(data, init_val);
-#endif
-
-#if defined RTE_ARCH_X86
- if (likely(crc32_alg & CRC32_SSE42))
- return crc32c_sse42_u64_mimic(data, init_val);
+#if defined(RTE_ARCH_X86)
+ rte_hash_crc_set_alg(CRC32_SSE42_x64);
+#elif defined(RTE_ARCH_ARM64) && defined(__ARM_FEATURE_CRC32)
+ rte_hash_crc_set_alg(CRC32_ARM64);
+#else
+ rte_hash_crc_set_alg(CRC32_SW);
#endif
-
- return crc32c_2words(data, init_val);
}
-#endif
-
/**
* Calculate CRC32 hash on user-supplied byte array.
*