eal: allow worker lcore stacks to be allocated from hugepage memory
Checks
Commit Message
Add support for using hugepages for worker lcore stack memory. The
intent is to improve performance by reducing stack memory related TLB
misses and also by using memory local to the NUMA node of each lcore.
EAL option '--huge-worker-stack [stack-size-kbytes]' is added to allow
the feature to be enabled at runtime. If the size is not specified,
the system pthread stack size will be used.
Signed-off-by: Don Wallwork <donw@xsightlabs.com>
---
lib/eal/common/eal_common_options.c | 31 ++++++++++++++
lib/eal/common/eal_internal_cfg.h | 4 ++
lib/eal/common/eal_options.h | 2 +
lib/eal/linux/eal.c | 65 ++++++++++++++++++++++++++++-
4 files changed, 100 insertions(+), 2 deletions(-)
Comments
> From: Don Wallwork [mailto:donw@xsightlabs.com]
> Sent: Monday, 2 May 2022 16.11
>
> Add support for using hugepages for worker lcore stack memory. The
> intent is to improve performance by reducing stack memory related TLB
> misses and also by using memory local to the NUMA node of each lcore.
>
> EAL option '--huge-worker-stack [stack-size-kbytes]' is added to allow
> the feature to be enabled at runtime. If the size is not specified,
> the system pthread stack size will be used.
>
> Signed-off-by: Don Wallwork <donw@xsightlabs.com>
> ---
> lib/eal/common/eal_common_options.c | 31 ++++++++++++++
> lib/eal/common/eal_internal_cfg.h | 4 ++
> lib/eal/common/eal_options.h | 2 +
> lib/eal/linux/eal.c | 65 ++++++++++++++++++++++++++++-
> 4 files changed, 100 insertions(+), 2 deletions(-)
>
Acked-by: Morten Brørup <mb@smartsharesystems.com>
> -----Original Message-----
> From: Don Wallwork <donw@xsightlabs.com>
> Sent: Monday, May 2, 2022 22:11
> To: dev@dpdk.org
> Cc: donw@xsightlabs.com; stephen@networkplumber.org; mb@smartsharesystems.com; Burakov, Anatoly
> <anatoly.burakov@intel.com>; dmitry.kozliuk@gmail.com; Richardson, Bruce <bruce.richardson@intel.com>;
> Honnappa.Nagarahalli@arm.com; nd@arm.com
> Subject: [PATCH] eal: allow worker lcore stacks to be allocated from hugepage memory
>
> Add support for using hugepages for worker lcore stack memory. The
> intent is to improve performance by reducing stack memory related TLB
> misses and also by using memory local to the NUMA node of each lcore.
>
> EAL option '--huge-worker-stack [stack-size-kbytes]' is added to allow
> the feature to be enabled at runtime. If the size is not specified,
> the system pthread stack size will be used.
>
> Signed-off-by: Don Wallwork <donw@xsightlabs.com>
> ---
> lib/eal/common/eal_common_options.c | 31 ++++++++++++++
> lib/eal/common/eal_internal_cfg.h | 4 ++
> lib/eal/common/eal_options.h | 2 +
> lib/eal/linux/eal.c | 65 ++++++++++++++++++++++++++++-
> 4 files changed, 100 insertions(+), 2 deletions(-)
>
> diff --git a/lib/eal/common/eal_common_options.c b/lib/eal/common/eal_common_options.c
> index f247a42455..be9db9ee37 100644
> --- a/lib/eal/common/eal_common_options.c
> +++ b/lib/eal/common/eal_common_options.c
> @@ -103,6 +103,7 @@ eal_long_options[] = {
> {OPT_TELEMETRY, 0, NULL, OPT_TELEMETRY_NUM },
> {OPT_NO_TELEMETRY, 0, NULL, OPT_NO_TELEMETRY_NUM },
> {OPT_FORCE_MAX_SIMD_BITWIDTH, 1, NULL, OPT_FORCE_MAX_SIMD_BITWIDTH_NUM},
> + {OPT_HUGE_WORKER_STACK, 2, NULL, OPT_HUGE_WORKER_STACK_NUM },
>
> {0, 0, NULL, 0 }
> };
> @@ -1618,6 +1619,22 @@ eal_parse_huge_unlink(const char *arg, struct hugepage_file_discipline *out)
> return -1;
> }
>
> +static int
> +eal_parse_huge_worker_stack(const char *arg, size_t *huge_worker_stack_size)
> +{
> + size_t worker_stack_size;
> + if (arg == NULL) {
> + *huge_worker_stack_size = USE_OS_STACK_SIZE;
> + return 0;
> + }
> + worker_stack_size = atoi(arg);
> + if (worker_stack_size == 0)
> + return -1;
Should we also to check "worker_stack_size *1024 < PTHREAD_STACK_MIN" ?
> +
> + *huge_worker_stack_size = worker_stack_size * 1024;
> + return 0;
> +}
> +
> --
> 2.17.1
On 5/3/2022 9:08 AM, Wang, Haiyue wrote:
>> -----Original Message-----
>> From: Don Wallwork <donw@xsightlabs.com>
>> Sent: Monday, May 2, 2022 22:11
>> To: dev@dpdk.org
>> Cc: donw@xsightlabs.com; stephen@networkplumber.org; mb@smartsharesystems.com; Burakov, Anatoly
>> <anatoly.burakov@intel.com>; dmitry.kozliuk@gmail.com; Richardson, Bruce <bruce.richardson@intel.com>;
>> Honnappa.Nagarahalli@arm.com; nd@arm.com
>> Subject: [PATCH] eal: allow worker lcore stacks to be allocated from hugepage memory
>>
>> Add support for using hugepages for worker lcore stack memory. The
>> intent is to improve performance by reducing stack memory related TLB
>> misses and also by using memory local to the NUMA node of each lcore.
>>
>> EAL option '--huge-worker-stack [stack-size-kbytes]' is added to allow
>> the feature to be enabled at runtime. If the size is not specified,
>> the system pthread stack size will be used.
>>
>> Signed-off-by: Don Wallwork <donw@xsightlabs.com>
>> ---
>> lib/eal/common/eal_common_options.c | 31 ++++++++++++++
>> lib/eal/common/eal_internal_cfg.h | 4 ++
>> lib/eal/common/eal_options.h | 2 +
>> lib/eal/linux/eal.c | 65 ++++++++++++++++++++++++++++-
>> 4 files changed, 100 insertions(+), 2 deletions(-)
>>
>> diff --git a/lib/eal/common/eal_common_options.c b/lib/eal/common/eal_common_options.c
>> index f247a42455..be9db9ee37 100644
>> --- a/lib/eal/common/eal_common_options.c
>> +++ b/lib/eal/common/eal_common_options.c
>> @@ -103,6 +103,7 @@ eal_long_options[] = {
>> {OPT_TELEMETRY, 0, NULL, OPT_TELEMETRY_NUM },
>> {OPT_NO_TELEMETRY, 0, NULL, OPT_NO_TELEMETRY_NUM },
>> {OPT_FORCE_MAX_SIMD_BITWIDTH, 1, NULL, OPT_FORCE_MAX_SIMD_BITWIDTH_NUM},
>> + {OPT_HUGE_WORKER_STACK, 2, NULL, OPT_HUGE_WORKER_STACK_NUM },
>>
>> {0, 0, NULL, 0 }
>> };
>> @@ -1618,6 +1619,22 @@ eal_parse_huge_unlink(const char *arg, struct hugepage_file_discipline *out)
>> return -1;
>> }
>>
>> +static int
>> +eal_parse_huge_worker_stack(const char *arg, size_t *huge_worker_stack_size)
>> +{
>> + size_t worker_stack_size;
>> + if (arg == NULL) {
>> + *huge_worker_stack_size = USE_OS_STACK_SIZE;
>> + return 0;
>> + }
>> + worker_stack_size = atoi(arg);
>> + if (worker_stack_size == 0)
>> + return -1;
> Should we also to check "worker_stack_size *1024 < PTHREAD_STACK_MIN" ?
This may be too restrictive in certain environments. For example,
memory constrained platforms may require a smaller worker stack size
than this limit would allow.
>> +
>> + *huge_worker_stack_size = worker_stack_size * 1024;
>> + return 0;
>> +}
>> +
>
>> --
>> 2.17.1
> -----Original Message-----
> From: Don Wallwork <donw@xsightlabs.com>
> Sent: Wednesday, May 4, 2022 03:47
> To: Wang, Haiyue <haiyue.wang@intel.com>; dev@dpdk.org
> Cc: stephen@networkplumber.org; mb@smartsharesystems.com; Burakov, Anatoly <anatoly.burakov@intel.com>;
> dmitry.kozliuk@gmail.com; Richardson, Bruce <bruce.richardson@intel.com>; Honnappa.Nagarahalli@arm.com;
> nd@arm.com
> Subject: Re: [PATCH] eal: allow worker lcore stacks to be allocated from hugepage memory
>
> On 5/3/2022 9:08 AM, Wang, Haiyue wrote:
> >> -----Original Message-----
> >> From: Don Wallwork <donw@xsightlabs.com>
> >> Sent: Monday, May 2, 2022 22:11
> >> To: dev@dpdk.org
> >> Cc: donw@xsightlabs.com; stephen@networkplumber.org; mb@smartsharesystems.com; Burakov, Anatoly
> >> <anatoly.burakov@intel.com>; dmitry.kozliuk@gmail.com; Richardson, Bruce
> <bruce.richardson@intel.com>;
> >> Honnappa.Nagarahalli@arm.com; nd@arm.com
> >> Subject: [PATCH] eal: allow worker lcore stacks to be allocated from hugepage memory
> >>
> >> Add support for using hugepages for worker lcore stack memory. The
> >> intent is to improve performance by reducing stack memory related TLB
> >> misses and also by using memory local to the NUMA node of each lcore.
> >>
> >> EAL option '--huge-worker-stack [stack-size-kbytes]' is added to allow
> >> the feature to be enabled at runtime. If the size is not specified,
> >> the system pthread stack size will be used.
> >>
> >> Signed-off-by: Don Wallwork <donw@xsightlabs.com>
> >> ---
> >> lib/eal/common/eal_common_options.c | 31 ++++++++++++++
> >> lib/eal/common/eal_internal_cfg.h | 4 ++
> >> lib/eal/common/eal_options.h | 2 +
> >> lib/eal/linux/eal.c | 65 ++++++++++++++++++++++++++++-
> >> 4 files changed, 100 insertions(+), 2 deletions(-)
> >>
> >> diff --git a/lib/eal/common/eal_common_options.c b/lib/eal/common/eal_common_options.c
> >> index f247a42455..be9db9ee37 100644
> >> --- a/lib/eal/common/eal_common_options.c
> >> +++ b/lib/eal/common/eal_common_options.c
> >> @@ -103,6 +103,7 @@ eal_long_options[] = {
> >> {OPT_TELEMETRY, 0, NULL, OPT_TELEMETRY_NUM },
> >> {OPT_NO_TELEMETRY, 0, NULL, OPT_NO_TELEMETRY_NUM },
> >> {OPT_FORCE_MAX_SIMD_BITWIDTH, 1, NULL, OPT_FORCE_MAX_SIMD_BITWIDTH_NUM},
> >> + {OPT_HUGE_WORKER_STACK, 2, NULL, OPT_HUGE_WORKER_STACK_NUM },
> >>
> >> {0, 0, NULL, 0 }
> >> };
> >> @@ -1618,6 +1619,22 @@ eal_parse_huge_unlink(const char *arg, struct hugepage_file_discipline *out)
> >> return -1;
> >> }
> >>
> >> +static int
> >> +eal_parse_huge_worker_stack(const char *arg, size_t *huge_worker_stack_size)
> >> +{
> >> + size_t worker_stack_size;
> >> + if (arg == NULL) {
> >> + *huge_worker_stack_size = USE_OS_STACK_SIZE;
> >> + return 0;
> >> + }
> >> + worker_stack_size = atoi(arg);
> >> + if (worker_stack_size == 0)
> >> + return -1;
> > Should we also to check "worker_stack_size *1024 < PTHREAD_STACK_MIN" ?
> This may be too restrictive in certain environments. For example,
> memory constrained platforms may require a smaller worker stack size
> than this limit would allow.
Understood, thanks.
> >> +
> >> + *huge_worker_stack_size = worker_stack_size * 1024;
> >> + return 0;
> >> +}
> >> +
> >
> >> --
> >> 2.17.1
@@ -103,6 +103,7 @@ eal_long_options[] = {
{OPT_TELEMETRY, 0, NULL, OPT_TELEMETRY_NUM },
{OPT_NO_TELEMETRY, 0, NULL, OPT_NO_TELEMETRY_NUM },
{OPT_FORCE_MAX_SIMD_BITWIDTH, 1, NULL, OPT_FORCE_MAX_SIMD_BITWIDTH_NUM},
+ {OPT_HUGE_WORKER_STACK, 2, NULL, OPT_HUGE_WORKER_STACK_NUM },
{0, 0, NULL, 0 }
};
@@ -1618,6 +1619,22 @@ eal_parse_huge_unlink(const char *arg, struct hugepage_file_discipline *out)
return -1;
}
+static int
+eal_parse_huge_worker_stack(const char *arg, size_t *huge_worker_stack_size)
+{
+ size_t worker_stack_size;
+ if (arg == NULL) {
+ *huge_worker_stack_size = USE_OS_STACK_SIZE;
+ return 0;
+ }
+ worker_stack_size = atoi(arg);
+ if (worker_stack_size == 0)
+ return -1;
+
+ *huge_worker_stack_size = worker_stack_size * 1024;
+ return 0;
+}
+
int
eal_parse_common_option(int opt, const char *optarg,
struct internal_config *conf)
@@ -1921,6 +1938,15 @@ eal_parse_common_option(int opt, const char *optarg,
}
break;
+ case OPT_HUGE_WORKER_STACK_NUM:
+ if (eal_parse_huge_worker_stack(optarg,
+ &conf->huge_worker_stack_size) < 0) {
+ RTE_LOG(ERR, EAL, "invalid parameter for --"
+ OPT_HUGE_WORKER_STACK"\n");
+ return -1;
+ }
+ break;
+
/* don't know what to do, leave this to caller */
default:
return 1;
@@ -2235,5 +2261,10 @@ eal_common_usage(void)
" --"OPT_NO_PCI" Disable PCI\n"
" --"OPT_NO_HPET" Disable HPET\n"
" --"OPT_NO_SHCONF" No shared config (mmap'd files)\n"
+ " --"OPT_HUGE_WORKER_STACK"[=size]\n"
+ " Allocate worker thread stacks from\n"
+ " hugepage memory. Size is in units of\n"
+ " kbytes and defaults to system thread\n"
+ " stack size if not specified.\n"
"\n", RTE_MAX_LCORE);
}
@@ -48,6 +48,9 @@ struct hugepage_file_discipline {
bool unlink_existing;
};
+/** Worker hugepage stack size should default to OS value. */
+#define USE_OS_STACK_SIZE ((size_t)~0)
+
/**
* internal configuration
*/
@@ -102,6 +105,7 @@ struct internal_config {
unsigned int no_telemetry; /**< true to disable Telemetry */
struct simd_bitwidth max_simd_bitwidth;
/**< max simd bitwidth path to use */
+ size_t huge_worker_stack_size; /**< worker thread stack size in bytes */
};
void eal_reset_internal_config(struct internal_config *internal_cfg);
@@ -87,6 +87,8 @@ enum {
OPT_NO_TELEMETRY_NUM,
#define OPT_FORCE_MAX_SIMD_BITWIDTH "force-max-simd-bitwidth"
OPT_FORCE_MAX_SIMD_BITWIDTH_NUM,
+#define OPT_HUGE_WORKER_STACK "huge-worker-stack"
+ OPT_HUGE_WORKER_STACK_NUM,
OPT_LONG_MAX_NUM
};
@@ -1144,8 +1144,69 @@ rte_eal_init(int argc, char **argv)
lcore_config[i].state = WAIT;
/* create a thread for each lcore */
- ret = pthread_create(&lcore_config[i].thread_id, NULL,
- eal_thread_loop, (void *)(uintptr_t)i);
+ if (internal_conf->huge_worker_stack_size == 0) {
+ ret = pthread_create(&lcore_config[i].thread_id, NULL,
+ eal_thread_loop,
+ (void *)(uintptr_t)i);
+ } else {
+ /* Allocate NUMA aware stack memory and set
+ * pthread attributes
+ */
+ pthread_attr_t attr;
+ size_t stack_size;
+ void *stack_ptr;
+
+ if (pthread_attr_init(&attr) != 0) {
+ rte_eal_init_alert("Cannot init pthread "
+ "attributes");
+ rte_errno = EFAULT;
+ return -1;
+ }
+ if (internal_conf->huge_worker_stack_size ==
+ USE_OS_STACK_SIZE) {
+ if (pthread_attr_getstacksize(&attr,
+ &stack_size) != 0) {
+ rte_errno = EFAULT;
+ return -1;
+ }
+ } else {
+ stack_size =
+ internal_conf->huge_worker_stack_size;
+ }
+ stack_ptr =
+ rte_zmalloc_socket("lcore_stack",
+ stack_size,
+ stack_size,
+ rte_lcore_to_socket_id(i));
+
+ if (stack_ptr == NULL) {
+ rte_eal_init_alert("Cannot allocate stack "
+ "memory for worker lcore");
+ rte_errno = ENOMEM;
+ return -1;
+ }
+
+ if (pthread_attr_setstack(&attr,
+ stack_ptr,
+ stack_size) != 0) {
+ rte_eal_init_alert("Cannot set pthread "
+ "stack attributes");
+ rte_errno = EFAULT;
+ return -1;
+ }
+
+ /* create a thread for each lcore */
+ ret = pthread_create(&lcore_config[i].thread_id, &attr,
+ eal_thread_loop,
+ (void *)(uintptr_t)i);
+
+ if (pthread_attr_destroy(&attr) != 0) {
+ rte_eal_init_alert("Cannot destroy pthread "
+ "attributes");
+ rte_errno = EFAULT;
+ return -1;
+ }
+ }
if (ret != 0)
rte_panic("Cannot create thread\n");