@@ -52,3 +52,7 @@
; Ignore visibility fix of local functions in experimental gpudev library
[suppress_file]
soname_regexp = ^librte_gpudev\.
+
+; Ignore libabigail false-positive in clang builds, after moving code.
+[suppress_function]
+ name = rte_eal_remote_launch
@@ -5,11 +5,13 @@
#include <errno.h>
#include <rte_launch.h>
+#include <rte_eal_trace.h>
#include <rte_atomic.h>
#include <rte_pause.h>
#include <rte_lcore.h>
#include "eal_private.h"
+#include "eal_thread.h"
/*
* Wait until a lcore finished its job.
@@ -18,12 +20,44 @@ int
rte_eal_wait_lcore(unsigned worker_id)
{
while (__atomic_load_n(&lcore_config[worker_id].state,
- __ATOMIC_ACQUIRE) != WAIT)
+ __ATOMIC_ACQUIRE) != WAIT)
rte_pause();
return lcore_config[worker_id].ret;
}
+/*
+ * Send a message to a worker lcore identified by worker_id to call a
+ * function f with argument arg. Once the execution is done, the
+ * remote lcore switches to WAIT state.
+ */
+int
+rte_eal_remote_launch(lcore_function_t *f, void *arg, unsigned int worker_id)
+{
+ int rc = -EBUSY;
+
+ /* Check if the worker is in 'WAIT' state. Use acquire order
+ * since 'state' variable is used as the guard variable.
+ */
+ if (__atomic_load_n(&lcore_config[worker_id].state,
+ __ATOMIC_ACQUIRE) != WAIT)
+ goto finish;
+
+ lcore_config[worker_id].arg = arg;
+ /* Ensure that all the memory operations are completed
+ * before the worker thread starts running the function.
+ * Use worker thread function as the guard variable.
+ */
+ __atomic_store_n(&lcore_config[worker_id].f, f, __ATOMIC_RELEASE);
+
+ eal_thread_wake_worker(worker_id);
+ rc = 0;
+
+finish:
+ rte_eal_trace_thread_remote_launch(f, arg, worker_id, rc);
+ return rc;
+}
+
/*
* Check that every WORKER lcores are in WAIT state, then call
* rte_eal_remote_launch() for all of them. If call_main is true
@@ -9,6 +9,7 @@
#include <assert.h>
#include <string.h>
+#include <rte_eal_trace.h>
#include <rte_errno.h>
#include <rte_lcore.h>
#include <rte_log.h>
@@ -163,6 +164,68 @@ __rte_thread_uninit(void)
RTE_PER_LCORE(_lcore_id) = LCORE_ID_ANY;
}
+/* main loop of threads */
+__rte_noreturn void *
+eal_thread_loop(void *arg)
+{
+ unsigned int lcore_id = (uintptr_t)arg;
+ char cpuset[RTE_CPU_AFFINITY_STR_LEN];
+ int ret;
+
+ __rte_thread_init(lcore_id, &lcore_config[lcore_id].cpuset);
+
+ ret = eal_thread_dump_current_affinity(cpuset, sizeof(cpuset));
+ RTE_LOG(DEBUG, EAL, "lcore %u is ready (tid=%zx;cpuset=[%s%s])\n",
+ lcore_id, (uintptr_t)pthread_self(), cpuset,
+ ret == 0 ? "" : "...");
+
+ rte_eal_trace_thread_lcore_ready(lcore_id, cpuset);
+
+ /* read on our pipe to get commands */
+ while (1) {
+ lcore_function_t *f;
+ void *fct_arg;
+
+ eal_thread_wait_command();
+
+ /* Set the state to 'RUNNING'. Use release order
+ * since 'state' variable is used as the guard variable.
+ */
+ __atomic_store_n(&lcore_config[lcore_id].state, RUNNING,
+ __ATOMIC_RELEASE);
+
+ eal_thread_ack_command();
+
+ /* Load 'f' with acquire order to ensure that
+ * the memory operations from the main thread
+ * are accessed only after update to 'f' is visible.
+ * Wait till the update to 'f' is visible to the worker.
+ */
+ while ((f = __atomic_load_n(&lcore_config[lcore_id].f,
+ __ATOMIC_ACQUIRE)) == NULL)
+ rte_pause();
+
+ /* call the function and store the return value */
+ fct_arg = lcore_config[lcore_id].arg;
+ ret = f(fct_arg);
+ lcore_config[lcore_id].ret = ret;
+ lcore_config[lcore_id].f = NULL;
+ lcore_config[lcore_id].arg = NULL;
+
+ /* Store the state with release order to ensure that
+ * the memory operations from the worker thread
+ * are completed before the state is updated.
+ * Use 'state' as the guard variable.
+ */
+ __atomic_store_n(&lcore_config[lcore_id].state, WAIT,
+ __ATOMIC_RELEASE);
+ }
+
+ /* never reached */
+ /* pthread_exit(NULL); */
+ /* return NULL; */
+}
+
enum __rte_ctrl_thread_status {
CTRL_THREAD_LAUNCHING, /* Yet to call pthread_create function */
CTRL_THREAD_RUNNING, /* Control thread is running successfully */
@@ -58,4 +58,31 @@ eal_thread_dump_affinity(rte_cpuset_t *cpuset, char *str, unsigned int size);
int
eal_thread_dump_current_affinity(char *str, unsigned int size);
+/**
+ * Called by the main thread to wake up a worker in 'WAIT' state.
+ * This function blocks until the worker acknowledge it started processing a
+ * new command.
+ * This function is private to EAL.
+ *
+ * @param worker_id
+ * The lcore_id of a worker thread.
+ */
+void
+eal_thread_wake_worker(unsigned int worker_id);
+
+/**
+ * Called by a worker thread to sleep after entering 'WAIT' state.
+ * This function is private to EAL.
+ */
+void
+eal_thread_wait_command(void);
+
+/**
+ * Called by a worker thread to acknowledge new command after leaving 'WAIT'
+ * state.
+ * This function is private to EAL.
+ */
+void
+eal_thread_ack_command(void);
+
#endif /* EAL_THREAD_H */
@@ -579,7 +579,6 @@ int
rte_eal_init(int argc, char **argv)
{
int i, fctret, ret;
- pthread_t thread_id;
static uint32_t run_once;
uint32_t has_run = 0;
char cpuset[RTE_CPU_AFFINITY_STR_LEN];
@@ -604,8 +603,6 @@ rte_eal_init(int argc, char **argv)
return -1;
}
- thread_id = pthread_self();
-
eal_reset_internal_config(internal_conf);
/* clone argv to report out later in telemetry */
@@ -794,8 +791,8 @@ rte_eal_init(int argc, char **argv)
ret = eal_thread_dump_current_affinity(cpuset, sizeof(cpuset));
- RTE_LOG(DEBUG, EAL, "Main lcore %u is ready (tid=%p;cpuset=[%s%s])\n",
- config->main_lcore, thread_id, cpuset,
+ RTE_LOG(DEBUG, EAL, "Main lcore %u is ready (tid=%zx;cpuset=[%s%s])\n",
+ config->main_lcore, (uintptr_t)pthread_self(), cpuset,
ret == 0 ? "" : "...");
RTE_LCORE_FOREACH_WORKER(i) {
@@ -20,138 +20,10 @@
#include <rte_per_lcore.h>
#include <rte_eal.h>
#include <rte_lcore.h>
-#include <rte_eal_trace.h>
#include "eal_private.h"
#include "eal_thread.h"
-/*
- * Send a message to a worker lcore identified by worker_id to call a
- * function f with argument arg. Once the execution is done, the
- * remote lcore switches to WAIT state.
- */
-int
-rte_eal_remote_launch(int (*f)(void *), void *arg, unsigned worker_id)
-{
- int n;
- char c = 0;
- int m2w = lcore_config[worker_id].pipe_main2worker[1];
- int w2m = lcore_config[worker_id].pipe_worker2main[0];
- int rc = -EBUSY;
-
- /* Check if the worker is in 'WAIT' state. Use acquire order
- * since 'state' variable is used as the guard variable.
- */
- if (__atomic_load_n(&lcore_config[worker_id].state,
- __ATOMIC_ACQUIRE) != WAIT)
- goto finish;
-
- lcore_config[worker_id].arg = arg;
- /* Ensure that all the memory operations are completed
- * before the worker thread starts running the function.
- * Use worker thread function as the guard variable.
- */
- __atomic_store_n(&lcore_config[worker_id].f, f, __ATOMIC_RELEASE);
-
- /* send message */
- n = 0;
- while (n == 0 || (n < 0 && errno == EINTR))
- n = write(m2w, &c, 1);
- if (n < 0)
- rte_panic("cannot write on configuration pipe\n");
-
- /* wait ack */
- do {
- n = read(w2m, &c, 1);
- } while (n < 0 && errno == EINTR);
-
- if (n <= 0)
- rte_panic("cannot read on configuration pipe\n");
-
- rc = 0;
-finish:
- rte_eal_trace_thread_remote_launch(f, arg, worker_id, rc);
- return rc;
-}
-
-/* main loop of threads */
-__rte_noreturn void *
-eal_thread_loop(void *arg)
-{
- unsigned int lcore_id = (uintptr_t)arg;
- char c;
- int n, ret;
- unsigned lcore_id;
- int m2w, w2m;
- char cpuset[RTE_CPU_AFFINITY_STR_LEN];
-
- m2w = lcore_config[lcore_id].pipe_main2worker[0];
- w2m = lcore_config[lcore_id].pipe_worker2main[1];
-
- __rte_thread_init(lcore_id, &lcore_config[lcore_id].cpuset);
-
- ret = eal_thread_dump_current_affinity(cpuset, sizeof(cpuset));
- RTE_LOG(DEBUG, EAL, "lcore %u is ready (tid=%p;cpuset=[%s%s])\n",
- lcore_id, pthread_self(), cpuset, ret == 0 ? "" : "...");
-
- rte_eal_trace_thread_lcore_ready(lcore_id, cpuset);
-
- /* read on our pipe to get commands */
- while (1) {
- lcore_function_t *f;
- void *fct_arg;
-
- /* wait command */
- do {
- n = read(m2w, &c, 1);
- } while (n < 0 && errno == EINTR);
-
- if (n <= 0)
- rte_panic("cannot read on configuration pipe\n");
-
- /* Set the state to 'RUNNING'. Use release order
- * since 'state' variable is used as the guard variable.
- */
- __atomic_store_n(&lcore_config[lcore_id].state, RUNNING,
- __ATOMIC_RELEASE);
-
- /* send ack */
- n = 0;
- while (n == 0 || (n < 0 && errno == EINTR))
- n = write(w2m, &c, 1);
- if (n < 0)
- rte_panic("cannot write on configuration pipe\n");
-
- /* Load 'f' with acquire order to ensure that
- * the memory operations from the main thread
- * are accessed only after update to 'f' is visible.
- * Wait till the update to 'f' is visible to the worker.
- */
- while ((f = __atomic_load_n(&lcore_config[lcore_id].f,
- __ATOMIC_ACQUIRE)) == NULL)
- rte_pause();
-
- /* call the function and store the return value */
- fct_arg = lcore_config[lcore_id].arg;
- ret = f(fct_arg);
- lcore_config[lcore_id].ret = ret;
- lcore_config[lcore_id].f = NULL;
- lcore_config[lcore_id].arg = NULL;
-
- /* Store the state with release order to ensure that
- * the memory operations from the worker thread
- * are completed before the state is updated.
- * Use 'state' as the guard variable.
- */
- __atomic_store_n(&lcore_config[lcore_id].state, WAIT,
- __ATOMIC_RELEASE);
- }
-
- /* never reached */
- /* pthread_exit(NULL); */
- /* return NULL; */
-}
-
/* require calling thread tid by gettid() */
int rte_sys_gettid(void)
{
@@ -862,7 +862,6 @@ int
rte_eal_init(int argc, char **argv)
{
int i, fctret, ret;
- pthread_t thread_id;
static uint32_t run_once;
uint32_t has_run = 0;
const char *p;
@@ -890,7 +889,6 @@ rte_eal_init(int argc, char **argv)
p = strrchr(argv[0], '/');
strlcpy(logid, p ? p + 1 : argv[0], sizeof(logid));
- thread_id = pthread_self();
eal_reset_internal_config(internal_conf);
@@ -1129,7 +1127,7 @@ rte_eal_init(int argc, char **argv)
ret = eal_thread_dump_current_affinity(cpuset, sizeof(cpuset));
RTE_LOG(DEBUG, EAL, "Main lcore %u is ready (tid=%zx;cpuset=[%s%s])\n",
- config->main_lcore, (uintptr_t)thread_id, cpuset,
+ config->main_lcore, (uintptr_t)pthread_self(), cpuset,
ret == 0 ? "" : "...");
RTE_LCORE_FOREACH_WORKER(i) {
@@ -14,139 +14,11 @@
#include <rte_log.h>
#include <rte_eal.h>
#include <rte_lcore.h>
-#include <rte_eal_trace.h>
+#include <rte_string_fns.h>
#include "eal_private.h"
#include "eal_thread.h"
-/*
- * Send a message to a worker lcore identified by worker_id to call a
- * function f with argument arg. Once the execution is done, the
- * remote lcore switches to WAIT state.
- */
-int
-rte_eal_remote_launch(int (*f)(void *), void *arg, unsigned int worker_id)
-{
- int n;
- char c = 0;
- int m2w = lcore_config[worker_id].pipe_main2worker[1];
- int w2m = lcore_config[worker_id].pipe_worker2main[0];
- int rc = -EBUSY;
-
- /* Check if the worker is in 'WAIT' state. Use acquire order
- * since 'state' variable is used as the guard variable.
- */
- if (__atomic_load_n(&lcore_config[worker_id].state,
- __ATOMIC_ACQUIRE) != WAIT)
- goto finish;
-
- lcore_config[worker_id].arg = arg;
- /* Ensure that all the memory operations are completed
- * before the worker thread starts running the function.
- * Use worker thread function pointer as the guard variable.
- */
- __atomic_store_n(&lcore_config[worker_id].f, f, __ATOMIC_RELEASE);
-
- /* send message */
- n = 0;
- while (n == 0 || (n < 0 && errno == EINTR))
- n = write(m2w, &c, 1);
- if (n < 0)
- rte_panic("cannot write on configuration pipe\n");
-
- /* wait ack */
- do {
- n = read(w2m, &c, 1);
- } while (n < 0 && errno == EINTR);
-
- if (n <= 0)
- rte_panic("cannot read on configuration pipe\n");
-
- rc = 0;
-finish:
- rte_eal_trace_thread_remote_launch(f, arg, worker_id, rc);
- return rc;
-}
-
-/* main loop of threads */
-__rte_noreturn void *
-eal_thread_loop(void *arg)
-{
- unsigned int lcore_id = (uintptr_t)arg;
- char c;
- int n, ret;
- int m2w, w2m;
- char cpuset[RTE_CPU_AFFINITY_STR_LEN];
-
-
- m2w = lcore_config[lcore_id].pipe_main2worker[0];
- w2m = lcore_config[lcore_id].pipe_worker2main[1];
-
- __rte_thread_init(lcore_id, &lcore_config[lcore_id].cpuset);
-
- ret = eal_thread_dump_current_affinity(cpuset, sizeof(cpuset));
- RTE_LOG(DEBUG, EAL, "lcore %u is ready (tid=%zx;cpuset=[%s%s])\n",
- lcore_id, (uintptr_t)pthread_self(), cpuset,
- ret == 0 ? "" : "...");
-
- rte_eal_trace_thread_lcore_ready(lcore_id, cpuset);
-
- /* read on our pipe to get commands */
- while (1) {
- lcore_function_t *f;
- void *fct_arg;
-
- /* wait command */
- do {
- n = read(m2w, &c, 1);
- } while (n < 0 && errno == EINTR);
-
- if (n <= 0)
- rte_panic("cannot read on configuration pipe\n");
-
- /* Set the state to 'RUNNING'. Use release order
- * since 'state' variable is used as the guard variable.
- */
- __atomic_store_n(&lcore_config[lcore_id].state, RUNNING,
- __ATOMIC_RELEASE);
-
- /* send ack */
- n = 0;
- while (n == 0 || (n < 0 && errno == EINTR))
- n = write(w2m, &c, 1);
- if (n < 0)
- rte_panic("cannot write on configuration pipe\n");
-
- /* Load 'f' with acquire order to ensure that
- * the memory operations from the main thread
- * are accessed only after update to 'f' is visible.
- * Wait till the update to 'f' is visible to the worker.
- */
- while ((f = __atomic_load_n(&lcore_config[lcore_id].f,
- __ATOMIC_ACQUIRE)) == NULL)
- rte_pause();
-
- /* call the function and store the return value */
- fct_arg = lcore_config[lcore_id].arg;
- ret = f(fct_arg);
- lcore_config[lcore_id].ret = ret;
- lcore_config[lcore_id].f = NULL;
- lcore_config[lcore_id].arg = NULL;
-
- /* Store the state with release order to ensure that
- * the memory operations from the worker thread
- * are completed before the state is updated.
- * Use 'state' as the guard variable.
- */
- __atomic_store_n(&lcore_config[lcore_id].state, WAIT,
- __ATOMIC_RELEASE);
- }
-
- /* never reached */
- /* pthread_exit(NULL); */
- /* return NULL; */
-}
-
/* require calling thread tid by gettid() */
int rte_sys_gettid(void)
{
new file mode 100644
@@ -0,0 +1,63 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2022 Red Hat, Inc.
+ */
+
+#include <errno.h>
+#include <unistd.h>
+
+#include <rte_debug.h>
+
+#include "eal_private.h"
+
+void
+eal_thread_wake_worker(unsigned int worker_id)
+{
+ int m2w = lcore_config[worker_id].pipe_main2worker[1];
+ int w2m = lcore_config[worker_id].pipe_worker2main[0];
+ char c = 0;
+ int n;
+
+ do {
+ n = write(m2w, &c, 1);
+ } while (n == 0 || (n < 0 && errno == EINTR));
+ if (n < 0)
+ rte_panic("cannot write on configuration pipe\n");
+
+ do {
+ n = read(w2m, &c, 1);
+ } while (n < 0 && errno == EINTR);
+ if (n <= 0)
+ rte_panic("cannot read on configuration pipe\n");
+}
+
+void
+eal_thread_wait_command(void)
+{
+ unsigned int lcore_id = rte_lcore_id();
+ int m2w;
+ char c;
+ int n;
+
+ m2w = lcore_config[lcore_id].pipe_main2worker[0];
+ do {
+ n = read(m2w, &c, 1);
+ } while (n < 0 && errno == EINTR);
+ if (n <= 0)
+ rte_panic("cannot read on configuration pipe\n");
+}
+
+void
+eal_thread_ack_command(void)
+{
+ unsigned int lcore_id = rte_lcore_id();
+ char c = 0;
+ int w2m;
+ int n;
+
+ w2m = lcore_config[lcore_id].pipe_worker2main[1];
+ do {
+ n = write(w2m, &c, 1);
+ } while (n == 0 || (n < 0 && errno == EINTR));
+ if (n < 0)
+ rte_panic("cannot write on configuration pipe\n");
+}
@@ -3,9 +3,10 @@
sources += files(
'eal_file.c',
+ 'eal_filesystem.c',
+ 'eal_firmware.c',
'eal_unix_memory.c',
+ 'eal_unix_thread.c',
'eal_unix_timer.c',
- 'eal_firmware.c',
- 'eal_filesystem.c',
'rte_thread.c',
)
@@ -11,124 +11,62 @@
#include <rte_per_lcore.h>
#include <rte_common.h>
#include <rte_memory.h>
-#include <eal_thread.h>
#include "eal_private.h"
+#include "eal_thread.h"
#include "eal_windows.h"
-/*
- * Send a message to a worker lcore identified by worker_id to call a
- * function f with argument arg. Once the execution is done, the
- * remote lcore switches to WAIT state.
- */
-int
-rte_eal_remote_launch(lcore_function_t *f, void *arg, unsigned int worker_id)
+void
+eal_thread_wake_worker(unsigned int worker_id)
{
- int n;
- char c = 0;
int m2w = lcore_config[worker_id].pipe_main2worker[1];
int w2m = lcore_config[worker_id].pipe_worker2main[0];
+ char c = 0;
+ int n;
- /* Check if the worker is in 'WAIT' state. Use acquire order
- * since 'state' variable is used as the guard variable.
- */
- if (__atomic_load_n(&lcore_config[worker_id].state,
- __ATOMIC_ACQUIRE) != WAIT)
- return -EBUSY;
-
- lcore_config[worker_id].arg = arg;
- /* Ensure that all the memory operations are completed
- * before the worker thread starts running the function.
- * Use worker thread function as the guard variable.
- */
- __atomic_store_n(&lcore_config[worker_id].f, f, __ATOMIC_RELEASE);
-
- /* send message */
- n = 0;
- while (n == 0 || (n < 0 && errno == EINTR))
+ do {
n = _write(m2w, &c, 1);
+ } while (n == 0 || (n < 0 && errno == EINTR));
if (n < 0)
rte_panic("cannot write on configuration pipe\n");
- /* wait ack */
do {
n = _read(w2m, &c, 1);
} while (n < 0 && errno == EINTR);
-
if (n <= 0)
rte_panic("cannot read on configuration pipe\n");
-
- return 0;
}
-/* main loop of threads */
-void *
-eal_thread_loop(void *arg)
+void
+eal_thread_wait_command(void)
{
- unsigned int lcore_id = (uintptr_t)arg;
+ unsigned int lcore_id = rte_lcore_id();
+ int m2w;
char c;
- int n, ret;
- int m2w, w2m;
- char cpuset[RTE_CPU_AFFINITY_STR_LEN];
+ int n;
m2w = lcore_config[lcore_id].pipe_main2worker[0];
- w2m = lcore_config[lcore_id].pipe_worker2main[1];
+ do {
+ n = _read(m2w, &c, 1);
+ } while (n < 0 && errno == EINTR);
+ if (n <= 0)
+ rte_panic("cannot read on configuration pipe\n");
+}
- __rte_thread_init(lcore_id, &lcore_config[lcore_id].cpuset);
-
- RTE_LOG(DEBUG, EAL, "lcore %u is ready (tid=%zx;cpuset=[%s])\n",
- lcore_id, (uintptr_t)pthread_self(), cpuset);
-
- /* read on our pipe to get commands */
- while (1) {
- lcore_function_t *f;
- void *fct_arg;
-
- /* wait command */
- do {
- n = _read(m2w, &c, 1);
- } while (n < 0 && errno == EINTR);
-
- if (n <= 0)
- rte_panic("cannot read on configuration pipe\n");
-
- /* Set the state to 'RUNNING'. Use release order
- * since 'state' variable is used as the guard variable.
- */
- __atomic_store_n(&lcore_config[lcore_id].state, RUNNING,
- __ATOMIC_RELEASE);
-
- /* send ack */
- n = 0;
- while (n == 0 || (n < 0 && errno == EINTR))
- n = _write(w2m, &c, 1);
- if (n < 0)
- rte_panic("cannot write on configuration pipe\n");
-
- /* Load 'f' with acquire order to ensure that
- * the memory operations from the main thread
- * are accessed only after update to 'f' is visible.
- * Wait till the update to 'f' is visible to the worker.
- */
- while ((f = __atomic_load_n(&lcore_config[lcore_id].f,
- __ATOMIC_ACQUIRE)) == NULL)
- rte_pause();
-
- /* call the function and store the return value */
- fct_arg = lcore_config[lcore_id].arg;
- ret = f(fct_arg);
- lcore_config[lcore_id].ret = ret;
- lcore_config[lcore_id].f = NULL;
- lcore_config[lcore_id].arg = NULL;
-
- /* Store the state with release order to ensure that
- * the memory operations from the worker thread
- * are completed before the state is updated.
- * Use 'state' as the guard variable.
- */
- __atomic_store_n(&lcore_config[lcore_id].state, WAIT,
- __ATOMIC_RELEASE);
- }
+void
+eal_thread_ack_command(void)
+{
+ unsigned int lcore_id = rte_lcore_id();
+ char c = 0;
+ int w2m;
+ int n;
+
+ w2m = lcore_config[lcore_id].pipe_worker2main[1];
+ do {
+ n = _write(w2m, &c, 1);
+ } while (n == 0 || (n < 0 && errno == EINTR));
+ if (n < 0)
+ rte_panic("cannot write on configuration pipe\n");
}
/* function to create threads */