[32/39] eventdev: add routine to launch eventmode workers
Checks
Commit Message
With eventmode, workers could be drafted differently according to the
capabilities of the underlying event device. The added function would
receive an array of such workers and probes the eventmode properties to
choose the worker.
Signed-off-by: Anoob Joseph <anoobj@marvell.com>
Signed-off-by: Lukasz Bartosik <lbartosik@marvell.com>
---
lib/librte_eventdev/rte_eventdev_version.map | 1 +
lib/librte_eventdev/rte_eventmode_helper.c | 240 +++++++++++++++++++++
lib/librte_eventdev/rte_eventmode_helper.h | 49 +++++
.../rte_eventmode_helper_internal.h | 3 +
4 files changed, 293 insertions(+)
Comments
Hi Anoob,
I've listed a few notes in-line.
> -----Original Message-----
> From: Anoob Joseph <anoobj@marvell.com>
> Sent: Monday, June 3, 2019 12:33 PM
> To: Jerin Jacob <jerinj@marvell.com>; Rao, Nikhil <nikhil.rao@intel.com>;
> Carrillo, Erik G <erik.g.carrillo@intel.com>; Gujjar, Abhinandan S
> <abhinandan.gujjar@intel.com>; Richardson, Bruce
> <bruce.richardson@intel.com>; De Lara Guarch, Pablo
> <pablo.de.lara.guarch@intel.com>
> Cc: Anoob Joseph <anoobj@marvell.com>; Narayana Prasad
> <pathreya@marvell.com>; dev@dpdk.org; Lukasz Bartosik
> <lbartosik@marvell.com>; Pavan Nikhilesh <pbhagavatula@marvell.com>;
> Hemant Agrawal <hemant.agrawal@nxp.com>; Nipun Gupta
> <nipun.gupta@nxp.com>; Van Haaren, Harry
> <harry.van.haaren@intel.com>; Mattias Rönnblom
> <mattias.ronnblom@ericsson.com>; Ma, Liang J <liang.j.ma@intel.com>
> Subject: [PATCH 32/39] eventdev: add routine to launch eventmode workers
>
> With eventmode, workers could be drafted differently according to the
> capabilities of the underlying event device. The added function would
> receive an array of such workers and probes the eventmode properties to
> choose the worker.
>
> Signed-off-by: Anoob Joseph <anoobj@marvell.com>
> Signed-off-by: Lukasz Bartosik <lbartosik@marvell.com>
> ---
<...snipped...>
> +
> +/* Event mode packet tx types */
> +enum rte_eventmode_helper_tx_types {
> + RTE_EVETNMODE_HELPER_TX_TYPE_INVALID = 0,
A couple of characters are transposed in the above.
> + RTE_EVENTMODE_HELPER_TX_TYPE_MAX = 16
> +};
> +
> struct rte_eventmode_helper_conf {
> enum rte_eventmode_helper_pkt_transfer_mode mode;
> /**< Packet transfer mode of the application */ @@ -41,6
> +55,20 @@ struct rte_eventmode_helper_event_link_info {
> /**< Lcore to be polling on this port */ };
>
I believe anonymous unions and structures should be annotated with RTE_STD_C11 below and in other places throughout the series.
> +/* Workers registered by the application */ struct
> +rte_eventmode_helper_app_worker_params {
> + union {
> + struct {
> + uint64_t burst : 4;
> + /**< Specify status of rx type burst */
> + };
> + uint64_t u64;
> + } cap;
> + /**< Capabilities of this worker */
> + void (*worker_thread)(void *mode_conf);
> + /**< Worker thread */
> +};
> +
> /* Common helper functions for command line parsing */
>
> /**
> @@ -157,6 +185,27 @@ uint8_t __rte_experimental
> rte_eventmode_helper_get_tx_queue(struct rte_eventmode_helper_conf
> *mode_conf,
> uint8_t eventdev_id);
>
The doxygen documentation for __rte_experimental functions in general should have:
* @warning
* @b EXPERIMENTAL: this API may change without prior notice
as well.
> +/**
> + * Launch eventmode worker
> + *
> + * The application can request the eventmode helper subsystem to launch
> +the
> + * worker based on the capabilities of event device and the options
> +selected
> + * while initializing the eventmode.
> + *
> + * @param mode_conf
> + * Configuration of the mode in which app is doing packet handling
> + * @param app_wrkr
> + * List of all the workers registered by application, along with it's
> + * capabilities
> + * @param nb_wrkr_param
> + * Number of workers passed by the application
> + *
> + */
> +void __rte_experimental
> +rte_eventmode_helper_launch_worker(struct
> rte_eventmode_helper_conf *mode_conf,
> + struct rte_eventmode_helper_app_worker_params
> *app_wrkr,
> + uint8_t nb_wrkr_param);
> +
> #ifdef __cplusplus
> }
> #endif
<...snipped...>
Regards,
Erik
Hi Anoob,
One other observation in-line:
> -----Original Message-----
> From: Anoob Joseph <anoobj@marvell.com>
> Sent: Monday, June 3, 2019 12:33 PM
> To: Jerin Jacob <jerinj@marvell.com>; Rao, Nikhil <nikhil.rao@intel.com>;
> Carrillo, Erik G <erik.g.carrillo@intel.com>; Gujjar, Abhinandan S
> <abhinandan.gujjar@intel.com>; Richardson, Bruce
> <bruce.richardson@intel.com>; De Lara Guarch, Pablo
> <pablo.de.lara.guarch@intel.com>
> Cc: Anoob Joseph <anoobj@marvell.com>; Narayana Prasad
> <pathreya@marvell.com>; dev@dpdk.org; Lukasz Bartosik
> <lbartosik@marvell.com>; Pavan Nikhilesh <pbhagavatula@marvell.com>;
> Hemant Agrawal <hemant.agrawal@nxp.com>; Nipun Gupta
> <nipun.gupta@nxp.com>; Van Haaren, Harry
> <harry.van.haaren@intel.com>; Mattias Rönnblom
> <mattias.ronnblom@ericsson.com>; Ma, Liang J <liang.j.ma@intel.com>
> Subject: [PATCH 32/39] eventdev: add routine to launch eventmode workers
>
> With eventmode, workers could be drafted differently according to the
> capabilities of the underlying event device. The added function would
> receive an array of such workers and probes the eventmode properties to
> choose the worker.
>
> Signed-off-by: Anoob Joseph <anoobj@marvell.com>
> Signed-off-by: Lukasz Bartosik <lbartosik@marvell.com>
> ---
<...Snipped...>
> +
> +void __rte_experimental
> +rte_eventmode_helper_launch_worker(struct
> rte_eventmode_helper_conf *mode_conf,
> + struct rte_eventmode_helper_app_worker_params
> *app_wrkr,
> + uint8_t nb_wrkr_param)
> +{
> + struct rte_eventmode_helper_app_worker_params *match_wrkr;
> + uint32_t lcore_id;
> + struct eventmode_conf *em_conf;
> +
> + if (mode_conf == NULL) {
> + RTE_EM_HLPR_LOG_ERR("Invalid conf");
> + return;
> + }
> +
> + if (mode_conf->mode_params == NULL) {
> + RTE_EM_HLPR_LOG_ERR("Invalid mode params");
> + return;
> + }
> +
> + /* Get eventmode conf */
> + em_conf = (struct eventmode_conf *)(mode_conf->mode_params);
> +
> + /* Get core ID */
> + lcore_id = rte_lcore_id();
> +
> + /* TODO check capability for rx core */
> +
> + /* Check if this is rx core */
> + if (em_conf->eth_core_mask & (1 << lcore_id)) {
In the above, eth_core_mask is a uint32_t, but RTE_MAX_LCORE=128 in the common config.
> + rte_eventmode_helper_start_worker_eth_core(em_conf,
> lcore_id);
> + return;
> + }
> +
> + if (app_wrkr == NULL || nb_wrkr_param == 0) {
> + RTE_EM_HLPR_LOG_ERR("Invalid args");
> + return;
> + }
> +
> + /*
> + * This is a regular worker thread. The application would be
> + * registering multiple workers with various capabilities. The
> + * worker to be run will be selected by the capabilities of the
> + * event device configured.
> + */
> +
> + /* Get the first matching worker for the event device */
> + match_wrkr = rte_eventmode_helper_find_worker(lcore_id,
> + em_conf,
> + app_wrkr,
> + nb_wrkr_param);
> +
> + if (match_wrkr == NULL) {
> + RTE_EM_HLPR_LOG_ERR(
> + "No matching worker registered for lcore %d",
> lcore_id);
> + goto clean_and_exit;
> + }
> +
> + /* Verify sanity of the matched worker */
> + if (rte_eventmode_helper_verify_match_worker(match_wrkr) != 1)
> {
> + RTE_EM_HLPR_LOG_ERR("Error in validating the matched
> worker");
> + goto clean_and_exit;
> + }
> +
> + /* Launch the worker thread */
> + match_wrkr->worker_thread(mode_conf);
> +
> +clean_and_exit:
> +
> + /* Flag eth_cores to stop, if started */
> + rte_eventmode_helper_stop_worker_eth_core();
> +}
<...Snipped...>
Regards,
Erik
Regards
Sunil Kumar Kori
>-----Original Message-----
>From: dev <dev-bounces@dpdk.org> On Behalf Of Anoob Joseph
>Sent: Monday, June 3, 2019 11:03 PM
>To: Jerin Jacob Kollanukkaran <jerinj@marvell.com>; Nikhil Rao
><nikhil.rao@intel.com>; Erik Gabriel Carrillo <erik.g.carrillo@intel.com>;
>Abhinandan Gujjar <abhinandan.gujjar@intel.com>; Bruce Richardson
><bruce.richardson@intel.com>; Pablo de Lara
><pablo.de.lara.guarch@intel.com>
>Cc: Anoob Joseph <anoobj@marvell.com>; Narayana Prasad Raju Athreya
><pathreya@marvell.com>; dev@dpdk.org; Lukas Bartosik
><lbartosik@marvell.com>; Pavan Nikhilesh Bhagavatula
><pbhagavatula@marvell.com>; Hemant Agrawal
><hemant.agrawal@nxp.com>; Nipun Gupta <nipun.gupta@nxp.com>; Harry
>van Haaren <harry.van.haaren@intel.com>; Mattias Rönnblom
><mattias.ronnblom@ericsson.com>; Liang Ma <liang.j.ma@intel.com>
>Subject: [EXT] [dpdk-dev] [PATCH 32/39] eventdev: add routine to launch
>eventmode workers
>
>External Email
>
>----------------------------------------------------------------------
>With eventmode, workers could be drafted differently according to the
>capabilities of the underlying event device. The added function would receive
>an array of such workers and probes the eventmode properties to choose the
>worker.
>
>Signed-off-by: Anoob Joseph <anoobj@marvell.com>
>Signed-off-by: Lukasz Bartosik <lbartosik@marvell.com>
>---
> lib/librte_eventdev/rte_eventdev_version.map | 1 +
> lib/librte_eventdev/rte_eventmode_helper.c | 240
>+++++++++++++++++++++
> lib/librte_eventdev/rte_eventmode_helper.h | 49 +++++
> .../rte_eventmode_helper_internal.h | 3 +
> 4 files changed, 293 insertions(+)
>
>diff --git a/lib/librte_eventdev/rte_eventdev_version.map
>b/lib/librte_eventdev/rte_eventdev_version.map
>index 3cf926a..665836e 100644
>--- a/lib/librte_eventdev/rte_eventdev_version.map
>+++ b/lib/librte_eventdev/rte_eventdev_version.map
>@@ -135,4 +135,5 @@ EXPERIMENTAL {
> rte_eventmode_helper_display_conf;
> rte_eventmode_helper_get_event_lcore_links;
> rte_eventmode_helper_get_tx_queue;
>+ rte_eventmode_helper_launch_worker;
> };
>diff --git a/lib/librte_eventdev/rte_eventmode_helper.c
>b/lib/librte_eventdev/rte_eventmode_helper.c
>index e7670e0..77a5a4e 100644
>--- a/lib/librte_eventdev/rte_eventmode_helper.c
>+++ b/lib/librte_eventdev/rte_eventmode_helper.c
>@@ -2,6 +2,7 @@
> * Copyright (C) 2019 Marvell International Ltd.
> */
> #include <getopt.h>
>+#include <stdbool.h>
>
> #include <rte_ethdev.h>
> #include <rte_eventdev.h>
>@@ -13,6 +14,8 @@
>
> #define CMD_LINE_OPT_TRANSFER_MODE "transfer-mode"
>
>+static volatile bool eth_core_running;
>+
> static const char short_options[] =
> ""
> ;
>@@ -111,6 +114,16 @@ internal_get_eventdev_params(struct
>eventmode_conf *em_conf,
> return &(em_conf->eventdev_config[i]); }
>
>+static inline bool
>+internal_dev_has_burst_mode(uint8_t dev_id) {
>+ struct rte_event_dev_info dev_info;
>+
>+ rte_event_dev_info_get(dev_id, &dev_info);
>+ return (dev_info.event_dev_cap &
>RTE_EVENT_DEV_CAP_BURST_MODE) ?
>+ true : false;
>+}
>+
> /* Global functions */
>
> void __rte_experimental
>@@ -980,3 +993,230 @@ rte_eventmode_helper_get_tx_queue(struct
>rte_eventmode_helper_conf *mode_conf,
> return eventdev_config->nb_eventqueue - 1; }
>
>+/* Helper functions for launching workers */
>+
>+static int32_t
>+rte_eventmode_helper_start_worker_eth_core(struct eventmode_conf
>*em_conf,
>+ uint32_t lcore_id)
Internal functions must not be prefixed with "rte_",
>+{
>+ uint32_t service_id[EVENT_MODE_MAX_ADAPTERS_PER_RX_CORE];
>+ struct rx_adapter_conf *rx_adapter;
>+ int service_count = 0;
>+ int adapter_id;
>+ int32_t ret;
>+ int i;
>+
>+ RTE_EM_HLPR_LOG_INFO(
>+ "Entering eth_core processing on lcore %u", lcore_id);
>+
>+ /*
>+ * Need to parse adapter conf to see which of all Rx adapters need
>+ * to be handled by this core.
>+ */
>+ for (i = 0; i < em_conf->nb_rx_adapter; i++) {
>+ /* Check if we have exceeded the max allowed */
>+ if (service_count >
>EVENT_MODE_MAX_ADAPTERS_PER_RX_CORE) {
>+ RTE_EM_HLPR_LOG_ERR(
>+ "Exceeded the max allowed adapters per rx
>core");
>+ break;
>+ }
>+
>+ rx_adapter = &(em_conf->rx_adapter[i]);
>+ if (rx_adapter->rx_core_id != lcore_id)
>+ continue;
>+
>+ /* Adapter need to be handled by this core */
>+ adapter_id = rx_adapter->adapter_id;
>+
>+ /* Get the service ID for the adapters */
>+ ret = rte_event_eth_rx_adapter_service_id_get(adapter_id,
>+ &(service_id[service_count]));
>+
>+ if (ret != -ESRCH && ret != 0) {
>+ RTE_EM_HLPR_LOG_ERR(
>+ "Error getting service ID used by Rx adapter");
>+ return ret;
>+ }
>+
>+ /* Update service count */
>+ service_count++;
>+ }
>+
>+ eth_core_running = true;
>+
>+ while (eth_core_running) {
>+ for (i = 0; i < service_count; i++) {
>+ /* Initiate adapter service */
>+ rte_service_run_iter_on_app_lcore(service_id[i], 0);
>+ }
>+ }
>+
>+ return 0;
>+}
>+
>+static int32_t
>+rte_eventmode_helper_stop_worker_eth_core(void)
>+{
>+ if (eth_core_running) {
>+ RTE_EM_HLPR_LOG_INFO("Stopping rx cores\n");
>+ eth_core_running = false;
>+ }
>+ return 0;
>+}
>+
>+static struct rte_eventmode_helper_app_worker_params *
>+rte_eventmode_helper_find_worker(uint32_t lcore_id,
>+ struct eventmode_conf *em_conf,
>+ struct rte_eventmode_helper_app_worker_params
>*app_wrkrs,
>+ uint8_t nb_wrkr_param)
>+{
>+ struct rte_eventmode_helper_event_link_info *link = NULL;
>+ uint8_t eventdev_id;
>+ struct eventdev_params *eventdev_config;
>+ int i;
>+ struct rte_eventmode_helper_app_worker_params curr_conf = {
>+ {{0} }, NULL};
>+ struct rte_eventmode_helper_app_worker_params *tmp_wrkr;
>+
>+ /*
>+ * Event device to be used will be derived from the first lcore-event
>+ * link.
>+ *
>+ * Assumption: All lcore-event links tied to a core would be using the
>+ * same event device. in other words, one core would be polling on
>+ * queues of a single event device only.
>+ */
>+
>+ /* Get a link for this lcore */
>+ for (i = 0; i < em_conf->nb_link; i++) {
>+ link = &(em_conf->link[i]);
>+ if (link->lcore_id == lcore_id)
>+ break;
>+ }
>+
>+ if (link == NULL) {
>+ RTE_EM_HLPR_LOG_ERR(
>+ "No valid link found for lcore(%d)", lcore_id);
>+ return NULL;
>+ }
>+
>+ /* Get event dev ID */
>+ eventdev_id = link->eventdev_id;
>+
>+ /* Get the corresponding eventdev config */
>+ eventdev_config = internal_get_eventdev_params(em_conf,
>eventdev_id);
>+
>+ /* Populate the curr_conf with the capabilities */
>+
>+ /* Check for burst mode */
>+ if (internal_dev_has_burst_mode(eventdev_id))
>+ curr_conf.cap.burst =
>RTE_EVENTMODE_HELPER_RX_TYPE_BURST;
>+ else
>+ curr_conf.cap.burst =
>RTE_EVENTMODE_HELPER_RX_TYPE_NON_BURST;
>+
>+ /* Now parse the passed list and see if we have matching capabilities
>+*/
>+
>+ /* Initialize the pointer used to traverse the list */
>+ tmp_wrkr = app_wrkrs;
>+
>+ for (i = 0; i < nb_wrkr_param; i++, tmp_wrkr++) {
>+
>+ /* Skip this if capabilities are not matching */
>+ if (tmp_wrkr->cap.u64 != curr_conf.cap.u64)
>+ continue;
>+
>+ /* If the checks pass, we have a match */
>+ return tmp_wrkr;
>+ }
>+
>+ /* TODO required for ATQ */
>+ RTE_SET_USED(eventdev_config);
>+
>+ return NULL;
>+}
>+
>+static int
>+rte_eventmode_helper_verify_match_worker(
>+ struct rte_eventmode_helper_app_worker_params *match_wrkr) {
>+ /* Verify registered worker */
>+ if (match_wrkr->worker_thread == NULL) {
>+ RTE_EM_HLPR_LOG_ERR("No worker registered for second
>stage");
>+ return 0;
>+ }
>+
>+ /* Success */
>+ return 1;
>+}
>+
>+void __rte_experimental
>+rte_eventmode_helper_launch_worker(struct rte_eventmode_helper_conf
>*mode_conf,
>+ struct rte_eventmode_helper_app_worker_params
>*app_wrkr,
>+ uint8_t nb_wrkr_param)
>+{
>+ struct rte_eventmode_helper_app_worker_params *match_wrkr;
>+ uint32_t lcore_id;
>+ struct eventmode_conf *em_conf;
>+
>+ if (mode_conf == NULL) {
>+ RTE_EM_HLPR_LOG_ERR("Invalid conf");
>+ return;
>+ }
>+
>+ if (mode_conf->mode_params == NULL) {
>+ RTE_EM_HLPR_LOG_ERR("Invalid mode params");
>+ return;
>+ }
>+
>+ /* Get eventmode conf */
>+ em_conf = (struct eventmode_conf *)(mode_conf->mode_params);
>+
>+ /* Get core ID */
>+ lcore_id = rte_lcore_id();
>+
>+ /* TODO check capability for rx core */
>+
>+ /* Check if this is rx core */
>+ if (em_conf->eth_core_mask & (1 << lcore_id)) {
>+ rte_eventmode_helper_start_worker_eth_core(em_conf,
>lcore_id);
>+ return;
>+ }
>+
>+ if (app_wrkr == NULL || nb_wrkr_param == 0) {
>+ RTE_EM_HLPR_LOG_ERR("Invalid args");
>+ return;
>+ }
>+
>+ /*
>+ * This is a regular worker thread. The application would be
>+ * registering multiple workers with various capabilities. The
>+ * worker to be run will be selected by the capabilities of the
>+ * event device configured.
>+ */
>+
>+ /* Get the first matching worker for the event device */
>+ match_wrkr = rte_eventmode_helper_find_worker(lcore_id,
>+ em_conf,
>+ app_wrkr,
>+ nb_wrkr_param);
>+
>+ if (match_wrkr == NULL) {
>+ RTE_EM_HLPR_LOG_ERR(
>+ "No matching worker registered for lcore %d",
>lcore_id);
>+ goto clean_and_exit;
>+ }
>+
>+ /* Verify sanity of the matched worker */
>+ if (rte_eventmode_helper_verify_match_worker(match_wrkr) != 1) {
>+ RTE_EM_HLPR_LOG_ERR("Error in validating the matched
>worker");
>+ goto clean_and_exit;
>+ }
>+
>+ /* Launch the worker thread */
>+ match_wrkr->worker_thread(mode_conf);
>+
>+clean_and_exit:
>+
>+ /* Flag eth_cores to stop, if started */
>+ rte_eventmode_helper_stop_worker_eth_core();
>+}
>diff --git a/lib/librte_eventdev/rte_eventmode_helper.h
>b/lib/librte_eventdev/rte_eventmode_helper.h
>index cd6d708..1235ca4 100644
>--- a/lib/librte_eventdev/rte_eventmode_helper.h
>+++ b/lib/librte_eventdev/rte_eventmode_helper.h
>@@ -17,6 +17,20 @@ enum rte_eventmode_helper_pkt_transfer_mode {
> RTE_EVENTMODE_HELPER_PKT_TRANSFER_MODE_EVENT,
> };
>
>+/* Event mode packet rx types */
>+enum rte_eventmode_helper_rx_types {
>+ RTE_EVENTMODE_HELPER_RX_TYPE_INVALID = 0,
>+ RTE_EVENTMODE_HELPER_RX_TYPE_NON_BURST,
>+ RTE_EVENTMODE_HELPER_RX_TYPE_BURST,
>+ RTE_EVENTMODE_HELPER_RX_TYPE_MAX = 16
>+};
>+
>+/* Event mode packet tx types */
>+enum rte_eventmode_helper_tx_types {
>+ RTE_EVETNMODE_HELPER_TX_TYPE_INVALID = 0,
>+ RTE_EVENTMODE_HELPER_TX_TYPE_MAX = 16
>+};
>+
> struct rte_eventmode_helper_conf {
> enum rte_eventmode_helper_pkt_transfer_mode mode;
> /**< Packet transfer mode of the application */ @@ -41,6
>+55,20 @@ struct rte_eventmode_helper_event_link_info {
> /**< Lcore to be polling on this port */ };
>
>+/* Workers registered by the application */ struct
>+rte_eventmode_helper_app_worker_params {
>+ union {
>+ struct {
>+ uint64_t burst : 4;
>+ /**< Specify status of rx type burst */
>+ };
>+ uint64_t u64;
>+ } cap;
>+ /**< Capabilities of this worker */
>+ void (*worker_thread)(void *mode_conf);
>+ /**< Worker thread */
>+};
>+
> /* Common helper functions for command line parsing */
>
> /**
>@@ -157,6 +185,27 @@ uint8_t __rte_experimental
>rte_eventmode_helper_get_tx_queue(struct rte_eventmode_helper_conf
>*mode_conf,
> uint8_t eventdev_id);
>
>+/**
>+ * Launch eventmode worker
>+ *
>+ * The application can request the eventmode helper subsystem to launch
>+the
>+ * worker based on the capabilities of event device and the options
>+selected
>+ * while initializing the eventmode.
>+ *
>+ * @param mode_conf
>+ * Configuration of the mode in which app is doing packet handling
>+ * @param app_wrkr
>+ * List of all the workers registered by application, along with it's
>+ * capabilities
>+ * @param nb_wrkr_param
>+ * Number of workers passed by the application
>+ *
>+ */
>+void __rte_experimental
>+rte_eventmode_helper_launch_worker(struct rte_eventmode_helper_conf
>*mode_conf,
>+ struct rte_eventmode_helper_app_worker_params
>*app_wrkr,
>+ uint8_t nb_wrkr_param);
>+
> #ifdef __cplusplus
> }
> #endif
>diff --git a/lib/librte_eventdev/rte_eventmode_helper_internal.h
>b/lib/librte_eventdev/rte_eventmode_helper_internal.h
>index 499cf5d..906766c 100644
>--- a/lib/librte_eventdev/rte_eventmode_helper_internal.h
>+++ b/lib/librte_eventdev/rte_eventmode_helper_internal.h
>@@ -48,6 +48,9 @@
> #define EVENT_MODE_MAX_LCORE_LINKS \
> (EVENT_MODE_MAX_EVENT_DEVS *
>EVENT_MODE_MAX_EVENT_QUEUES_PER_DEV)
>
>+/* Max adapters that one Rx core can handle */ #define
>+EVENT_MODE_MAX_ADAPTERS_PER_RX_CORE
>EVENT_MODE_MAX_RX_ADAPTERS
>+
> /* Event dev params */
> struct eventdev_params {
> uint8_t eventdev_id;
>--
>2.7.4
Hi Erik,
Please see inline.
Thanks,
Anoob
> -----Original Message-----
> From: dev <dev-bounces@dpdk.org> On Behalf Of Carrillo, Erik G
> Sent: Monday, June 10, 2019 8:02 PM
> To: Anoob Joseph <anoobj@marvell.com>; Jerin Jacob Kollanukkaran
> <jerinj@marvell.com>; Rao, Nikhil <nikhil.rao@intel.com>; Gujjar,
> Abhinandan S <abhinandan.gujjar@intel.com>; Richardson, Bruce
> <bruce.richardson@intel.com>; De Lara Guarch, Pablo
> <pablo.de.lara.guarch@intel.com>
> Cc: Narayana Prasad Raju Athreya <pathreya@marvell.com>; dev@dpdk.org;
> Lukas Bartosik <lbartosik@marvell.com>; Pavan Nikhilesh Bhagavatula
> <pbhagavatula@marvell.com>; Hemant Agrawal
> <hemant.agrawal@nxp.com>; Nipun Gupta <nipun.gupta@nxp.com>; Van
> Haaren, Harry <harry.van.haaren@intel.com>; Mattias Rönnblom
> <mattias.ronnblom@ericsson.com>; Ma, Liang J <liang.j.ma@intel.com>
> Subject: [EXT] Re: [dpdk-dev] [PATCH 32/39] eventdev: add routine to launch
> eventmode workers
>
> External Email
>
> ----------------------------------------------------------------------
> Hi Anoob,
>
> I've listed a few notes in-line.
>
> > -----Original Message-----
> > From: Anoob Joseph <anoobj@marvell.com>
> > Sent: Monday, June 3, 2019 12:33 PM
> > To: Jerin Jacob <jerinj@marvell.com>; Rao, Nikhil
> > <nikhil.rao@intel.com>; Carrillo, Erik G <erik.g.carrillo@intel.com>;
> > Gujjar, Abhinandan S <abhinandan.gujjar@intel.com>; Richardson, Bruce
> > <bruce.richardson@intel.com>; De Lara Guarch, Pablo
> > <pablo.de.lara.guarch@intel.com>
> > Cc: Anoob Joseph <anoobj@marvell.com>; Narayana Prasad
> > <pathreya@marvell.com>; dev@dpdk.org; Lukasz Bartosik
> > <lbartosik@marvell.com>; Pavan Nikhilesh <pbhagavatula@marvell.com>;
> > Hemant Agrawal <hemant.agrawal@nxp.com>; Nipun Gupta
> > <nipun.gupta@nxp.com>; Van Haaren, Harry
> <harry.van.haaren@intel.com>;
> > Mattias Rönnblom <mattias.ronnblom@ericsson.com>; Ma, Liang J
> > <liang.j.ma@intel.com>
> > Subject: [PATCH 32/39] eventdev: add routine to launch eventmode
> > workers
> >
> > With eventmode, workers could be drafted differently according to the
> > capabilities of the underlying event device. The added function would
> > receive an array of such workers and probes the eventmode properties
> > to choose the worker.
> >
> > Signed-off-by: Anoob Joseph <anoobj@marvell.com>
> > Signed-off-by: Lukasz Bartosik <lbartosik@marvell.com>
> > ---
>
> <...snipped...>
>
> > +
> > +/* Event mode packet tx types */
> > +enum rte_eventmode_helper_tx_types {
> > + RTE_EVETNMODE_HELPER_TX_TYPE_INVALID = 0,
>
> A couple of characters are transposed in the above.
[Anoob] Will fix it in v2.
>
> > + RTE_EVENTMODE_HELPER_TX_TYPE_MAX = 16 };
> > +
> > struct rte_eventmode_helper_conf {
> > enum rte_eventmode_helper_pkt_transfer_mode mode;
> > /**< Packet transfer mode of the application */ @@ -41,6
> > +55,20 @@ struct rte_eventmode_helper_event_link_info {
> > /**< Lcore to be polling on this port */ };
> >
>
> I believe anonymous unions and structures should be annotated with
> RTE_STD_C11 below and in other places throughout the series.
[Anoob] Will fix in v2.
>
> > +/* Workers registered by the application */ struct
> > +rte_eventmode_helper_app_worker_params {
> > + union {
> > + struct {
> > + uint64_t burst : 4;
> > + /**< Specify status of rx type burst */
> > + };
> > + uint64_t u64;
> > + } cap;
> > + /**< Capabilities of this worker */
> > + void (*worker_thread)(void *mode_conf);
> > + /**< Worker thread */
> > +};
> > +
> > /* Common helper functions for command line parsing */
> >
> > /**
> > @@ -157,6 +185,27 @@ uint8_t __rte_experimental
> > rte_eventmode_helper_get_tx_queue(struct
> rte_eventmode_helper_conf
> > *mode_conf,
> > uint8_t eventdev_id);
> >
>
> The doxygen documentation for __rte_experimental functions in general
> should have:
>
> * @warning
> * @b EXPERIMENTAL: this API may change without prior notice
>
> as well.
>
[Anoob] Will add in v2.
> > +/**
> > + * Launch eventmode worker
> > + *
> > + * The application can request the eventmode helper subsystem to
> > +launch the
> > + * worker based on the capabilities of event device and the options
> > +selected
> > + * while initializing the eventmode.
> > + *
> > + * @param mode_conf
> > + * Configuration of the mode in which app is doing packet handling
> > + * @param app_wrkr
> > + * List of all the workers registered by application, along with it's
> > + * capabilities
> > + * @param nb_wrkr_param
> > + * Number of workers passed by the application
> > + *
> > + */
> > +void __rte_experimental
> > +rte_eventmode_helper_launch_worker(struct
> > rte_eventmode_helper_conf *mode_conf,
> > + struct rte_eventmode_helper_app_worker_params
> > *app_wrkr,
> > + uint8_t nb_wrkr_param);
> > +
> > #ifdef __cplusplus
> > }
> > #endif
>
> <...snipped...>
>
> Regards,
> Erik
Hi Erik,
Please see inline.
Thanks,
Anoob
> -----Original Message-----
> From: Carrillo, Erik G <erik.g.carrillo@intel.com>
> Sent: Monday, June 10, 2019 8:16 PM
> To: Anoob Joseph <anoobj@marvell.com>; Jerin Jacob Kollanukkaran
> <jerinj@marvell.com>; Rao, Nikhil <nikhil.rao@intel.com>; Gujjar,
> Abhinandan S <abhinandan.gujjar@intel.com>; Richardson, Bruce
> <bruce.richardson@intel.com>; De Lara Guarch, Pablo
> <pablo.de.lara.guarch@intel.com>
> Cc: Narayana Prasad Raju Athreya <pathreya@marvell.com>; dev@dpdk.org;
> Lukas Bartosik <lbartosik@marvell.com>; Pavan Nikhilesh Bhagavatula
> <pbhagavatula@marvell.com>; Hemant Agrawal
> <hemant.agrawal@nxp.com>; Nipun Gupta <nipun.gupta@nxp.com>; Van
> Haaren, Harry <harry.van.haaren@intel.com>; Mattias Rönnblom
> <mattias.ronnblom@ericsson.com>; Ma, Liang J <liang.j.ma@intel.com>
> Subject: RE: [PATCH 32/39] eventdev: add routine to launch eventmode
> workers
>
> Hi Anoob,
>
> One other observation in-line:
>
> > -----Original Message-----
> > From: Anoob Joseph <anoobj@marvell.com>
> > Sent: Monday, June 3, 2019 12:33 PM
> > To: Jerin Jacob <jerinj@marvell.com>; Rao, Nikhil
> > <nikhil.rao@intel.com>; Carrillo, Erik G <erik.g.carrillo@intel.com>;
> > Gujjar, Abhinandan S <abhinandan.gujjar@intel.com>; Richardson, Bruce
> > <bruce.richardson@intel.com>; De Lara Guarch, Pablo
> > <pablo.de.lara.guarch@intel.com>
> > Cc: Anoob Joseph <anoobj@marvell.com>; Narayana Prasad
> > <pathreya@marvell.com>; dev@dpdk.org; Lukasz Bartosik
> > <lbartosik@marvell.com>; Pavan Nikhilesh <pbhagavatula@marvell.com>;
> > Hemant Agrawal <hemant.agrawal@nxp.com>; Nipun Gupta
> > <nipun.gupta@nxp.com>; Van Haaren, Harry
> <harry.van.haaren@intel.com>;
> > Mattias Rönnblom <mattias.ronnblom@ericsson.com>; Ma, Liang J
> > <liang.j.ma@intel.com>
> > Subject: [PATCH 32/39] eventdev: add routine to launch eventmode
> > workers
> >
> > With eventmode, workers could be drafted differently according to the
> > capabilities of the underlying event device. The added function would
> > receive an array of such workers and probes the eventmode properties
> > to choose the worker.
> >
> > Signed-off-by: Anoob Joseph <anoobj@marvell.com>
> > Signed-off-by: Lukasz Bartosik <lbartosik@marvell.com>
> > ---
>
> <...Snipped...>
>
> > +
> > +void __rte_experimental
> > +rte_eventmode_helper_launch_worker(struct
> > rte_eventmode_helper_conf *mode_conf,
> > + struct rte_eventmode_helper_app_worker_params
> > *app_wrkr,
> > + uint8_t nb_wrkr_param)
> > +{
> > + struct rte_eventmode_helper_app_worker_params *match_wrkr;
> > + uint32_t lcore_id;
> > + struct eventmode_conf *em_conf;
> > +
> > + if (mode_conf == NULL) {
> > + RTE_EM_HLPR_LOG_ERR("Invalid conf");
> > + return;
> > + }
> > +
> > + if (mode_conf->mode_params == NULL) {
> > + RTE_EM_HLPR_LOG_ERR("Invalid mode params");
> > + return;
> > + }
> > +
> > + /* Get eventmode conf */
> > + em_conf = (struct eventmode_conf *)(mode_conf->mode_params);
> > +
> > + /* Get core ID */
> > + lcore_id = rte_lcore_id();
> > +
> > + /* TODO check capability for rx core */
> > +
> > + /* Check if this is rx core */
> > + if (em_conf->eth_core_mask & (1 << lcore_id)) {
>
> In the above, eth_core_mask is a uint32_t, but RTE_MAX_LCORE=128 in the
> common config.
[Anoob] Will fix this in v2. Will use rte_bitmask APIs instead.
>
> > + rte_eventmode_helper_start_worker_eth_core(em_conf,
> > lcore_id);
> > + return;
> > + }
> > +
> > + if (app_wrkr == NULL || nb_wrkr_param == 0) {
> > + RTE_EM_HLPR_LOG_ERR("Invalid args");
> > + return;
> > + }
> > +
> > + /*
> > + * This is a regular worker thread. The application would be
> > + * registering multiple workers with various capabilities. The
> > + * worker to be run will be selected by the capabilities of the
> > + * event device configured.
> > + */
> > +
> > + /* Get the first matching worker for the event device */
> > + match_wrkr = rte_eventmode_helper_find_worker(lcore_id,
> > + em_conf,
> > + app_wrkr,
> > + nb_wrkr_param);
> > +
> > + if (match_wrkr == NULL) {
> > + RTE_EM_HLPR_LOG_ERR(
> > + "No matching worker registered for lcore %d",
> > lcore_id);
> > + goto clean_and_exit;
> > + }
> > +
> > + /* Verify sanity of the matched worker */
> > + if (rte_eventmode_helper_verify_match_worker(match_wrkr) != 1)
> > {
> > + RTE_EM_HLPR_LOG_ERR("Error in validating the matched
> > worker");
> > + goto clean_and_exit;
> > + }
> > +
> > + /* Launch the worker thread */
> > + match_wrkr->worker_thread(mode_conf);
> > +
> > +clean_and_exit:
> > +
> > + /* Flag eth_cores to stop, if started */
> > + rte_eventmode_helper_stop_worker_eth_core();
> > +}
>
> <...Snipped...>
>
> Regards,
> Erik
@@ -135,4 +135,5 @@ EXPERIMENTAL {
rte_eventmode_helper_display_conf;
rte_eventmode_helper_get_event_lcore_links;
rte_eventmode_helper_get_tx_queue;
+ rte_eventmode_helper_launch_worker;
};
@@ -2,6 +2,7 @@
* Copyright (C) 2019 Marvell International Ltd.
*/
#include <getopt.h>
+#include <stdbool.h>
#include <rte_ethdev.h>
#include <rte_eventdev.h>
@@ -13,6 +14,8 @@
#define CMD_LINE_OPT_TRANSFER_MODE "transfer-mode"
+static volatile bool eth_core_running;
+
static const char short_options[] =
""
;
@@ -111,6 +114,16 @@ internal_get_eventdev_params(struct eventmode_conf *em_conf,
return &(em_conf->eventdev_config[i]);
}
+static inline bool
+internal_dev_has_burst_mode(uint8_t dev_id)
+{
+ struct rte_event_dev_info dev_info;
+
+ rte_event_dev_info_get(dev_id, &dev_info);
+ return (dev_info.event_dev_cap & RTE_EVENT_DEV_CAP_BURST_MODE) ?
+ true : false;
+}
+
/* Global functions */
void __rte_experimental
@@ -980,3 +993,230 @@ rte_eventmode_helper_get_tx_queue(struct rte_eventmode_helper_conf *mode_conf,
return eventdev_config->nb_eventqueue - 1;
}
+/* Helper functions for launching workers */
+
+static int32_t
+rte_eventmode_helper_start_worker_eth_core(struct eventmode_conf *em_conf,
+ uint32_t lcore_id)
+{
+ uint32_t service_id[EVENT_MODE_MAX_ADAPTERS_PER_RX_CORE];
+ struct rx_adapter_conf *rx_adapter;
+ int service_count = 0;
+ int adapter_id;
+ int32_t ret;
+ int i;
+
+ RTE_EM_HLPR_LOG_INFO(
+ "Entering eth_core processing on lcore %u", lcore_id);
+
+ /*
+ * Need to parse adapter conf to see which of all Rx adapters need
+ * to be handled by this core.
+ */
+ for (i = 0; i < em_conf->nb_rx_adapter; i++) {
+ /* Check if we have exceeded the max allowed */
+ if (service_count > EVENT_MODE_MAX_ADAPTERS_PER_RX_CORE) {
+ RTE_EM_HLPR_LOG_ERR(
+ "Exceeded the max allowed adapters per rx core");
+ break;
+ }
+
+ rx_adapter = &(em_conf->rx_adapter[i]);
+ if (rx_adapter->rx_core_id != lcore_id)
+ continue;
+
+ /* Adapter need to be handled by this core */
+ adapter_id = rx_adapter->adapter_id;
+
+ /* Get the service ID for the adapters */
+ ret = rte_event_eth_rx_adapter_service_id_get(adapter_id,
+ &(service_id[service_count]));
+
+ if (ret != -ESRCH && ret != 0) {
+ RTE_EM_HLPR_LOG_ERR(
+ "Error getting service ID used by Rx adapter");
+ return ret;
+ }
+
+ /* Update service count */
+ service_count++;
+ }
+
+ eth_core_running = true;
+
+ while (eth_core_running) {
+ for (i = 0; i < service_count; i++) {
+ /* Initiate adapter service */
+ rte_service_run_iter_on_app_lcore(service_id[i], 0);
+ }
+ }
+
+ return 0;
+}
+
+static int32_t
+rte_eventmode_helper_stop_worker_eth_core(void)
+{
+ if (eth_core_running) {
+ RTE_EM_HLPR_LOG_INFO("Stopping rx cores\n");
+ eth_core_running = false;
+ }
+ return 0;
+}
+
+static struct rte_eventmode_helper_app_worker_params *
+rte_eventmode_helper_find_worker(uint32_t lcore_id,
+ struct eventmode_conf *em_conf,
+ struct rte_eventmode_helper_app_worker_params *app_wrkrs,
+ uint8_t nb_wrkr_param)
+{
+ struct rte_eventmode_helper_event_link_info *link = NULL;
+ uint8_t eventdev_id;
+ struct eventdev_params *eventdev_config;
+ int i;
+ struct rte_eventmode_helper_app_worker_params curr_conf = {
+ {{0} }, NULL};
+ struct rte_eventmode_helper_app_worker_params *tmp_wrkr;
+
+ /*
+ * Event device to be used will be derived from the first lcore-event
+ * link.
+ *
+ * Assumption: All lcore-event links tied to a core would be using the
+ * same event device. in other words, one core would be polling on
+ * queues of a single event device only.
+ */
+
+ /* Get a link for this lcore */
+ for (i = 0; i < em_conf->nb_link; i++) {
+ link = &(em_conf->link[i]);
+ if (link->lcore_id == lcore_id)
+ break;
+ }
+
+ if (link == NULL) {
+ RTE_EM_HLPR_LOG_ERR(
+ "No valid link found for lcore(%d)", lcore_id);
+ return NULL;
+ }
+
+ /* Get event dev ID */
+ eventdev_id = link->eventdev_id;
+
+ /* Get the corresponding eventdev config */
+ eventdev_config = internal_get_eventdev_params(em_conf, eventdev_id);
+
+ /* Populate the curr_conf with the capabilities */
+
+ /* Check for burst mode */
+ if (internal_dev_has_burst_mode(eventdev_id))
+ curr_conf.cap.burst = RTE_EVENTMODE_HELPER_RX_TYPE_BURST;
+ else
+ curr_conf.cap.burst = RTE_EVENTMODE_HELPER_RX_TYPE_NON_BURST;
+
+ /* Now parse the passed list and see if we have matching capabilities */
+
+ /* Initialize the pointer used to traverse the list */
+ tmp_wrkr = app_wrkrs;
+
+ for (i = 0; i < nb_wrkr_param; i++, tmp_wrkr++) {
+
+ /* Skip this if capabilities are not matching */
+ if (tmp_wrkr->cap.u64 != curr_conf.cap.u64)
+ continue;
+
+ /* If the checks pass, we have a match */
+ return tmp_wrkr;
+ }
+
+ /* TODO required for ATQ */
+ RTE_SET_USED(eventdev_config);
+
+ return NULL;
+}
+
+static int
+rte_eventmode_helper_verify_match_worker(
+ struct rte_eventmode_helper_app_worker_params *match_wrkr)
+{
+ /* Verify registered worker */
+ if (match_wrkr->worker_thread == NULL) {
+ RTE_EM_HLPR_LOG_ERR("No worker registered for second stage");
+ return 0;
+ }
+
+ /* Success */
+ return 1;
+}
+
+void __rte_experimental
+rte_eventmode_helper_launch_worker(struct rte_eventmode_helper_conf *mode_conf,
+ struct rte_eventmode_helper_app_worker_params *app_wrkr,
+ uint8_t nb_wrkr_param)
+{
+ struct rte_eventmode_helper_app_worker_params *match_wrkr;
+ uint32_t lcore_id;
+ struct eventmode_conf *em_conf;
+
+ if (mode_conf == NULL) {
+ RTE_EM_HLPR_LOG_ERR("Invalid conf");
+ return;
+ }
+
+ if (mode_conf->mode_params == NULL) {
+ RTE_EM_HLPR_LOG_ERR("Invalid mode params");
+ return;
+ }
+
+ /* Get eventmode conf */
+ em_conf = (struct eventmode_conf *)(mode_conf->mode_params);
+
+ /* Get core ID */
+ lcore_id = rte_lcore_id();
+
+ /* TODO check capability for rx core */
+
+ /* Check if this is rx core */
+ if (em_conf->eth_core_mask & (1 << lcore_id)) {
+ rte_eventmode_helper_start_worker_eth_core(em_conf, lcore_id);
+ return;
+ }
+
+ if (app_wrkr == NULL || nb_wrkr_param == 0) {
+ RTE_EM_HLPR_LOG_ERR("Invalid args");
+ return;
+ }
+
+ /*
+ * This is a regular worker thread. The application would be
+ * registering multiple workers with various capabilities. The
+ * worker to be run will be selected by the capabilities of the
+ * event device configured.
+ */
+
+ /* Get the first matching worker for the event device */
+ match_wrkr = rte_eventmode_helper_find_worker(lcore_id,
+ em_conf,
+ app_wrkr,
+ nb_wrkr_param);
+
+ if (match_wrkr == NULL) {
+ RTE_EM_HLPR_LOG_ERR(
+ "No matching worker registered for lcore %d", lcore_id);
+ goto clean_and_exit;
+ }
+
+ /* Verify sanity of the matched worker */
+ if (rte_eventmode_helper_verify_match_worker(match_wrkr) != 1) {
+ RTE_EM_HLPR_LOG_ERR("Error in validating the matched worker");
+ goto clean_and_exit;
+ }
+
+ /* Launch the worker thread */
+ match_wrkr->worker_thread(mode_conf);
+
+clean_and_exit:
+
+ /* Flag eth_cores to stop, if started */
+ rte_eventmode_helper_stop_worker_eth_core();
+}
@@ -17,6 +17,20 @@ enum rte_eventmode_helper_pkt_transfer_mode {
RTE_EVENTMODE_HELPER_PKT_TRANSFER_MODE_EVENT,
};
+/* Event mode packet rx types */
+enum rte_eventmode_helper_rx_types {
+ RTE_EVENTMODE_HELPER_RX_TYPE_INVALID = 0,
+ RTE_EVENTMODE_HELPER_RX_TYPE_NON_BURST,
+ RTE_EVENTMODE_HELPER_RX_TYPE_BURST,
+ RTE_EVENTMODE_HELPER_RX_TYPE_MAX = 16
+};
+
+/* Event mode packet tx types */
+enum rte_eventmode_helper_tx_types {
+ RTE_EVETNMODE_HELPER_TX_TYPE_INVALID = 0,
+ RTE_EVENTMODE_HELPER_TX_TYPE_MAX = 16
+};
+
struct rte_eventmode_helper_conf {
enum rte_eventmode_helper_pkt_transfer_mode mode;
/**< Packet transfer mode of the application */
@@ -41,6 +55,20 @@ struct rte_eventmode_helper_event_link_info {
/**< Lcore to be polling on this port */
};
+/* Workers registered by the application */
+struct rte_eventmode_helper_app_worker_params {
+ union {
+ struct {
+ uint64_t burst : 4;
+ /**< Specify status of rx type burst */
+ };
+ uint64_t u64;
+ } cap;
+ /**< Capabilities of this worker */
+ void (*worker_thread)(void *mode_conf);
+ /**< Worker thread */
+};
+
/* Common helper functions for command line parsing */
/**
@@ -157,6 +185,27 @@ uint8_t __rte_experimental
rte_eventmode_helper_get_tx_queue(struct rte_eventmode_helper_conf *mode_conf,
uint8_t eventdev_id);
+/**
+ * Launch eventmode worker
+ *
+ * The application can request the eventmode helper subsystem to launch the
+ * worker based on the capabilities of event device and the options selected
+ * while initializing the eventmode.
+ *
+ * @param mode_conf
+ * Configuration of the mode in which app is doing packet handling
+ * @param app_wrkr
+ * List of all the workers registered by application, along with it's
+ * capabilities
+ * @param nb_wrkr_param
+ * Number of workers passed by the application
+ *
+ */
+void __rte_experimental
+rte_eventmode_helper_launch_worker(struct rte_eventmode_helper_conf *mode_conf,
+ struct rte_eventmode_helper_app_worker_params *app_wrkr,
+ uint8_t nb_wrkr_param);
+
#ifdef __cplusplus
}
#endif
@@ -48,6 +48,9 @@
#define EVENT_MODE_MAX_LCORE_LINKS \
(EVENT_MODE_MAX_EVENT_DEVS * EVENT_MODE_MAX_EVENT_QUEUES_PER_DEV)
+/* Max adapters that one Rx core can handle */
+#define EVENT_MODE_MAX_ADAPTERS_PER_RX_CORE EVENT_MODE_MAX_RX_ADAPTERS
+
/* Event dev params */
struct eventdev_params {
uint8_t eventdev_id;