[v1,13/13] examples/l3fwd-graph: introduce generic worker model
Checks
Commit Message
Add new parameter "model" to choose generic or rtc worker model.
And in generic model, the node will affinity to worker core successively.
Note:
only support one RX node for remote model in current implementation.
./dpdk-l3fwd-graph -l 8,9,10,11 -n 4 -- -p 0x1 --config="(0,0,9)" -P
--model="generic"
Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
Signed-off-by: Cunming Liang <cunming.liang@intel.com>
Signed-off-by: Zhirun Yan <zhirun.yan@intel.com>
---
examples/l3fwd-graph/main.c | 218 +++++++++++++++++++++++++++++-------
1 file changed, 179 insertions(+), 39 deletions(-)
Comments
On Thu, Nov 17, 2022 at 10:41 AM Zhirun Yan <zhirun.yan@intel.com> wrote:
>
> Add new parameter "model" to choose generic or rtc worker model.
> And in generic model, the node will affinity to worker core successively.
>
> Note:
> only support one RX node for remote model in current implementation.
>
> ./dpdk-l3fwd-graph -l 8,9,10,11 -n 4 -- -p 0x1 --config="(0,0,9)" -P
> --model="generic"
Patch apply issue, please rebase with main.
See https://patches.dpdk.org/project/dpdk/patch/20221117050926.136974-14-zhirun.yan@intel.com/
>
> Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> Signed-off-by: Cunming Liang <cunming.liang@intel.com>
> Signed-off-by: Zhirun Yan <zhirun.yan@intel.com>
> ---
> examples/l3fwd-graph/main.c | 218 +++++++++++++++++++++++++++++-------
> 1 file changed, 179 insertions(+), 39 deletions(-)
>
> diff --git a/examples/l3fwd-graph/main.c b/examples/l3fwd-graph/main.c
> index 6dcb6ee92b..c145a3e3e8 100644
> --- a/examples/l3fwd-graph/main.c
> +++ b/examples/l3fwd-graph/main.c
> @@ -147,6 +147,19 @@ static struct ipv4_l3fwd_lpm_route ipv4_l3fwd_lpm_route_array[] = {
> {RTE_IPV4(198, 18, 6, 0), 24, 6}, {RTE_IPV4(198, 18, 7, 0), 24, 7},
> };
>
> +static int
> +check_worker_model_params(void)
> +{
> + if (rte_graph_worker_model_get() == RTE_GRAPH_MODEL_GENERIC &&
> + nb_lcore_params > 1) {
> + printf("Exceeded max number of lcore params for remote model: %hu\n",
> + nb_lcore_params);
> + return -1;
> + }
> +
> + return 0;
> +}
> +
> static int
> check_lcore_params(void)
> {
> @@ -291,6 +304,20 @@ parse_max_pkt_len(const char *pktlen)
> return len;
> }
>
> +static int
> +parse_worker_model(const char *model)
> +{
> + if (strcmp(model, WORKER_MODEL_DEFAULT) == 0)
> + return RTE_GRAPH_MODEL_DEFAULT;
> + else if (strcmp(model, WORKER_MODEL_GENERIC) == 0) {
> + rte_graph_worker_model_set(RTE_GRAPH_MODEL_GENERIC);
> + return RTE_GRAPH_MODEL_GENERIC;
> + }
> + rte_exit(EXIT_FAILURE, "Invalid worker model: %s", model);
> +
> + return RTE_GRAPH_MODEL_MAX;
> +}
> +
> static int
> parse_portmask(const char *portmask)
> {
> @@ -404,6 +431,7 @@ static const char short_options[] = "p:" /* portmask */
> #define CMD_LINE_OPT_NO_NUMA "no-numa"
> #define CMD_LINE_OPT_MAX_PKT_LEN "max-pkt-len"
> #define CMD_LINE_OPT_PER_PORT_POOL "per-port-pool"
> +#define CMD_LINE_OPT_WORKER_MODEL "model"
> enum {
> /* Long options mapped to a short option */
>
> @@ -416,6 +444,7 @@ enum {
> CMD_LINE_OPT_NO_NUMA_NUM,
> CMD_LINE_OPT_MAX_PKT_LEN_NUM,
> CMD_LINE_OPT_PARSE_PER_PORT_POOL,
> + CMD_LINE_OPT_WORKER_MODEL_TYPE,
> };
>
> static const struct option lgopts[] = {
> @@ -424,6 +453,7 @@ static const struct option lgopts[] = {
> {CMD_LINE_OPT_NO_NUMA, 0, 0, CMD_LINE_OPT_NO_NUMA_NUM},
> {CMD_LINE_OPT_MAX_PKT_LEN, 1, 0, CMD_LINE_OPT_MAX_PKT_LEN_NUM},
> {CMD_LINE_OPT_PER_PORT_POOL, 0, 0, CMD_LINE_OPT_PARSE_PER_PORT_POOL},
> + {CMD_LINE_OPT_WORKER_MODEL, 1, 0, CMD_LINE_OPT_WORKER_MODEL_TYPE},
> {NULL, 0, 0, 0},
> };
>
> @@ -498,6 +528,11 @@ parse_args(int argc, char **argv)
> per_port_pool = 1;
> break;
>
> + case CMD_LINE_OPT_WORKER_MODEL_TYPE:
> + printf("Use new worker model: %s\n", optarg);
> + parse_worker_model(optarg);
> + break;
> +
> default:
> print_usage(prgname);
> return -1;
> @@ -735,6 +770,140 @@ config_port_max_pkt_len(struct rte_eth_conf *conf,
> return 0;
> }
>
> +static void
> +graph_config_generic(struct rte_graph_param graph_conf)
> +{
> + uint16_t nb_patterns = graph_conf.nb_node_patterns;
> + int worker_count = rte_lcore_count() - 1;
> + int main_lcore_id = rte_get_main_lcore();
> + int worker_lcore = main_lcore_id;
> + rte_graph_t main_graph_id = 0;
> + struct rte_node *node_tmp;
> + struct lcore_conf *qconf;
> + struct rte_graph *graph;
> + rte_graph_t graph_id;
> + rte_graph_off_t off;
> + int n_rx_node = 0;
> + rte_node_t count;
> + rte_edge_t i;
> + int ret;
> +
> + for (int j = 0; j < nb_lcore_params; j++) {
> + qconf = &lcore_conf[lcore_params[j].lcore_id];
> + /* Add rx node patterns of all lcore */
> + for (i = 0; i < qconf->n_rx_queue; i++) {
> + char *node_name = qconf->rx_queue_list[i].node_name;
> +
> + graph_conf.node_patterns[nb_patterns + n_rx_node + i] = node_name;
> + n_rx_node++;
> + ret = rte_node_model_generic_set_lcore_affinity(node_name,
> + lcore_params[j].lcore_id);
> + if (ret == 0)
> + printf("Set node %s affinity to lcore %u\n", node_name,
> + lcore_params[j].lcore_id);
> + }
> + }
> +
> + graph_conf.nb_node_patterns = nb_patterns + n_rx_node;
> + graph_conf.socket_id = rte_lcore_to_socket_id(main_lcore_id);
> +
> + snprintf(qconf->name, sizeof(qconf->name), "worker_%u",
> + main_lcore_id);
> +
> + /* create main graph */
> + main_graph_id = rte_graph_create(qconf->name, &graph_conf);
> + if (main_graph_id == RTE_GRAPH_ID_INVALID)
> + rte_exit(EXIT_FAILURE,
> + "rte_graph_create(): main_graph_id invalid for lcore %u\n",
> + main_lcore_id);
> +
> + qconf->graph_id = main_graph_id;
> + qconf->graph = rte_graph_lookup(qconf->name);
> + /* >8 End of graph initialization. */
> + if (!qconf->graph)
> + rte_exit(EXIT_FAILURE,
> + "rte_graph_lookup(): graph %s not found\n",
> + qconf->name);
> +
> + graph = qconf->graph;
> + rte_graph_foreach_node(count, off, graph, node_tmp) {
> + worker_lcore = rte_get_next_lcore(worker_lcore, true, 1);
> +
> + /* Need to set the node Lcore affinity before clone graph for each lcore */
> + if (node_tmp->lcore_id == RTE_MAX_LCORE) {
> + ret = rte_node_model_generic_set_lcore_affinity(node_tmp->name,
> + worker_lcore);
> + if (ret == 0)
> + printf("Set node %s affinity to lcore %u\n",
> + node_tmp->name, worker_lcore);
> + }
> + }
> +
> + worker_lcore = main_lcore_id;
> + for (int i = 0; i < worker_count; i++) {
> + worker_lcore = rte_get_next_lcore(worker_lcore, true, 1);
> +
> + qconf = &lcore_conf[worker_lcore];
> + snprintf(qconf->name, sizeof(qconf->name), "cloned-%u", worker_lcore);
> + graph_id = rte_graph_clone(main_graph_id, qconf->name);
> + ret = rte_graph_bind_core(graph_id, worker_lcore);
> + if (ret == 0)
> + printf("bind graph %d to lcore %u\n", graph_id, worker_lcore);
> +
> + /* full cloned graph name */
> + snprintf(qconf->name, sizeof(qconf->name), "%s",
> + rte_graph_id_to_name(graph_id));
> + qconf->graph_id = graph_id;
> + qconf->graph = rte_graph_lookup(qconf->name);
> + if (!qconf->graph)
> + rte_exit(EXIT_FAILURE,
> + "Failed to lookup graph %s\n",
> + qconf->name);
> + continue;
> + }
> +}
> +
> +static void
> +graph_config_rtc(struct rte_graph_param graph_conf)
> +{
> + uint16_t nb_patterns = graph_conf.nb_node_patterns;
> + struct lcore_conf *qconf;
> + rte_graph_t graph_id;
> + uint32_t lcore_id;
> + rte_edge_t i;
> +
> + for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
> + if (rte_lcore_is_enabled(lcore_id) == 0)
> + continue;
> +
> + qconf = &lcore_conf[lcore_id];
> + /* Skip graph creation if no source exists */
> + if (!qconf->n_rx_queue)
> + continue;
> + /* Add rx node patterns of this lcore */
> + for (i = 0; i < qconf->n_rx_queue; i++) {
> + graph_conf.node_patterns[nb_patterns + i] =
> + qconf->rx_queue_list[i].node_name;
> + }
> + graph_conf.nb_node_patterns = nb_patterns + i;
> + graph_conf.socket_id = rte_lcore_to_socket_id(lcore_id);
> + snprintf(qconf->name, sizeof(qconf->name), "worker_%u",
> + lcore_id);
> + graph_id = rte_graph_create(qconf->name, &graph_conf);
> + if (graph_id == RTE_GRAPH_ID_INVALID)
> + rte_exit(EXIT_FAILURE,
> + "rte_graph_create(): graph_id invalid for lcore %u\n",
> + lcore_id);
> + qconf->graph_id = graph_id;
> + qconf->graph = rte_graph_lookup(qconf->name);
> + /* >8 End of graph initialization. */
> + if (!qconf->graph)
> + rte_exit(EXIT_FAILURE,
> + "rte_graph_lookup(): graph %s not found\n",
> + qconf->name);
> + }
> +}
> +
> int
> main(int argc, char **argv)
> {
> @@ -759,6 +928,7 @@ main(int argc, char **argv)
> uint16_t nb_patterns;
> uint8_t rewrite_len;
> uint32_t lcore_id;
> + uint16_t model;
> int ret;
>
> /* Init EAL */
> @@ -787,6 +957,9 @@ main(int argc, char **argv)
> if (check_lcore_params() < 0)
> rte_exit(EXIT_FAILURE, "check_lcore_params() failed\n");
>
> + if (check_worker_model_params() < 0)
> + rte_exit(EXIT_FAILURE, "check_worker_model_params() failed\n");
> +
> ret = init_lcore_rx_queues();
> if (ret < 0)
> rte_exit(EXIT_FAILURE, "init_lcore_rx_queues() failed\n");
> @@ -1026,46 +1199,13 @@ main(int argc, char **argv)
>
> memset(&graph_conf, 0, sizeof(graph_conf));
> graph_conf.node_patterns = node_patterns;
> + graph_conf.nb_node_patterns = nb_patterns;
>
> - for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
> - rte_graph_t graph_id;
> - rte_edge_t i;
> -
> - if (rte_lcore_is_enabled(lcore_id) == 0)
> - continue;
> -
> - qconf = &lcore_conf[lcore_id];
> -
> - /* Skip graph creation if no source exists */
> - if (!qconf->n_rx_queue)
> - continue;
> -
> - /* Add rx node patterns of this lcore */
> - for (i = 0; i < qconf->n_rx_queue; i++) {
> - graph_conf.node_patterns[nb_patterns + i] =
> - qconf->rx_queue_list[i].node_name;
> - }
> -
> - graph_conf.nb_node_patterns = nb_patterns + i;
> - graph_conf.socket_id = rte_lcore_to_socket_id(lcore_id);
> -
> - snprintf(qconf->name, sizeof(qconf->name), "worker_%u",
> - lcore_id);
> -
> - graph_id = rte_graph_create(qconf->name, &graph_conf);
> - if (graph_id == RTE_GRAPH_ID_INVALID)
> - rte_exit(EXIT_FAILURE,
> - "rte_graph_create(): graph_id invalid"
> - " for lcore %u\n", lcore_id);
> -
> - qconf->graph_id = graph_id;
> - qconf->graph = rte_graph_lookup(qconf->name);
> - /* >8 End of graph initialization. */
> - if (!qconf->graph)
> - rte_exit(EXIT_FAILURE,
> - "rte_graph_lookup(): graph %s not found\n",
> - qconf->name);
> - }
> + model = rte_graph_worker_model_get();
> + if (model == RTE_GRAPH_MODEL_DEFAULT)
> + graph_config_rtc(graph_conf);
> + else if (model == RTE_GRAPH_MODEL_GENERIC)
> + graph_config_generic(graph_conf);
>
> memset(&rewrite_data, 0, sizeof(rewrite_data));
> rewrite_len = sizeof(rewrite_data);
> --
> 2.25.1
>
> -----Original Message-----
> From: Jerin Jacob <jerinjacobk@gmail.com>
> Sent: Monday, February 20, 2023 10:20 PM
> To: Yan, Zhirun <zhirun.yan@intel.com>
> Cc: dev@dpdk.org; jerinj@marvell.com; kirankumark@marvell.com;
> ndabilpuram@marvell.com; Liang, Cunming <cunming.liang@intel.com>; Wang,
> Haiyue <haiyue.wang@intel.com>
> Subject: Re: [PATCH v1 13/13] examples/l3fwd-graph: introduce generic worker
> model
>
> On Thu, Nov 17, 2022 at 10:41 AM Zhirun Yan <zhirun.yan@intel.com> wrote:
> >
> > Add new parameter "model" to choose generic or rtc worker model.
> > And in generic model, the node will affinity to worker core successively.
> >
> > Note:
> > only support one RX node for remote model in current implementation.
> >
> > ./dpdk-l3fwd-graph -l 8,9,10,11 -n 4 -- -p 0x1 --config="(0,0,9)" -P
> > --model="generic"
>
> Patch apply issue, please rebase with main.
> See https://patches.dpdk.org/project/dpdk/patch/20221117050926.136974-14-
> zhirun.yan@intel.com/
>
Will fix in next version. Thanks for your comments.
> >
> > Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
> > Signed-off-by: Cunming Liang <cunming.liang@intel.com>
> > Signed-off-by: Zhirun Yan <zhirun.yan@intel.com>
> > ---
> > examples/l3fwd-graph/main.c | 218
> > +++++++++++++++++++++++++++++-------
> > 1 file changed, 179 insertions(+), 39 deletions(-)
> >
> > diff --git a/examples/l3fwd-graph/main.c b/examples/l3fwd-graph/main.c
> > index 6dcb6ee92b..c145a3e3e8 100644
> > --- a/examples/l3fwd-graph/main.c
> > +++ b/examples/l3fwd-graph/main.c
> > @@ -147,6 +147,19 @@ static struct ipv4_l3fwd_lpm_route
> ipv4_l3fwd_lpm_route_array[] = {
> > {RTE_IPV4(198, 18, 6, 0), 24, 6}, {RTE_IPV4(198, 18, 7, 0),
> > 24, 7}, };
> >
> > +static int
> > +check_worker_model_params(void)
> > +{
> > + if (rte_graph_worker_model_get() == RTE_GRAPH_MODEL_GENERIC &&
> > + nb_lcore_params > 1) {
> > + printf("Exceeded max number of lcore params for remote
> model: %hu\n",
> > + nb_lcore_params);
> > + return -1;
> > + }
> > +
> > + return 0;
> > +}
> > +
> > static int
> > check_lcore_params(void)
> > {
> > @@ -291,6 +304,20 @@ parse_max_pkt_len(const char *pktlen)
> > return len;
> > }
> >
> > +static int
> > +parse_worker_model(const char *model) {
> > + if (strcmp(model, WORKER_MODEL_DEFAULT) == 0)
> > + return RTE_GRAPH_MODEL_DEFAULT;
> > + else if (strcmp(model, WORKER_MODEL_GENERIC) == 0) {
> > + rte_graph_worker_model_set(RTE_GRAPH_MODEL_GENERIC);
> > + return RTE_GRAPH_MODEL_GENERIC;
> > + }
> > + rte_exit(EXIT_FAILURE, "Invalid worker model: %s", model);
> > +
> > + return RTE_GRAPH_MODEL_MAX;
> > +}
> > +
> > static int
> > parse_portmask(const char *portmask)
> > {
> > @@ -404,6 +431,7 @@ static const char short_options[] = "p:" /* portmask */
> > #define CMD_LINE_OPT_NO_NUMA "no-numa"
> > #define CMD_LINE_OPT_MAX_PKT_LEN "max-pkt-len"
> > #define CMD_LINE_OPT_PER_PORT_POOL "per-port-pool"
> > +#define CMD_LINE_OPT_WORKER_MODEL "model"
> > enum {
> > /* Long options mapped to a short option */
> >
> > @@ -416,6 +444,7 @@ enum {
> > CMD_LINE_OPT_NO_NUMA_NUM,
> > CMD_LINE_OPT_MAX_PKT_LEN_NUM,
> > CMD_LINE_OPT_PARSE_PER_PORT_POOL,
> > + CMD_LINE_OPT_WORKER_MODEL_TYPE,
> > };
> >
> > static const struct option lgopts[] = { @@ -424,6 +453,7 @@ static
> > const struct option lgopts[] = {
> > {CMD_LINE_OPT_NO_NUMA, 0, 0, CMD_LINE_OPT_NO_NUMA_NUM},
> > {CMD_LINE_OPT_MAX_PKT_LEN, 1, 0,
> CMD_LINE_OPT_MAX_PKT_LEN_NUM},
> > {CMD_LINE_OPT_PER_PORT_POOL, 0, 0,
> > CMD_LINE_OPT_PARSE_PER_PORT_POOL},
> > + {CMD_LINE_OPT_WORKER_MODEL, 1, 0,
> > + CMD_LINE_OPT_WORKER_MODEL_TYPE},
> > {NULL, 0, 0, 0},
> > };
> >
> > @@ -498,6 +528,11 @@ parse_args(int argc, char **argv)
> > per_port_pool = 1;
> > break;
> >
> > + case CMD_LINE_OPT_WORKER_MODEL_TYPE:
> > + printf("Use new worker model: %s\n", optarg);
> > + parse_worker_model(optarg);
> > + break;
> > +
> > default:
> > print_usage(prgname);
> > return -1;
> > @@ -735,6 +770,140 @@ config_port_max_pkt_len(struct rte_eth_conf
> *conf,
> > return 0;
> > }
> >
> > +static void
> > +graph_config_generic(struct rte_graph_param graph_conf) {
> > + uint16_t nb_patterns = graph_conf.nb_node_patterns;
> > + int worker_count = rte_lcore_count() - 1;
> > + int main_lcore_id = rte_get_main_lcore();
> > + int worker_lcore = main_lcore_id;
> > + rte_graph_t main_graph_id = 0;
> > + struct rte_node *node_tmp;
> > + struct lcore_conf *qconf;
> > + struct rte_graph *graph;
> > + rte_graph_t graph_id;
> > + rte_graph_off_t off;
> > + int n_rx_node = 0;
> > + rte_node_t count;
> > + rte_edge_t i;
> > + int ret;
> > +
> > + for (int j = 0; j < nb_lcore_params; j++) {
> > + qconf = &lcore_conf[lcore_params[j].lcore_id];
> > + /* Add rx node patterns of all lcore */
> > + for (i = 0; i < qconf->n_rx_queue; i++) {
> > + char *node_name =
> > + qconf->rx_queue_list[i].node_name;
> > +
> > + graph_conf.node_patterns[nb_patterns + n_rx_node + i] =
> node_name;
> > + n_rx_node++;
> > + ret = rte_node_model_generic_set_lcore_affinity(node_name,
> > + lcore_params[j].lcore_id);
> > + if (ret == 0)
> > + printf("Set node %s affinity to lcore %u\n", node_name,
> > + lcore_params[j].lcore_id);
> > + }
> > + }
> > +
> > + graph_conf.nb_node_patterns = nb_patterns + n_rx_node;
> > + graph_conf.socket_id = rte_lcore_to_socket_id(main_lcore_id);
> > +
> > + snprintf(qconf->name, sizeof(qconf->name), "worker_%u",
> > + main_lcore_id);
> > +
> > + /* create main graph */
> > + main_graph_id = rte_graph_create(qconf->name, &graph_conf);
> > + if (main_graph_id == RTE_GRAPH_ID_INVALID)
> > + rte_exit(EXIT_FAILURE,
> > + "rte_graph_create(): main_graph_id invalid for lcore %u\n",
> > + main_lcore_id);
> > +
> > + qconf->graph_id = main_graph_id;
> > + qconf->graph = rte_graph_lookup(qconf->name);
> > + /* >8 End of graph initialization. */
> > + if (!qconf->graph)
> > + rte_exit(EXIT_FAILURE,
> > + "rte_graph_lookup(): graph %s not found\n",
> > + qconf->name);
> > +
> > + graph = qconf->graph;
> > + rte_graph_foreach_node(count, off, graph, node_tmp) {
> > + worker_lcore = rte_get_next_lcore(worker_lcore, true,
> > + 1);
> > +
> > + /* Need to set the node Lcore affinity before clone graph for each
> lcore */
> > + if (node_tmp->lcore_id == RTE_MAX_LCORE) {
> > + ret = rte_node_model_generic_set_lcore_affinity(node_tmp-
> >name,
> > + worker_lcore);
> > + if (ret == 0)
> > + printf("Set node %s affinity to lcore %u\n",
> > + node_tmp->name, worker_lcore);
> > + }
> > + }
> > +
> > + worker_lcore = main_lcore_id;
> > + for (int i = 0; i < worker_count; i++) {
> > + worker_lcore = rte_get_next_lcore(worker_lcore, true,
> > + 1);
> > +
> > + qconf = &lcore_conf[worker_lcore];
> > + snprintf(qconf->name, sizeof(qconf->name), "cloned-%u",
> worker_lcore);
> > + graph_id = rte_graph_clone(main_graph_id, qconf->name);
> > + ret = rte_graph_bind_core(graph_id, worker_lcore);
> > + if (ret == 0)
> > + printf("bind graph %d to lcore %u\n",
> > + graph_id, worker_lcore);
> > +
> > + /* full cloned graph name */
> > + snprintf(qconf->name, sizeof(qconf->name), "%s",
> > + rte_graph_id_to_name(graph_id));
> > + qconf->graph_id = graph_id;
> > + qconf->graph = rte_graph_lookup(qconf->name);
> > + if (!qconf->graph)
> > + rte_exit(EXIT_FAILURE,
> > + "Failed to lookup graph %s\n",
> > + qconf->name);
> > + continue;
> > + }
> > +}
> > +
> > +static void
> > +graph_config_rtc(struct rte_graph_param graph_conf) {
> > + uint16_t nb_patterns = graph_conf.nb_node_patterns;
> > + struct lcore_conf *qconf;
> > + rte_graph_t graph_id;
> > + uint32_t lcore_id;
> > + rte_edge_t i;
> > +
> > + for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
> > + if (rte_lcore_is_enabled(lcore_id) == 0)
> > + continue;
> > +
> > + qconf = &lcore_conf[lcore_id];
> > + /* Skip graph creation if no source exists */
> > + if (!qconf->n_rx_queue)
> > + continue;
> > + /* Add rx node patterns of this lcore */
> > + for (i = 0; i < qconf->n_rx_queue; i++) {
> > + graph_conf.node_patterns[nb_patterns + i] =
> > + qconf->rx_queue_list[i].node_name;
> > + }
> > + graph_conf.nb_node_patterns = nb_patterns + i;
> > + graph_conf.socket_id = rte_lcore_to_socket_id(lcore_id);
> > + snprintf(qconf->name, sizeof(qconf->name), "worker_%u",
> > + lcore_id);
> > + graph_id = rte_graph_create(qconf->name, &graph_conf);
> > + if (graph_id == RTE_GRAPH_ID_INVALID)
> > + rte_exit(EXIT_FAILURE,
> > + "rte_graph_create(): graph_id invalid for lcore %u\n",
> > + lcore_id);
> > + qconf->graph_id = graph_id;
> > + qconf->graph = rte_graph_lookup(qconf->name);
> > + /* >8 End of graph initialization. */
> > + if (!qconf->graph)
> > + rte_exit(EXIT_FAILURE,
> > + "rte_graph_lookup(): graph %s not found\n",
> > + qconf->name);
> > + }
> > +}
> > +
> > int
> > main(int argc, char **argv)
> > {
> > @@ -759,6 +928,7 @@ main(int argc, char **argv)
> > uint16_t nb_patterns;
> > uint8_t rewrite_len;
> > uint32_t lcore_id;
> > + uint16_t model;
> > int ret;
> >
> > /* Init EAL */
> > @@ -787,6 +957,9 @@ main(int argc, char **argv)
> > if (check_lcore_params() < 0)
> > rte_exit(EXIT_FAILURE, "check_lcore_params()
> > failed\n");
> >
> > + if (check_worker_model_params() < 0)
> > + rte_exit(EXIT_FAILURE, "check_worker_model_params()
> > + failed\n");
> > +
> > ret = init_lcore_rx_queues();
> > if (ret < 0)
> > rte_exit(EXIT_FAILURE, "init_lcore_rx_queues()
> > failed\n"); @@ -1026,46 +1199,13 @@ main(int argc, char **argv)
> >
> > memset(&graph_conf, 0, sizeof(graph_conf));
> > graph_conf.node_patterns = node_patterns;
> > + graph_conf.nb_node_patterns = nb_patterns;
> >
> > - for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
> > - rte_graph_t graph_id;
> > - rte_edge_t i;
> > -
> > - if (rte_lcore_is_enabled(lcore_id) == 0)
> > - continue;
> > -
> > - qconf = &lcore_conf[lcore_id];
> > -
> > - /* Skip graph creation if no source exists */
> > - if (!qconf->n_rx_queue)
> > - continue;
> > -
> > - /* Add rx node patterns of this lcore */
> > - for (i = 0; i < qconf->n_rx_queue; i++) {
> > - graph_conf.node_patterns[nb_patterns + i] =
> > - qconf->rx_queue_list[i].node_name;
> > - }
> > -
> > - graph_conf.nb_node_patterns = nb_patterns + i;
> > - graph_conf.socket_id = rte_lcore_to_socket_id(lcore_id);
> > -
> > - snprintf(qconf->name, sizeof(qconf->name), "worker_%u",
> > - lcore_id);
> > -
> > - graph_id = rte_graph_create(qconf->name, &graph_conf);
> > - if (graph_id == RTE_GRAPH_ID_INVALID)
> > - rte_exit(EXIT_FAILURE,
> > - "rte_graph_create(): graph_id invalid"
> > - " for lcore %u\n", lcore_id);
> > -
> > - qconf->graph_id = graph_id;
> > - qconf->graph = rte_graph_lookup(qconf->name);
> > - /* >8 End of graph initialization. */
> > - if (!qconf->graph)
> > - rte_exit(EXIT_FAILURE,
> > - "rte_graph_lookup(): graph %s not found\n",
> > - qconf->name);
> > - }
> > + model = rte_graph_worker_model_get();
> > + if (model == RTE_GRAPH_MODEL_DEFAULT)
> > + graph_config_rtc(graph_conf);
> > + else if (model == RTE_GRAPH_MODEL_GENERIC)
> > + graph_config_generic(graph_conf);
> >
> > memset(&rewrite_data, 0, sizeof(rewrite_data));
> > rewrite_len = sizeof(rewrite_data);
> > --
> > 2.25.1
> >
@@ -147,6 +147,19 @@ static struct ipv4_l3fwd_lpm_route ipv4_l3fwd_lpm_route_array[] = {
{RTE_IPV4(198, 18, 6, 0), 24, 6}, {RTE_IPV4(198, 18, 7, 0), 24, 7},
};
+static int
+check_worker_model_params(void)
+{
+ if (rte_graph_worker_model_get() == RTE_GRAPH_MODEL_GENERIC &&
+ nb_lcore_params > 1) {
+ printf("Exceeded max number of lcore params for remote model: %hu\n",
+ nb_lcore_params);
+ return -1;
+ }
+
+ return 0;
+}
+
static int
check_lcore_params(void)
{
@@ -291,6 +304,20 @@ parse_max_pkt_len(const char *pktlen)
return len;
}
+static int
+parse_worker_model(const char *model)
+{
+ if (strcmp(model, WORKER_MODEL_DEFAULT) == 0)
+ return RTE_GRAPH_MODEL_DEFAULT;
+ else if (strcmp(model, WORKER_MODEL_GENERIC) == 0) {
+ rte_graph_worker_model_set(RTE_GRAPH_MODEL_GENERIC);
+ return RTE_GRAPH_MODEL_GENERIC;
+ }
+ rte_exit(EXIT_FAILURE, "Invalid worker model: %s", model);
+
+ return RTE_GRAPH_MODEL_MAX;
+}
+
static int
parse_portmask(const char *portmask)
{
@@ -404,6 +431,7 @@ static const char short_options[] = "p:" /* portmask */
#define CMD_LINE_OPT_NO_NUMA "no-numa"
#define CMD_LINE_OPT_MAX_PKT_LEN "max-pkt-len"
#define CMD_LINE_OPT_PER_PORT_POOL "per-port-pool"
+#define CMD_LINE_OPT_WORKER_MODEL "model"
enum {
/* Long options mapped to a short option */
@@ -416,6 +444,7 @@ enum {
CMD_LINE_OPT_NO_NUMA_NUM,
CMD_LINE_OPT_MAX_PKT_LEN_NUM,
CMD_LINE_OPT_PARSE_PER_PORT_POOL,
+ CMD_LINE_OPT_WORKER_MODEL_TYPE,
};
static const struct option lgopts[] = {
@@ -424,6 +453,7 @@ static const struct option lgopts[] = {
{CMD_LINE_OPT_NO_NUMA, 0, 0, CMD_LINE_OPT_NO_NUMA_NUM},
{CMD_LINE_OPT_MAX_PKT_LEN, 1, 0, CMD_LINE_OPT_MAX_PKT_LEN_NUM},
{CMD_LINE_OPT_PER_PORT_POOL, 0, 0, CMD_LINE_OPT_PARSE_PER_PORT_POOL},
+ {CMD_LINE_OPT_WORKER_MODEL, 1, 0, CMD_LINE_OPT_WORKER_MODEL_TYPE},
{NULL, 0, 0, 0},
};
@@ -498,6 +528,11 @@ parse_args(int argc, char **argv)
per_port_pool = 1;
break;
+ case CMD_LINE_OPT_WORKER_MODEL_TYPE:
+ printf("Use new worker model: %s\n", optarg);
+ parse_worker_model(optarg);
+ break;
+
default:
print_usage(prgname);
return -1;
@@ -735,6 +770,140 @@ config_port_max_pkt_len(struct rte_eth_conf *conf,
return 0;
}
+static void
+graph_config_generic(struct rte_graph_param graph_conf)
+{
+ uint16_t nb_patterns = graph_conf.nb_node_patterns;
+ int worker_count = rte_lcore_count() - 1;
+ int main_lcore_id = rte_get_main_lcore();
+ int worker_lcore = main_lcore_id;
+ rte_graph_t main_graph_id = 0;
+ struct rte_node *node_tmp;
+ struct lcore_conf *qconf;
+ struct rte_graph *graph;
+ rte_graph_t graph_id;
+ rte_graph_off_t off;
+ int n_rx_node = 0;
+ rte_node_t count;
+ rte_edge_t i;
+ int ret;
+
+ for (int j = 0; j < nb_lcore_params; j++) {
+ qconf = &lcore_conf[lcore_params[j].lcore_id];
+ /* Add rx node patterns of all lcore */
+ for (i = 0; i < qconf->n_rx_queue; i++) {
+ char *node_name = qconf->rx_queue_list[i].node_name;
+
+ graph_conf.node_patterns[nb_patterns + n_rx_node + i] = node_name;
+ n_rx_node++;
+ ret = rte_node_model_generic_set_lcore_affinity(node_name,
+ lcore_params[j].lcore_id);
+ if (ret == 0)
+ printf("Set node %s affinity to lcore %u\n", node_name,
+ lcore_params[j].lcore_id);
+ }
+ }
+
+ graph_conf.nb_node_patterns = nb_patterns + n_rx_node;
+ graph_conf.socket_id = rte_lcore_to_socket_id(main_lcore_id);
+
+ snprintf(qconf->name, sizeof(qconf->name), "worker_%u",
+ main_lcore_id);
+
+ /* create main graph */
+ main_graph_id = rte_graph_create(qconf->name, &graph_conf);
+ if (main_graph_id == RTE_GRAPH_ID_INVALID)
+ rte_exit(EXIT_FAILURE,
+ "rte_graph_create(): main_graph_id invalid for lcore %u\n",
+ main_lcore_id);
+
+ qconf->graph_id = main_graph_id;
+ qconf->graph = rte_graph_lookup(qconf->name);
+ /* >8 End of graph initialization. */
+ if (!qconf->graph)
+ rte_exit(EXIT_FAILURE,
+ "rte_graph_lookup(): graph %s not found\n",
+ qconf->name);
+
+ graph = qconf->graph;
+ rte_graph_foreach_node(count, off, graph, node_tmp) {
+ worker_lcore = rte_get_next_lcore(worker_lcore, true, 1);
+
+ /* Need to set the node Lcore affinity before clone graph for each lcore */
+ if (node_tmp->lcore_id == RTE_MAX_LCORE) {
+ ret = rte_node_model_generic_set_lcore_affinity(node_tmp->name,
+ worker_lcore);
+ if (ret == 0)
+ printf("Set node %s affinity to lcore %u\n",
+ node_tmp->name, worker_lcore);
+ }
+ }
+
+ worker_lcore = main_lcore_id;
+ for (int i = 0; i < worker_count; i++) {
+ worker_lcore = rte_get_next_lcore(worker_lcore, true, 1);
+
+ qconf = &lcore_conf[worker_lcore];
+ snprintf(qconf->name, sizeof(qconf->name), "cloned-%u", worker_lcore);
+ graph_id = rte_graph_clone(main_graph_id, qconf->name);
+ ret = rte_graph_bind_core(graph_id, worker_lcore);
+ if (ret == 0)
+ printf("bind graph %d to lcore %u\n", graph_id, worker_lcore);
+
+ /* full cloned graph name */
+ snprintf(qconf->name, sizeof(qconf->name), "%s",
+ rte_graph_id_to_name(graph_id));
+ qconf->graph_id = graph_id;
+ qconf->graph = rte_graph_lookup(qconf->name);
+ if (!qconf->graph)
+ rte_exit(EXIT_FAILURE,
+ "Failed to lookup graph %s\n",
+ qconf->name);
+ continue;
+ }
+}
+
+static void
+graph_config_rtc(struct rte_graph_param graph_conf)
+{
+ uint16_t nb_patterns = graph_conf.nb_node_patterns;
+ struct lcore_conf *qconf;
+ rte_graph_t graph_id;
+ uint32_t lcore_id;
+ rte_edge_t i;
+
+ for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
+ if (rte_lcore_is_enabled(lcore_id) == 0)
+ continue;
+
+ qconf = &lcore_conf[lcore_id];
+ /* Skip graph creation if no source exists */
+ if (!qconf->n_rx_queue)
+ continue;
+ /* Add rx node patterns of this lcore */
+ for (i = 0; i < qconf->n_rx_queue; i++) {
+ graph_conf.node_patterns[nb_patterns + i] =
+ qconf->rx_queue_list[i].node_name;
+ }
+ graph_conf.nb_node_patterns = nb_patterns + i;
+ graph_conf.socket_id = rte_lcore_to_socket_id(lcore_id);
+ snprintf(qconf->name, sizeof(qconf->name), "worker_%u",
+ lcore_id);
+ graph_id = rte_graph_create(qconf->name, &graph_conf);
+ if (graph_id == RTE_GRAPH_ID_INVALID)
+ rte_exit(EXIT_FAILURE,
+ "rte_graph_create(): graph_id invalid for lcore %u\n",
+ lcore_id);
+ qconf->graph_id = graph_id;
+ qconf->graph = rte_graph_lookup(qconf->name);
+ /* >8 End of graph initialization. */
+ if (!qconf->graph)
+ rte_exit(EXIT_FAILURE,
+ "rte_graph_lookup(): graph %s not found\n",
+ qconf->name);
+ }
+}
+
int
main(int argc, char **argv)
{
@@ -759,6 +928,7 @@ main(int argc, char **argv)
uint16_t nb_patterns;
uint8_t rewrite_len;
uint32_t lcore_id;
+ uint16_t model;
int ret;
/* Init EAL */
@@ -787,6 +957,9 @@ main(int argc, char **argv)
if (check_lcore_params() < 0)
rte_exit(EXIT_FAILURE, "check_lcore_params() failed\n");
+ if (check_worker_model_params() < 0)
+ rte_exit(EXIT_FAILURE, "check_worker_model_params() failed\n");
+
ret = init_lcore_rx_queues();
if (ret < 0)
rte_exit(EXIT_FAILURE, "init_lcore_rx_queues() failed\n");
@@ -1026,46 +1199,13 @@ main(int argc, char **argv)
memset(&graph_conf, 0, sizeof(graph_conf));
graph_conf.node_patterns = node_patterns;
+ graph_conf.nb_node_patterns = nb_patterns;
- for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
- rte_graph_t graph_id;
- rte_edge_t i;
-
- if (rte_lcore_is_enabled(lcore_id) == 0)
- continue;
-
- qconf = &lcore_conf[lcore_id];
-
- /* Skip graph creation if no source exists */
- if (!qconf->n_rx_queue)
- continue;
-
- /* Add rx node patterns of this lcore */
- for (i = 0; i < qconf->n_rx_queue; i++) {
- graph_conf.node_patterns[nb_patterns + i] =
- qconf->rx_queue_list[i].node_name;
- }
-
- graph_conf.nb_node_patterns = nb_patterns + i;
- graph_conf.socket_id = rte_lcore_to_socket_id(lcore_id);
-
- snprintf(qconf->name, sizeof(qconf->name), "worker_%u",
- lcore_id);
-
- graph_id = rte_graph_create(qconf->name, &graph_conf);
- if (graph_id == RTE_GRAPH_ID_INVALID)
- rte_exit(EXIT_FAILURE,
- "rte_graph_create(): graph_id invalid"
- " for lcore %u\n", lcore_id);
-
- qconf->graph_id = graph_id;
- qconf->graph = rte_graph_lookup(qconf->name);
- /* >8 End of graph initialization. */
- if (!qconf->graph)
- rte_exit(EXIT_FAILURE,
- "rte_graph_lookup(): graph %s not found\n",
- qconf->name);
- }
+ model = rte_graph_worker_model_get();
+ if (model == RTE_GRAPH_MODEL_DEFAULT)
+ graph_config_rtc(graph_conf);
+ else if (model == RTE_GRAPH_MODEL_GENERIC)
+ graph_config_generic(graph_conf);
memset(&rewrite_data, 0, sizeof(rewrite_data));
rewrite_len = sizeof(rewrite_data);