[RFC,v1,6/6] examples: add l2fwd-graph

Message ID 20220908020959.1675953-7-zhirun.yan@intel.com (mailing list archive)
State Superseded, archived
Delegated to: Thomas Monjalon
Headers
Series graph enhancement for multi-core dispatch |

Checks

Context Check Description
ci/checkpatch warning coding style issues
ci/Intel-compilation fail Compilation issues
ci/intel-Testing success Testing PASS

Commit Message

Yan, Zhirun Sept. 8, 2022, 2:09 a.m. UTC
  From: Haiyue Wang <haiyue.wang@intel.com>

l2fwd with graph function.
Adding 3 nodes for quick test, and the node will affinity to worker
core successively.

./dpdk-l2fwd-graph -l 7,8,9,10 -n 4 -a 0000:4b:00.0 --  -P

Signed-off-by: Haiyue Wang <haiyue.wang@intel.com>
Signed-off-by: Cunming Liang <cunming.liang@intel.com>
Signed-off-by: Zhirun Yan <zhirun.yan@intel.com>
---
 examples/l2fwd-graph/main.c      | 455 +++++++++++++++++++++++++++++++
 examples/l2fwd-graph/meson.build |  25 ++
 examples/l2fwd-graph/node.c      | 263 ++++++++++++++++++
 examples/l2fwd-graph/node.h      |  64 +++++
 examples/meson.build             |   1 +
 5 files changed, 808 insertions(+)
 create mode 100644 examples/l2fwd-graph/main.c
 create mode 100644 examples/l2fwd-graph/meson.build
 create mode 100644 examples/l2fwd-graph/node.c
 create mode 100644 examples/l2fwd-graph/node.h
  

Patch

diff --git a/examples/l2fwd-graph/main.c b/examples/l2fwd-graph/main.c
new file mode 100644
index 0000000000..88ffd84340
--- /dev/null
+++ b/examples/l2fwd-graph/main.c
@@ -0,0 +1,455 @@ 
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(c) 2022 Intel Corporation
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <inttypes.h>
+#include <sys/types.h>
+#include <sys/queue.h>
+#include <netinet/in.h>
+#include <setjmp.h>
+#include <stdarg.h>
+#include <ctype.h>
+#include <errno.h>
+#include <getopt.h>
+#include <signal.h>
+#include <stdbool.h>
+
+#include <rte_common.h>
+#include <rte_log.h>
+#include <rte_malloc.h>
+#include <rte_memory.h>
+#include <rte_memcpy.h>
+#include <rte_eal.h>
+#include <rte_launch.h>
+#include <rte_atomic.h>
+#include <rte_cycles.h>
+#include <rte_prefetch.h>
+#include <rte_lcore.h>
+#include <rte_per_lcore.h>
+#include <rte_branch_prediction.h>
+#include <rte_interrupts.h>
+#include <rte_random.h>
+#include <rte_debug.h>
+#include <rte_ether.h>
+#include <rte_ethdev.h>
+#include <rte_mempool.h>
+#include <rte_mbuf.h>
+#include <rte_string_fns.h>
+#include <rte_graph_worker.h>
+
+#include "node.h"
+
+#define MAX_PKT_BURST 32
+#define MEMPOOL_CACHE_SIZE 256
+#define RTE_TEST_RX_DESC_DEFAULT 1024
+#define RTE_TEST_TX_DESC_DEFAULT 1024
+
+/* ethernet addresses of ports */
+struct rte_ether_addr l2fwd_ports_eth_addr[RTE_MAX_ETHPORTS];
+
+static struct rte_eth_conf default_port_conf = {
+	.rxmode = {
+		.mq_mode = RTE_ETH_MQ_RX_RSS,
+		.split_hdr_size = 0,
+	},
+	.rx_adv_conf = {
+		.rss_conf = {
+			.rss_key = NULL,
+			.rss_hf = RTE_ETH_RSS_IP,
+		},
+	},
+	.txmode = {
+		.mq_mode = RTE_ETH_MQ_TX_NONE,
+	},
+};
+
+static struct rte_mempool *l2fwd_pktmbuf_pool;
+
+static volatile bool force_quit;
+static uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT;
+static uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT;
+static uint16_t nb_rxq = 1;
+
+/* Lcore conf */
+struct lcore_conf {
+	int enable;
+	uint16_t port_id;
+
+	struct rte_graph *graph;
+	char name[RTE_GRAPH_NAMESIZE];
+	rte_graph_t graph_id;
+} __rte_cache_aligned;
+
+static struct lcore_conf l2fwd_lcore_conf[RTE_MAX_LCORE];
+
+#define L2FWD_GRAPH_NAME_PREFIX "l2fwd_graph_"
+
+static int
+l2fwd_graph_loop(void *conf)
+{
+	struct lcore_conf *lconf;
+	struct rte_graph *graph;
+	uint32_t lcore_id;
+
+	RTE_SET_USED(conf);
+
+	lcore_id = rte_lcore_id();
+	lconf = &l2fwd_lcore_conf[lcore_id];
+	graph = lconf->graph;
+
+	if (!graph) {
+		RTE_LOG(INFO, L2FWD_GRAPH, "Lcore %u has nothing to do\n",
+			lcore_id);
+		return 0;
+	}
+
+	RTE_LOG(INFO, L2FWD_GRAPH,
+		"Entering graph loop on lcore %u, graph %s\n",
+		lcore_id, graph->name);
+
+	while (likely(!force_quit))
+		rte_graph_walk(graph);
+
+	RTE_LOG(INFO, L2FWD_GRAPH,
+		"Leaving graph loop on lcore %u, graph %s\n",
+		lcore_id, graph->name);
+
+	return 0;
+}
+
+static void
+print_stats(void)
+{
+	const char topLeft[] = {27, '[', '1', ';', '1', 'H', '\0'};
+	const char clr[] = {27, '[', '2', 'J', '\0'};
+	struct rte_graph_cluster_stats_param s_param;
+	struct rte_graph_cluster_stats *stats;
+	const char *pattern = L2FWD_GRAPH_NAME_PREFIX "*";
+
+	/* Prepare stats object */
+	memset(&s_param, 0, sizeof(s_param));
+	s_param.f = stdout;
+	s_param.socket_id = SOCKET_ID_ANY;
+	s_param.graph_patterns = &pattern;
+	s_param.nb_graph_patterns = 1;
+
+	stats = rte_graph_cluster_stats_create(&s_param);
+	if (stats == NULL)
+		rte_exit(EXIT_FAILURE, "Unable to create stats object\n");
+
+	while (!force_quit) {
+		/* Clear screen and move to top left */
+		printf("%s%s", clr, topLeft);
+		rte_graph_cluster_stats_get(stats, 0);
+		rte_delay_ms(1E3);
+	}
+
+	rte_graph_cluster_stats_destroy(stats);
+}
+
+static void
+signal_handler(int signum)
+{
+	if (signum == SIGINT || signum == SIGTERM) {
+		printf("\n\nSignal %d received, preparing to exit...\n",
+				signum);
+		force_quit = true;
+	}
+}
+
+enum {
+	/* Long options mapped to a short option */
+
+	/* First long only option value must be >= 256, so that we won't
+	 * conflict with short options
+	 */
+	CMD_LINE_OPT_MIN_NUM = 256,
+	CMD_LINE_OPT_RXQ_NUM,
+};
+
+int
+main(int argc, char **argv)
+{
+	static const char *default_patterns[] = {
+		"l2fwd_pkt_rx-0-0",
+		"l2fwd_pkt_fwd",
+		"l2fwd_pkt_tx",
+		"l2fwd_pkt_drop"
+	};
+	struct rte_graph_param graph_conf;
+	union l2fwd_node_ctx node_ctx;
+	struct lcore_conf *lconf;
+	uint16_t nb_ports_available = 0;
+	uint16_t nb_ports;
+	uint16_t portid;
+	uint16_t fwd_portid = 0;
+	unsigned int nb_lcores = 0;
+	unsigned int lcore_id;
+	unsigned int nb_mbufs;
+	rte_graph_t main_graph_id = RTE_GRAPH_ID_INVALID;
+	rte_graph_t graph_id;
+	uint16_t n;
+	int ret;
+
+	/* init EAL */
+	ret = rte_eal_init(argc, argv);
+	if (ret < 0)
+		rte_exit(EXIT_FAILURE, "Invalid EAL arguments\n");
+	argc -= ret;
+	argv += ret;
+
+	force_quit = false;
+	signal(SIGINT, signal_handler);
+	signal(SIGTERM, signal_handler);
+
+	nb_ports = rte_eth_dev_count_avail();
+	if (nb_ports == 0)
+		rte_exit(EXIT_FAILURE, "No Ethernet ports - bye\n");
+
+	nb_lcores = 2;
+	nb_mbufs = RTE_MAX(nb_ports * ((nb_rxd * nb_rxq) + nb_txd + MAX_PKT_BURST +
+			   nb_lcores * MEMPOOL_CACHE_SIZE), 8192U);
+
+	/* create the mbuf pool */
+	l2fwd_pktmbuf_pool = rte_pktmbuf_pool_create("mbuf_pool", nb_mbufs,
+						     MEMPOOL_CACHE_SIZE, 0,
+						     RTE_MBUF_DEFAULT_BUF_SIZE,
+						     rte_socket_id());
+	if (l2fwd_pktmbuf_pool == NULL)
+		rte_exit(EXIT_FAILURE, "Cannot init mbuf pool\n");
+
+	/* Initialise each port */
+	RTE_ETH_FOREACH_DEV(portid) {
+		struct rte_eth_conf local_port_conf = default_port_conf;
+		struct rte_eth_dev_info dev_info;
+		struct rte_eth_rxconf rxq_conf;
+		struct rte_eth_txconf txq_conf;
+
+		nb_ports_available++;
+
+		/* init port */
+		printf("Initializing port %u... ", portid);
+		fflush(stdout);
+
+		ret = rte_eth_dev_info_get(portid, &dev_info);
+		if (ret != 0)
+			rte_exit(EXIT_FAILURE,
+				"Error during getting device (port %u) info: %s\n",
+				portid, strerror(-ret));
+
+		if (dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE)
+			local_port_conf.txmode.offloads |=
+				RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE;
+
+		local_port_conf.rx_adv_conf.rss_conf.rss_hf &=
+			dev_info.flow_type_rss_offloads;
+		if (local_port_conf.rx_adv_conf.rss_conf.rss_hf !=
+		    default_port_conf.rx_adv_conf.rss_conf.rss_hf) {
+			printf("Port %u modified RSS hash function based on "
+			       "hardware support,"
+			       "requested:%#" PRIx64 " configured:%#" PRIx64
+			       "\n",
+			       portid, default_port_conf.rx_adv_conf.rss_conf.rss_hf,
+			       local_port_conf.rx_adv_conf.rss_conf.rss_hf);
+		}
+
+		ret = rte_eth_dev_configure(portid, nb_rxq, 1, &local_port_conf);
+		if (ret < 0)
+			rte_exit(EXIT_FAILURE, "Cannot configure device: err=%d, port=%u\n",
+				  ret, portid);
+
+		ret = rte_eth_dev_adjust_nb_rx_tx_desc(portid, &nb_rxd, &nb_txd);
+		if (ret < 0)
+			rte_exit(EXIT_FAILURE,
+				 "Cannot adjust number of descriptors: err=%d, port=%u\n",
+				 ret, portid);
+
+		ret = rte_eth_macaddr_get(portid,
+					  &l2fwd_ports_eth_addr[portid]);
+		if (ret < 0)
+			rte_exit(EXIT_FAILURE,
+				 "Cannot get MAC address: err=%d, port=%u\n",
+				 ret, portid);
+
+		/* init RX queues */
+		fflush(stdout);
+		for (n = 0;  n < nb_rxq; n++) {
+			rxq_conf = dev_info.default_rxconf;
+			rxq_conf.offloads = local_port_conf.rxmode.offloads;
+			ret = rte_eth_rx_queue_setup(portid, n, nb_rxd,
+						     rte_eth_dev_socket_id(portid),
+						     &rxq_conf,
+						     l2fwd_pktmbuf_pool);
+			if (ret < 0)
+				rte_exit(EXIT_FAILURE,
+					 "rte_eth_rx_queue_setup:err=%d, port=%u, queue=%u\n",
+					 ret, portid, n);
+		}
+
+		/* init one TX queue on each port */
+		fflush(stdout);
+		txq_conf = dev_info.default_txconf;
+		txq_conf.offloads = local_port_conf.txmode.offloads;
+		ret = rte_eth_tx_queue_setup(portid, 0, nb_txd,
+					     rte_eth_dev_socket_id(portid),
+					     &txq_conf);
+		if (ret < 0)
+			rte_exit(EXIT_FAILURE, "rte_eth_tx_queue_setup:err=%d, port=%u\n",
+				 ret, portid);
+
+		/* Start device */
+		ret = rte_eth_dev_start(portid);
+		if (ret < 0)
+			rte_exit(EXIT_FAILURE, "rte_eth_dev_start:err=%d, port=%u\n",
+				 ret, portid);
+
+		ret = rte_eth_promiscuous_enable(portid);
+		if (ret < 0)
+			printf("Failed to set port %u promiscuous mode!\n", portid);
+
+		printf("done:\n");
+
+		printf("Port %u, MAC address: %02X:%02X:%02X:%02X:%02X:%02X\n\n",
+		       portid,
+		       l2fwd_ports_eth_addr[portid].addr_bytes[0],
+		       l2fwd_ports_eth_addr[portid].addr_bytes[1],
+		       l2fwd_ports_eth_addr[portid].addr_bytes[2],
+		       l2fwd_ports_eth_addr[portid].addr_bytes[3],
+		       l2fwd_ports_eth_addr[portid].addr_bytes[4],
+		       l2fwd_ports_eth_addr[portid].addr_bytes[5]);
+
+		fwd_portid = portid;
+	}
+
+	if (!nb_ports_available) {
+		rte_exit(EXIT_FAILURE,
+			"All available ports are disabled. Please set portmask.\n");
+	}
+
+	l2fwd_rx_node_init(0, nb_rxq);
+
+	memset(&node_ctx, 0, sizeof(node_ctx));
+	node_ctx.fwd_ctx.dest_port = fwd_portid;
+	l2fwd_node_ctx_add(l2fwd_get_pkt_fwd_node_id(), &node_ctx);
+
+	memset(&node_ctx, 0, sizeof(node_ctx));
+	node_ctx.rxtx_ctx.port = fwd_portid;
+	node_ctx.rxtx_ctx.queue = 0;
+	l2fwd_node_ctx_add(l2fwd_get_pkt_tx_node_id(), &node_ctx);
+
+	/* Need to set the node Lcore affinity before creating graph */
+	for (n = 0, lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
+		if (rte_lcore_is_enabled(lcore_id) == 0 ||
+		    rte_get_main_lcore() == lcore_id)
+			continue;
+
+		if (n < RTE_DIM(default_patterns)) {
+			ret = rte_node_set_lcore_affinity(default_patterns[n], lcore_id);
+			if (ret == 0)
+				printf("Set node %s affinity to Lcore %u\n",
+				       default_patterns[n], lcore_id);
+		}
+
+		n++;
+	}
+
+	for (lcore_id = 0; lcore_id < RTE_MAX_LCORE; lcore_id++) {
+		lconf = &l2fwd_lcore_conf[lcore_id];
+
+		if (rte_lcore_is_enabled(lcore_id) == 0 ||
+		    rte_get_main_lcore() == lcore_id) {
+			lconf->graph_id = RTE_GRAPH_ID_INVALID;
+			continue;
+		}
+
+		if (main_graph_id != RTE_GRAPH_ID_INVALID) {
+			struct rte_graph_clone_param clone_prm;
+
+			snprintf(lconf->name, sizeof(lconf->name), "cloned-%u", lcore_id);
+			memset(&clone_prm, 0, sizeof(clone_prm));
+			clone_prm.lcore_id = lcore_id;
+			graph_id = rte_graph_clone(main_graph_id, lconf->name, &clone_prm);
+			if (graph_id == RTE_GRAPH_ID_INVALID)
+				rte_exit(EXIT_FAILURE,
+					 "Failed to clone graph for lcore %u\n",
+					 lcore_id);
+
+			/* full cloned graph name */
+			snprintf(lconf->name, sizeof(lconf->name), "%s",
+				 rte_graph_id_to_name(graph_id));
+			lconf->graph_id = graph_id;
+			lconf->graph = rte_graph_lookup(lconf->name);
+			if (!lconf->graph)
+				rte_exit(EXIT_FAILURE,
+					 "Failed to lookup graph %s\n",
+					 lconf->name);
+			//rte_graph_obj_dump(stdout, lconf->graph, true);
+			continue;
+		}
+
+		memset(&graph_conf, 0, sizeof(graph_conf));
+		graph_conf.socket_id = rte_lcore_to_socket_id(lcore_id);
+		graph_conf.nb_node_patterns = RTE_DIM(default_patterns);
+		graph_conf.node_patterns = default_patterns;
+
+		snprintf(lconf->name, sizeof(lconf->name), L2FWD_GRAPH_NAME_PREFIX "%u",
+			 lcore_id);
+
+		graph_id = rte_graph_create(lconf->name, &graph_conf);
+		if (graph_id == RTE_GRAPH_ID_INVALID)
+			rte_exit(EXIT_FAILURE,
+				 "Failed to create graph for lcore %u\n",
+				 lcore_id);
+
+		lconf->graph_id = graph_id;
+		lconf->graph = rte_graph_lookup(lconf->name);
+		if (!lconf->graph)
+			rte_exit(EXIT_FAILURE,
+				 "Failed to lookup graph %s\n",
+				 lconf->name);
+		//rte_graph_obj_dump(stdout, lconf->graph, true);
+		main_graph_id = graph_id;
+	}
+
+	rte_eal_mp_remote_launch(l2fwd_graph_loop, NULL, SKIP_MAIN);
+
+	if (rte_graph_has_stats_feature())
+		print_stats();
+
+	ret = 0;
+
+	printf("Waiting all graphs to quit ...\n");
+	RTE_LCORE_FOREACH_WORKER(lcore_id) {
+		if (rte_eal_wait_lcore(lcore_id) < 0) {
+			ret = -1;
+			break;
+		}
+
+		lconf = &l2fwd_lcore_conf[lcore_id];
+		printf("Lcore %u with graph %s quit!\n", lcore_id,
+		       lconf->graph != NULL ? lconf->name : "Empty");
+	}
+
+	RTE_LCORE_FOREACH_WORKER(lcore_id) {
+		graph_id = l2fwd_lcore_conf[lcore_id].graph_id;
+		if (graph_id != RTE_GRAPH_ID_INVALID)
+			rte_graph_destroy(graph_id);
+	}
+
+	RTE_ETH_FOREACH_DEV(portid) {
+		printf("Closing port %d...", portid);
+		ret = rte_eth_dev_stop(portid);
+		if (ret != 0)
+			printf("rte_eth_dev_stop: err=%d, port=%d\n",
+			       ret, portid);
+		rte_eth_dev_close(portid);
+		printf(" Done\n");
+	}
+	printf("Bye...\n");
+
+	return ret;
+}
diff --git a/examples/l2fwd-graph/meson.build b/examples/l2fwd-graph/meson.build
new file mode 100644
index 0000000000..ce417c7ffe
--- /dev/null
+++ b/examples/l2fwd-graph/meson.build
@@ -0,0 +1,25 @@ 
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(C) 2020 Marvell International Ltd.
+
+# meson file, for building this example as part of a main DPDK build.
+#
+# To build this example as a standalone application with an already-installed
+# DPDK instance, use 'make'
+
+allow_experimental_apis = true
+
+deps += ['graph', 'eal', 'lpm', 'ethdev']
+
+if dpdk_conf.has('RTE_MEMPOOL_RING')
+	deps += 'mempool_ring'
+endif
+
+if dpdk_conf.has('RTE_NET_ICE')
+	deps += 'net_ice'
+endif
+
+sources = files(
+	'node.c',
+	'main.c'
+)
+
diff --git a/examples/l2fwd-graph/node.c b/examples/l2fwd-graph/node.c
new file mode 100644
index 0000000000..f0354536ab
--- /dev/null
+++ b/examples/l2fwd-graph/node.c
@@ -0,0 +1,263 @@ 
+#include <rte_debug.h>
+#include <rte_ethdev.h>
+#include <rte_ether.h>
+#include <rte_graph.h>
+#include <rte_graph_worker.h>
+#include <rte_mbuf.h>
+
+#include "node.h"
+
+static struct l2fwd_node_ctx_head l2fwd_node_ctx_list =
+			STAILQ_HEAD_INITIALIZER(l2fwd_node_ctx_list);
+static rte_spinlock_t l2fwd_node_ctx_lock = RTE_SPINLOCK_INITIALIZER;
+
+int
+l2fwd_node_ctx_add(rte_node_t node_id, union l2fwd_node_ctx *ctx)
+{
+	struct l2fwd_node_ctx_elem *ctx_elem, *temp;
+	int ret;
+
+	ctx_elem = calloc(1, sizeof(*temp));
+	if (ctx_elem == NULL) {
+		RTE_LOG(ERR, L2FWD_GRAPH,
+			"Failed to calloc node %u context object\n", node_id);
+		return -ENOMEM;
+	}
+
+	ctx_elem->id = node_id;
+	rte_memcpy(&ctx_elem->ctx, ctx, sizeof(*ctx));
+
+	rte_spinlock_lock(&l2fwd_node_ctx_lock);
+
+	STAILQ_FOREACH(temp, &l2fwd_node_ctx_list, next) {
+		if (temp->id == node_id) {
+			ret = -EEXIST;
+			RTE_LOG(ERR, L2FWD_GRAPH,
+				"The node %u context exists\n", node_id);
+			rte_spinlock_unlock(&l2fwd_node_ctx_lock);
+			goto fail;
+		}
+	}
+
+	STAILQ_INSERT_TAIL(&l2fwd_node_ctx_list, ctx_elem, next);
+
+	rte_spinlock_unlock(&l2fwd_node_ctx_lock);
+
+	return 0;
+
+fail:
+	free(temp);
+	return ret;
+}
+
+int
+l2fwd_node_ctx_del(rte_node_t node_id)
+{
+	struct l2fwd_node_ctx_elem *ctx_elem, *found = NULL;
+
+	rte_spinlock_lock(&l2fwd_node_ctx_lock);
+
+	STAILQ_FOREACH(ctx_elem, &l2fwd_node_ctx_list, next) {
+		if (ctx_elem->id == node_id) {
+			STAILQ_REMOVE(&l2fwd_node_ctx_list, ctx_elem, l2fwd_node_ctx_elem, next);
+			found = ctx_elem;
+			break;
+		}
+	}
+
+	rte_spinlock_unlock(&l2fwd_node_ctx_lock);
+
+	if (found) {
+		free(found);
+		return 0;
+	} else {
+		return -1;
+	}
+}
+
+static int
+l2fwd_node_init(const struct rte_graph *graph, struct rte_node *node)
+{
+	struct l2fwd_node_ctx_elem *ctx_elem;
+	int ret = -1;
+
+	RTE_SET_USED(graph);
+
+	rte_spinlock_lock(&l2fwd_node_ctx_lock);
+
+	STAILQ_FOREACH(ctx_elem, &l2fwd_node_ctx_list, next) {
+		if (ctx_elem->id == node->id) {
+			rte_memcpy(node->ctx, ctx_elem->ctx.ctx, RTE_NODE_CTX_SZ);
+			ret = 0;
+			break;
+		}
+	}
+
+	rte_spinlock_unlock(&l2fwd_node_ctx_lock);
+
+	return ret;
+}
+
+static uint16_t
+l2fwd_pkt_rx_node_process(struct rte_graph *graph, struct rte_node *node,
+			  void **objs, uint16_t cnt)
+{
+	struct l2fwd_node_rxtx_ctx *ctx = (struct l2fwd_node_rxtx_ctx *)node->ctx;
+	uint16_t n_pkts;
+
+	RTE_SET_USED(objs);
+	RTE_SET_USED(cnt);
+
+	n_pkts = rte_eth_rx_burst(ctx->port, ctx->queue, (struct rte_mbuf **)node->objs,
+				  RTE_GRAPH_BURST_SIZE);
+	if (!n_pkts)
+		return 0;
+
+	node->idx = n_pkts;
+
+	rte_node_next_stream_move(graph, node, L2FWD_RX_NEXT_PKT_FWD);
+
+	return n_pkts;
+}
+
+static struct rte_node_register l2fwd_pkt_rx_node_base = {
+	.name = "l2fwd_pkt_rx",
+	.flags = RTE_NODE_SOURCE_F,
+	.init = l2fwd_node_init,
+	.process = l2fwd_pkt_rx_node_process,
+
+	.nb_edges = L2FWD_RX_NEXT_MAX,
+	.next_nodes = {
+		[L2FWD_RX_NEXT_PKT_FWD] = "l2fwd_pkt_fwd",
+	},
+};
+
+int l2fwd_rx_node_init(uint16_t portid, uint16_t nb_rxq)
+{
+	char name[RTE_NODE_NAMESIZE];
+	union l2fwd_node_ctx ctx;
+	uint32_t id;
+	uint16_t n;
+
+	for (n = 0; n < nb_rxq; n++) {
+		snprintf(name, sizeof(name), "%u-%u", portid, n);
+
+		/* Clone a new rx node with same edges as parent */
+		id = rte_node_clone(l2fwd_pkt_rx_node_base.id, name);
+		if (id == RTE_NODE_ID_INVALID)
+			return -EIO;
+
+		memset(&ctx, 0, sizeof(ctx));
+		ctx.rxtx_ctx.port = portid;
+		ctx.rxtx_ctx.queue = n;
+		l2fwd_node_ctx_add(id, &ctx);
+	}
+
+	return 0;
+}
+
+static uint16_t
+l2fwd_pkt_fwd_node_process(struct rte_graph *graph, struct rte_node *node,
+			   void **objs, uint16_t cnt)
+{
+	struct l2fwd_node_fwd_ctx *ctx = (struct l2fwd_node_fwd_ctx *)node->ctx;
+	struct rte_mbuf *mbuf, **pkts;
+	struct rte_ether_addr *dest;
+	struct rte_ether_hdr *eth;
+	uint16_t dest_port;
+	uint16_t i;
+	void *tmp;
+
+	dest_port = ctx->dest_port;
+	dest = &l2fwd_ports_eth_addr[dest_port];
+	pkts = (struct rte_mbuf **)objs;
+
+	for (i = 0; i < cnt; i++) {
+		mbuf = pkts[i];
+
+		eth = rte_pktmbuf_mtod(mbuf, struct rte_ether_hdr *);
+
+		/* 02:00:00:00:00:xx */
+		tmp = &eth->dst_addr.addr_bytes[0];
+		*((uint64_t *)tmp) = 0x000000000002 + ((uint64_t)dest_port << 40);
+
+		/* src addr */
+		rte_ether_addr_copy(dest, &eth->src_addr);
+	}
+
+	rte_node_enqueue(graph, node, L2FWD_FWD_NEXT_PKT_TX, objs, cnt);
+
+	return cnt;
+}
+
+static struct rte_node_register l2fwd_pkt_fwd_node = {
+	.name = "l2fwd_pkt_fwd",
+	.init = l2fwd_node_init,
+	.process = l2fwd_pkt_fwd_node_process,
+
+	.nb_edges = L2FWD_FWD_NEXT_MAX,
+	.next_nodes = {
+		[L2FWD_FWD_NEXT_PKT_TX] = "l2fwd_pkt_tx",
+	},
+};
+
+rte_node_t
+l2fwd_get_pkt_fwd_node_id(void)
+{
+	return l2fwd_pkt_fwd_node.id;
+}
+
+static uint16_t
+l2fwd_pkt_tx_node_process(struct rte_graph *graph, struct rte_node *node,
+			  void **objs, uint16_t nb_objs)
+{
+	struct l2fwd_node_rxtx_ctx *ctx = (struct l2fwd_node_rxtx_ctx *)node->ctx;
+	uint16_t count;
+
+	count = rte_eth_tx_burst(ctx->port, ctx->queue, (struct rte_mbuf **)objs,
+				 nb_objs);
+	if (count != nb_objs)
+		rte_node_enqueue(graph, node, L2FWD_TX_NEXT_PKT_DROP,
+				 &objs[count], nb_objs - count);
+
+	return count;
+}
+
+static struct rte_node_register l2fwd_pkt_tx_node = {
+	.name = "l2fwd_pkt_tx",
+	.init = l2fwd_node_init,
+	.process = l2fwd_pkt_tx_node_process,
+
+	.nb_edges = L2FWD_TX_NEXT_MAX,
+	.next_nodes = {
+		[L2FWD_TX_NEXT_PKT_DROP] = "l2fwd_pkt_drop",
+	},
+};
+
+rte_node_t
+l2fwd_get_pkt_tx_node_id(void)
+{
+	return l2fwd_pkt_tx_node.id;
+}
+
+static uint16_t
+l2fwd_pkt_drop_node_process(struct rte_graph *graph, struct rte_node *node,
+			    void **objs, uint16_t nb_objs)
+{
+	RTE_SET_USED(node);
+	RTE_SET_USED(graph);
+
+	rte_pktmbuf_free_bulk((struct rte_mbuf **)objs, nb_objs);
+
+	return nb_objs;
+}
+
+static struct rte_node_register l2fwd_pkt_drop_node = {
+	.name = "l2fwd_pkt_drop",
+	.process = l2fwd_pkt_drop_node_process,
+};
+
+RTE_NODE_REGISTER(l2fwd_pkt_rx_node_base);
+RTE_NODE_REGISTER(l2fwd_pkt_fwd_node);
+RTE_NODE_REGISTER(l2fwd_pkt_tx_node);
+RTE_NODE_REGISTER(l2fwd_pkt_drop_node);
diff --git a/examples/l2fwd-graph/node.h b/examples/l2fwd-graph/node.h
new file mode 100644
index 0000000000..201136308c
--- /dev/null
+++ b/examples/l2fwd-graph/node.h
@@ -0,0 +1,64 @@ 
+#ifndef __NODE_H__
+#define __NODE_H__
+
+#include <inttypes.h>
+#include <sys/queue.h>
+
+#include <rte_common.h>
+#include <rte_eal.h>
+
+#include "rte_graph.h"
+#include "rte_graph_worker.h"
+
+#define RTE_LOGTYPE_L2FWD_GRAPH RTE_LOGTYPE_USER1
+
+enum l2fwd_rx_next_nodes {
+	L2FWD_RX_NEXT_PKT_FWD,
+	L2FWD_RX_NEXT_MAX,
+};
+
+enum l2fwd_fwd_next_nodes {
+	L2FWD_FWD_NEXT_PKT_TX,
+	L2FWD_FWD_NEXT_MAX,
+};
+
+enum l2fwd_tx_next_nodes {
+	L2FWD_TX_NEXT_PKT_DROP,
+	L2FWD_TX_NEXT_MAX,
+};
+
+struct l2fwd_node_rxtx_ctx {
+	uint16_t port;
+	uint16_t queue;
+};
+
+struct l2fwd_node_fwd_ctx {
+	uint16_t dest_port;
+};
+
+union l2fwd_node_ctx {
+	struct l2fwd_node_rxtx_ctx rxtx_ctx;
+	struct l2fwd_node_fwd_ctx fwd_ctx;
+	uint8_t ctx[RTE_NODE_CTX_SZ];
+};
+
+struct l2fwd_node_ctx_elem {
+	STAILQ_ENTRY(l2fwd_node_ctx_elem) next;
+
+	rte_node_t id;
+
+	union l2fwd_node_ctx ctx;
+};
+
+STAILQ_HEAD(l2fwd_node_ctx_head, l2fwd_node_ctx_elem);
+
+extern struct rte_ether_addr l2fwd_ports_eth_addr[];
+
+int l2fwd_node_ctx_add(rte_node_t node_id, union l2fwd_node_ctx *ctx);
+int l2fwd_node_ctx_del(rte_node_t node_id);
+
+int l2fwd_rx_node_init(uint16_t portid, uint16_t nb_rxq);
+rte_node_t l2fwd_get_pkt_fwd_node_id(void);
+rte_node_t l2fwd_get_pkt_tx_node_id(void);
+
+#endif
diff --git a/examples/meson.build b/examples/meson.build
index 81e93799f2..7c38e63ebf 100644
--- a/examples/meson.build
+++ b/examples/meson.build
@@ -29,6 +29,7 @@  all_examples = [
         'l2fwd-cat',
         'l2fwd-crypto',
         'l2fwd-event',
+        'l2fwd-graph',
         'l2fwd-jobstats',
         'l2fwd-keepalive',
         'l3fwd',