diff mbox series

[v7,06/11] app/mldev: add test case to interleave inferences

Message ID 20230316211434.13409-7-syalavarthi@marvell.com (mailing list archive)
State Accepted, archived
Delegated to: Thomas Monjalon
Headers
Series Implementation of mldev test application |

Checks

Context Check Description
ci/checkpatch warning coding style issues

Commit Message

Srikanth Yalavarthi March 16, 2023, 9:14 p.m. UTC 
  Added test case to interleave inference requests from multiple
models. Interleaving would load and start all models and launch
inference requests for the models using available queue-pairs

Operations sequence when testing with N models and R reps,

(load + start) x N -> (enqueue + dequeue) x N x R ...
	-> (stop + unload) x N

Test can be executed by selecting "inference_interleave" test.

Signed-off-by: Srikanth Yalavarthi <syalavarthi@marvell.com>
Acked-by: Anup Prabhu <aprabhu@marvell.com>
---
 app/test-mldev/meson.build                    |   1 +
 app/test-mldev/ml_options.c                   |   3 +-
 app/test-mldev/test_inference_interleave.c    | 114 +++
 .../tools/img/mldev_inference_interleave.svg  | 669 ++++++++++++++++++
 doc/guides/tools/testmldev.rst                |  41 ++
 5 files changed, 827 insertions(+), 1 deletion(-)
 create mode 100644 app/test-mldev/test_inference_interleave.c
 create mode 100644 doc/guides/tools/img/mldev_inference_interleave.svg
diff mbox series

Patch

diff --git a/app/test-mldev/meson.build b/app/test-mldev/meson.build
index 475d76d126..41d22fb22c 100644
--- a/app/test-mldev/meson.build
+++ b/app/test-mldev/meson.build
@@ -18,6 +18,7 @@  sources = files(
         'test_model_ops.c',
         'test_inference_common.c',
         'test_inference_ordered.c',
+        'test_inference_interleave.c',
 )
 
 deps += ['mldev']
diff --git a/app/test-mldev/ml_options.c b/app/test-mldev/ml_options.c
index 7b56bca90e..649cb9d8d1 100644
--- a/app/test-mldev/ml_options.c
+++ b/app/test-mldev/ml_options.c
@@ -156,7 +156,8 @@  ml_dump_test_options(const char *testname)
 		printf("\n");
 	}
 
-	if (strcmp(testname, "inference_ordered") == 0) {
+	if ((strcmp(testname, "inference_ordered") == 0) ||
+	    (strcmp(testname, "inference_interleave") == 0)) {
 		printf("\t\t--filelist         : comma separated list of model, input and output\n"
 		       "\t\t--repetitions      : number of inference repetitions\n");
 		printf("\n");
diff --git a/app/test-mldev/test_inference_interleave.c b/app/test-mldev/test_inference_interleave.c
new file mode 100644
index 0000000000..9cf4cfa197
--- /dev/null
+++ b/app/test-mldev/test_inference_interleave.c
@@ -0,0 +1,114 @@ 
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright (c) 2022 Marvell.
+ */
+
+#include <rte_common.h>
+#include <rte_launch.h>
+
+#include "ml_common.h"
+#include "test_inference_common.h"
+
+static int
+test_inference_interleave_driver(struct ml_test *test, struct ml_options *opt)
+{
+	struct test_inference *t;
+	uint16_t fid = 0;
+	int ret = 0;
+
+	t = ml_test_priv(test);
+
+	ret = ml_inference_mldev_setup(test, opt);
+	if (ret != 0)
+		return ret;
+
+	ret = ml_inference_mem_setup(test, opt);
+	if (ret != 0)
+		return ret;
+
+	/* load and start all models */
+	for (fid = 0; fid < opt->nb_filelist; fid++) {
+		ret = ml_model_load(test, opt, &t->model[fid], fid);
+		if (ret != 0)
+			goto error;
+
+		ret = ml_model_start(test, opt, &t->model[fid], fid);
+		if (ret != 0)
+			goto error;
+
+		ret = ml_inference_iomem_setup(test, opt, fid);
+		if (ret != 0)
+			goto error;
+	}
+
+	/* launch inference requests */
+	ret = ml_inference_launch_cores(test, opt, 0, opt->nb_filelist - 1);
+	if (ret != 0) {
+		ml_err("failed to launch cores");
+		goto error;
+	}
+
+	rte_eal_mp_wait_lcore();
+
+	/* stop and unload all models */
+	for (fid = 0; fid < opt->nb_filelist; fid++) {
+		ret = ml_inference_result(test, opt, fid);
+		if (ret != ML_TEST_SUCCESS)
+			goto error;
+
+		ml_inference_iomem_destroy(test, opt, fid);
+
+		ret = ml_model_stop(test, opt, &t->model[fid], fid);
+		if (ret != 0)
+			goto error;
+
+		ret = ml_model_unload(test, opt, &t->model[fid], fid);
+		if (ret != 0)
+			goto error;
+	}
+
+	ml_inference_mem_destroy(test, opt);
+
+	ret = ml_inference_mldev_destroy(test, opt);
+	if (ret != 0)
+		return ret;
+
+	t->cmn.result = ML_TEST_SUCCESS;
+
+	return 0;
+
+error:
+	ml_inference_mem_destroy(test, opt);
+	for (fid = 0; fid < opt->nb_filelist; fid++) {
+		ml_inference_iomem_destroy(test, opt, fid);
+		ml_model_stop(test, opt, &t->model[fid], fid);
+		ml_model_unload(test, opt, &t->model[fid], fid);
+	}
+
+	t->cmn.result = ML_TEST_FAILED;
+
+	return ret;
+}
+
+static int
+test_inference_interleave_result(struct ml_test *test, struct ml_options *opt)
+{
+	struct test_inference *t;
+
+	RTE_SET_USED(opt);
+
+	t = ml_test_priv(test);
+
+	return t->cmn.result;
+}
+
+static const struct ml_test_ops inference_interleave = {
+	.cap_check = test_inference_cap_check,
+	.opt_check = test_inference_opt_check,
+	.opt_dump = test_inference_opt_dump,
+	.test_setup = test_inference_setup,
+	.test_destroy = test_inference_destroy,
+	.test_driver = test_inference_interleave_driver,
+	.test_result = test_inference_interleave_result,
+};
+
+ML_TEST_REGISTER(inference_interleave);
diff --git a/doc/guides/tools/img/mldev_inference_interleave.svg b/doc/guides/tools/img/mldev_inference_interleave.svg
new file mode 100644
index 0000000000..3a741ea627
--- /dev/null
+++ b/doc/guides/tools/img/mldev_inference_interleave.svg
@@ -0,0 +1,669 @@ 
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- SPDX-License-Identifier: BSD-3-Clause -->
+<!-- Copyright (c) 2022 Marvell. -->
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+
+<svg
+   width="224mm"
+   height="150mm"
+   viewBox="0 0 224 150"
+   version="1.1"
+   id="svg5369"
+   inkscape:version="1.2.1 (9c6d41e410, 2022-07-14)"
+   sodipodi:docname="inference_interleave.svg"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:xlink="http://www.w3.org/1999/xlink"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:svg="http://www.w3.org/2000/svg">
+  <sodipodi:namedview
+     id="namedview5371"
+     pagecolor="#ffffff"
+     bordercolor="#000000"
+     borderopacity="0.25"
+     inkscape:showpageshadow="2"
+     inkscape:pageopacity="0.0"
+     inkscape:pagecheckerboard="0"
+     inkscape:deskcolor="#d1d1d1"
+     inkscape:document-units="mm"
+     showgrid="false"
+     inkscape:zoom="0.74564394"
+     inkscape:cx="415.07747"
+     inkscape:cy="348.6919"
+     inkscape:window-width="1920"
+     inkscape:window-height="1017"
+     inkscape:window-x="1912"
+     inkscape:window-y="-8"
+     inkscape:window-maximized="1"
+     inkscape:current-layer="layer1" />
+  <defs
+     id="defs5366">
+    <linearGradient
+       inkscape:collect="always"
+       id="linearGradient19189">
+      <stop
+         style="stop-color:#ffeeaa;stop-opacity:1;"
+         offset="0"
+         id="stop19185" />
+      <stop
+         style="stop-color:#ffeeaa;stop-opacity:0;"
+         offset="1"
+         id="stop19187" />
+    </linearGradient>
+    <marker
+       style="overflow:visible"
+       id="TriangleStart"
+       refX="4"
+       refY="0"
+       orient="auto-start-reverse"
+       inkscape:stockid="TriangleStart"
+       markerWidth="5.3244081"
+       markerHeight="6.155385"
+       viewBox="0 0 5.3244081 6.1553851"
+       inkscape:isstock="true"
+       inkscape:collect="always"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="scale(0.5)"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:context-stroke;stroke-width:1pt"
+         d="M 5.77,0 -2.88,5 V -5 Z"
+         id="path135" />
+    </marker>
+    <marker
+       style="overflow:visible"
+       id="RoundedArrow"
+       refX="5"
+       refY="0"
+       orient="auto-start-reverse"
+       inkscape:stockid="RoundedArrow"
+       markerWidth="6.1347523"
+       markerHeight="5.9304948"
+       viewBox="0 0 6.1347524 5.9304951"
+       inkscape:isstock="true"
+       inkscape:collect="always"
+       preserveAspectRatio="xMidYMid">
+      <path
+         transform="scale(0.7)"
+         d="m -0.21114562,-4.1055728 6.42229122,3.21114561 a 1,1 90 0 1 0,1.78885438 L -0.21114562,4.1055728 A 1.236068,1.236068 31.717474 0 1 -2,3 v -6 a 1.236068,1.236068 148.28253 0 1 1.78885438,-1.1055728 z"
+         style="fill:context-stroke;fill-rule:evenodd;stroke:none"
+         id="path1367" />
+    </marker>
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#linearGradient19189"
+       id="linearGradient19191"
+       x1="12.169352"
+       y1="105"
+       x2="284.83066"
+       y2="105"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="matrix(0.79055445,0,0,0.74078976,29.505892,28.991272)" />
+  </defs>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1">
+    <g
+       id="g1477"
+       transform="translate(-34.903236,-31.774189)">
+      <rect
+         style="fill:url(#linearGradient19191);fill-opacity:1;fill-rule:evenodd;stroke:#000000;stroke-width:0.396267;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers"
+         id="rect16635"
+         width="215.18147"
+         height="139.60078"
+         x="39.3125"
+         y="36.973797"
+         ry="2.2354064" />
+      <text
+         xml:space="preserve"
+         style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:Arial;-inkscape-font-specification:'Arial Bold';text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers"
+         x="-169.3954"
+         y="214.97237"
+         id="text5181"
+         transform="rotate(-90)"><tspan
+           sodipodi:role="line"
+           id="tspan5179"
+           style="font-size:5.64444px;stroke-width:0.75"
+           x="-169.3954"
+           y="214.97237">test: inference_interleave</tspan></text>
+      <path
+         style="display:inline;fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#ff8500;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)"
+         d="m 138.05133,87.886263 17.45982,-10e-7"
+         id="path1912"
+         inkscape:connector-type="polyline"
+         inkscape:connector-curvature="0"
+         inkscape:connection-start="#rect1724-0"
+         inkscape:connection-end="#rect1679" />
+      <path
+         style="display:inline;fill:none;fill-rule:evenodd;stroke:#00fb00;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)"
+         d="m 191.51115,87.886262 17.45981,10e-7"
+         id="path1914"
+         inkscape:connector-type="polyline"
+         inkscape:connector-curvature="0"
+         inkscape:connection-start="#rect1679"
+         inkscape:connection-end="#rect1724" />
+      <path
+         style="display:inline;fill:none;fill-rule:evenodd;stroke:#ff8500;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)"
+         d="m 138.05133,109.78102 17.45982,-1e-5"
+         id="path1916"
+         inkscape:connector-type="polyline"
+         inkscape:connector-curvature="0"
+         inkscape:connection-start="#rect1724-4-8"
+         inkscape:connection-end="#rect1679-4" />
+      <path
+         style="display:inline;fill:none;fill-rule:evenodd;stroke:#00fb00;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)"
+         d="m 191.51115,109.78101 17.45981,1e-5"
+         id="path1918"
+         inkscape:connector-type="polyline"
+         inkscape:connector-curvature="0"
+         inkscape:connection-start="#rect1679-4"
+         inkscape:connection-end="#rect1724-4" />
+      <path
+         style="display:inline;fill:none;fill-rule:evenodd;stroke:#ff8500;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)"
+         d="m 138.05133,131.67576 17.45982,-1e-5"
+         id="path1920"
+         inkscape:connector-type="polyline"
+         inkscape:connector-curvature="0"
+         inkscape:connection-start="#rect1724-6-7"
+         inkscape:connection-end="#rect1679-8" />
+      <path
+         style="display:inline;fill:none;fill-rule:evenodd;stroke:#00fb00;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)"
+         d="m 191.51115,131.67575 17.45981,1e-5"
+         id="path1922"
+         inkscape:connector-type="polyline"
+         inkscape:connector-curvature="0"
+         inkscape:connection-start="#rect1679-8"
+         inkscape:connection-end="#rect1724-6" />
+      <path
+         style="display:inline;fill:none;fill-rule:evenodd;stroke:#000000;stroke-width:0.75;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;marker-start:url(#RoundedArrow);marker-end:url(#RoundedArrow)"
+         d="m 173.51116,60.08164 0,12.907336"
+         id="path1933"
+         inkscape:connector-type="polyline"
+         inkscape:connector-curvature="0"
+         inkscape:connection-start="#rect1811"
+         inkscape:connection-end="#rect1924" />
+      <rect
+         style="fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.368668;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:0.737336, 0.737336;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers"
+         id="rect1924"
+         width="46.97015"
+         height="73.58287"
+         x="150.02565"
+         y="72.988976"
+         ry="2.4685853" />
+      <text
+         xml:space="preserve"
+         style="font-size:6.35px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers"
+         x="-84.44075"
+         y="173.5065"
+         id="text4097"
+         transform="rotate(-90)"><tspan
+           sodipodi:role="line"
+           id="tspan4095"
+           style="font-size:6.35px;stroke-width:0.75"
+           x="-84.44075"
+           y="173.5065">Queue</tspan><tspan
+           sodipodi:role="line"
+           style="font-size:6.35px;stroke-width:0.75"
+           x="-92.37825"
+           y="173.5065"
+           id="tspan4099">Pair 0</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-size:6.35px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers"
+         x="-128.2318"
+         y="173.5065"
+         id="text4097-5"
+         transform="rotate(-90)"><tspan
+           sodipodi:role="line"
+           id="tspan4095-6"
+           style="font-size:6.35px;stroke-width:0.75"
+           x="-128.2318"
+           y="173.5065">Queue</tspan><tspan
+           sodipodi:role="line"
+           style="font-size:6.35px;stroke-width:0.75"
+           x="-136.1693"
+           y="173.5065"
+           id="tspan4099-4">Pair 2</tspan></text>
+      <rect
+         style="fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.5;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers"
+         id="rect1679"
+         width="36"
+         height="18"
+         x="155.51115"
+         y="78.886261"
+         ry="3" />
+      <rect
+         style="fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.5;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers"
+         id="rect1679-8"
+         width="36"
+         height="18"
+         x="155.51115"
+         y="122.67575"
+         ry="3" />
+      <rect
+         style="fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.5;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers"
+         id="rect1679-4"
+         width="36"
+         height="18"
+         x="155.51115"
+         y="100.78101"
+         ry="3" />
+      <text
+         xml:space="preserve"
+         style="font-size:6.35px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers"
+         x="-106.33705"
+         y="173.5065"
+         id="text4097-8"
+         transform="rotate(-90)"><tspan
+           sodipodi:role="line"
+           id="tspan4095-4"
+           style="font-size:6.35px;stroke-width:0.75"
+           x="-106.33705"
+           y="173.5065">Queue</tspan><tspan
+           sodipodi:role="line"
+           style="font-size:6.35px;stroke-width:0.75"
+           x="-114.27455"
+           y="173.5065"
+           id="tspan4099-5">Pair 1</tspan></text>
+      <rect
+         style="fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.388863;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers"
+         id="rect1811"
+         width="58.909527"
+         height="18.812746"
+         x="144.0564"
+         y="41.268894"
+         ry="2.2255962" />
+      <text
+         xml:space="preserve"
+         style="font-size:5.64444px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers"
+         x="-47.017281"
+         y="173.49187"
+         id="text4156"
+         transform="rotate(-90)"><tspan
+           sodipodi:role="line"
+           id="tspan4154"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:Arial;-inkscape-font-specification:'Arial Bold';stroke-width:0.75"
+           x="-47.017281"
+           y="173.49187">Machine Learning</tspan><tspan
+           sodipodi:role="line"
+           style="font-style:normal;font-variant:normal;font-weight:bold;font-stretch:normal;font-size:5.64444px;font-family:Arial;-inkscape-font-specification:'Arial Bold';stroke-width:0.75"
+           x="-54.07283"
+           y="173.49187"
+           id="tspan4158">Hardware Engine</tspan></text>
+      <rect
+         style="fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.368668;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:0.737336, 0.737336;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers"
+         id="rect1924-0"
+         width="46.97015"
+         height="73.58287"
+         x="98.42067"
+         y="72.988976"
+         ry="2.4685853" />
+      <text
+         xml:space="preserve"
+         style="font-size:5.64444px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers"
+         x="-88.32518"
+         y="122.24379"
+         id="text3708"
+         transform="rotate(-90)"><tspan
+           sodipodi:role="line"
+           id="tspan3706"
+           style="font-size:5.64444px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.75"
+           x="-88.32518"
+           y="122.24379">lcore 1</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-size:5.64444px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers"
+         x="-132.10504"
+         y="121.83865"
+         id="text3708-8"
+         transform="rotate(-90)"><tspan
+           sodipodi:role="line"
+           id="tspan3706-7"
+           style="font-size:5.64444px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.75"
+           x="-132.10504"
+           y="121.83865">lcore 5</tspan></text>
+      <rect
+         style="fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#ff8500;stroke-width:0.5;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers"
+         id="rect1724-0"
+         width="32.290321"
+         height="11.709678"
+         x="105.76101"
+         y="82.031425"
+         ry="3.0161259" />
+      <rect
+         style="fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#ff8500;stroke-width:0.5;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers"
+         id="rect1724-6-7"
+         width="32.290321"
+         height="11.709678"
+         x="105.76101"
+         y="125.82092"
+         ry="3.0161259"
+         inkscape:connector-avoid="true" />
+      <rect
+         style="fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#ff8500;stroke-width:0.5;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers"
+         id="rect1724-4-8"
+         width="32.290321"
+         height="11.709678"
+         x="105.76101"
+         y="103.92618"
+         ry="3.0161259"
+         inkscape:connector-avoid="true" />
+      <text
+         xml:space="preserve"
+         style="font-size:5.64444px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers"
+         x="-110.21718"
+         y="121.85381"
+         id="text3708-5"
+         transform="rotate(-90)"><tspan
+           sodipodi:role="line"
+           id="tspan3706-87"
+           style="font-size:5.64444px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.75"
+           x="-110.21718"
+           y="121.85381">lcore 3</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-size:5.64444px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers"
+         x="-150.82878"
+         y="121.79179"
+         id="text4542"
+         transform="rotate(-90)"><tspan
+           sodipodi:role="line"
+           id="tspan4540"
+           style="font-size:5.64444px;stroke-width:0.75"
+           x="-150.82878"
+           y="121.79179">Enqueue Workers</tspan></text>
+      <rect
+         style="fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.368668;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:0.737336, 0.737336;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers"
+         id="rect1924-7"
+         width="46.97015"
+         height="73.58287"
+         x="201.63062"
+         y="72.988976"
+         ry="2.4685853" />
+      <text
+         xml:space="preserve"
+         style="font-size:5.64444px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers"
+         x="-88.32518"
+         y="225.08443"
+         id="text3708-9"
+         transform="rotate(-90)"><tspan
+           sodipodi:role="line"
+           id="tspan3706-9"
+           style="font-size:5.64444px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.75"
+           x="-88.32518"
+           y="225.08443">lcore 2</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-size:5.64444px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers"
+         x="-110.21167"
+         y="225.07202"
+         id="text3708-7"
+         transform="rotate(-90)"><tspan
+           sodipodi:role="line"
+           id="tspan3706-8"
+           style="font-size:5.64444px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.75"
+           x="-110.21167"
+           y="225.07202">lcore 4</tspan></text>
+      <rect
+         style="fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#00fb00;stroke-width:0.5;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers"
+         id="rect1724"
+         width="32.290321"
+         height="11.709678"
+         x="208.97096"
+         y="82.031425"
+         ry="3.0161259" />
+      <rect
+         style="fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#00fb00;stroke-width:0.5;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers"
+         id="rect1724-4"
+         width="32.290321"
+         height="11.709678"
+         x="208.97096"
+         y="103.92618"
+         ry="3.0161259" />
+      <rect
+         style="fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#00fb00;stroke-width:0.5;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers"
+         id="rect1724-6"
+         width="32.290321"
+         height="11.709678"
+         x="208.97096"
+         y="125.82092"
+         ry="3.0161259" />
+      <text
+         xml:space="preserve"
+         style="font-size:5.64444px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers"
+         x="-132.1133"
+         y="225.06514"
+         id="text3708-78"
+         transform="rotate(-90)"><tspan
+           sodipodi:role="line"
+           id="tspan3706-0"
+           style="font-size:5.64444px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.75"
+           x="-132.1133"
+           y="225.06514">lcore 6</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-size:5.64444px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#000000;fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers"
+         x="-150.82878"
+         y="225.00725"
+         id="text4542-3"
+         transform="rotate(-90)"><tspan
+           sodipodi:role="line"
+           id="tspan4540-7"
+           style="font-size:5.64444px;stroke-width:0.75"
+           x="-150.82878"
+           y="225.00725">Dequeue Workers</tspan></text>
+      <path
+         style="display:inline;fill:none;fill-rule:evenodd;stroke:#5d36ff;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)"
+         d="m 79.368074,78.962117 26.440446,5.546991"
+         id="path6677"
+         inkscape:connector-type="polyline"
+         inkscape:connector-curvature="0"
+         inkscape:connection-end="#rect1724-0"
+         inkscape:connection-start="#rect6252" />
+      <path
+         style="display:inline;fill:none;fill-rule:evenodd;stroke:#5d36ff;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)"
+         d="M 78.978262,84.050781 112.13805,103.92618"
+         id="path6679"
+         inkscape:connector-type="polyline"
+         inkscape:connector-curvature="0"
+         inkscape:connection-end="#rect1724-4-8"
+         inkscape:connection-start="#rect6252" />
+      <path
+         style="display:inline;fill:none;fill-rule:evenodd;stroke:#5d36ff;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)"
+         d="m 73.959073,84.25738 42.026977,41.56354"
+         id="path6681"
+         inkscape:connector-type="polyline"
+         inkscape:connector-curvature="0"
+         inkscape:connection-end="#rect1724-6-7"
+         inkscape:connection-start="#rect6252" />
+      <path
+         style="display:inline;fill:none;fill-rule:evenodd;stroke:#5d36ff;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)"
+         d="M 79.368074,95.959838 105.76913,90.949016"
+         id="path6683"
+         inkscape:connector-type="polyline"
+         inkscape:connector-curvature="0"
+         inkscape:connection-end="#rect1724-0"
+         inkscape:connection-start="#rect6252-8" />
+      <path
+         style="display:inline;fill:none;fill-rule:evenodd;stroke:#5d36ff;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)"
+         d="m 79.368074,101.28215 26.416716,5.27791"
+         id="path7830"
+         inkscape:connector-type="polyline"
+         inkscape:connector-curvature="0"
+         inkscape:connection-end="#rect1724-4-8"
+         inkscape:connection-start="#rect6252-8" />
+      <path
+         style="display:inline;fill:none;fill-rule:evenodd;stroke:#5d36ff;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)"
+         d="m 79.069199,106.4283 32.903161,19.39262"
+         id="path7862"
+         inkscape:connector-type="polyline"
+         inkscape:connector-curvature="0"
+         inkscape:connection-end="#rect1724-6-7"
+         inkscape:connection-start="#rect6252-8" />
+      <path
+         style="display:inline;fill:none;fill-rule:evenodd;stroke:#5d36ff;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)"
+         d="M 79.069201,113.13371 111.97235,93.741103"
+         id="path7900"
+         inkscape:connector-type="polyline"
+         inkscape:connector-curvature="0"
+         inkscape:connection-end="#rect1724-0"
+         inkscape:connection-start="#rect6252-2" />
+      <path
+         style="display:inline;fill:none;fill-rule:evenodd;stroke:#5d36ff;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)"
+         d="m 79.368074,118.27987 26.416716,-5.2779"
+         id="path7932"
+         inkscape:connector-type="polyline"
+         inkscape:connector-curvature="0"
+         inkscape:connection-end="#rect1724-4-8"
+         inkscape:connection-start="#rect6252-2" />
+      <path
+         style="display:inline;fill:none;fill-rule:evenodd;stroke:#5d36ff;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)"
+         d="m 79.368074,123.60218 26.401056,5.01083"
+         id="path7998"
+         inkscape:connector-type="polyline"
+         inkscape:connector-curvature="0"
+         inkscape:connection-end="#rect1724-6-7"
+         inkscape:connection-start="#rect6252-2" />
+      <path
+         style="display:inline;fill:none;fill-rule:evenodd;stroke:#5d36ff;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)"
+         d="M 73.959074,135.30464 115.98605,93.741103"
+         id="path8000"
+         inkscape:connector-type="polyline"
+         inkscape:connector-curvature="0"
+         inkscape:connection-end="#rect1724-0"
+         inkscape:connection-start="#rect6252-6" />
+      <path
+         style="display:inline;fill:none;fill-rule:evenodd;stroke:#5d36ff;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)"
+         d="M 78.978264,135.51124 112.13805,115.63586"
+         id="path8002"
+         inkscape:connector-type="polyline"
+         inkscape:connector-curvature="0"
+         inkscape:connection-end="#rect1724-4-8"
+         inkscape:connection-start="#rect6252-6" />
+      <path
+         style="display:inline;fill:none;fill-rule:evenodd;stroke:#5d36ff;stroke-width:0.5;stroke-linecap:butt;stroke-linejoin:miter;stroke-dasharray:none;stroke-opacity:1;marker-end:url(#TriangleStart)"
+         d="m 79.368074,140.59991 26.440446,-5.54699"
+         id="path8004"
+         inkscape:connector-type="polyline"
+         inkscape:connector-curvature="0"
+         inkscape:connection-end="#rect1724-6-7"
+         inkscape:connection-start="#rect6252-6" />
+      <text
+         xml:space="preserve"
+         style="font-size:5.64444px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers"
+         x="-76.53363"
+         y="65.63237"
+         id="text3708-84"
+         transform="rotate(-90)"><tspan
+           sodipodi:role="line"
+           id="tspan3706-4"
+           style="font-size:5.64444px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.75"
+           x="-76.53363"
+           y="65.63237">Model 0</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-size:5.64444px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers"
+         x="-98.991623"
+         y="66.015465"
+         id="text3708-84-1"
+         transform="rotate(-90)"><tspan
+           sodipodi:role="line"
+           id="tspan3706-4-6"
+           style="font-size:5.64444px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.75"
+           x="-98.991623"
+           y="66.015465">Model 1</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-size:5.64444px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers"
+         x="-121.44823"
+         y="65.646149"
+         id="text3708-84-9"
+         transform="rotate(-90)"><tspan
+           sodipodi:role="line"
+           id="tspan3706-4-1"
+           style="font-size:5.64444px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.75"
+           x="-121.44823"
+           y="65.646149">Model 2</tspan></text>
+      <text
+         xml:space="preserve"
+         style="font-size:5.64444px;font-family:Arial;-inkscape-font-specification:Arial;text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:#0000ff;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:2.25, 0.750001;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers"
+         x="-143.9021"
+         y="65.625481"
+         id="text3708-84-5"
+         transform="rotate(-90)"><tspan
+           sodipodi:role="line"
+           id="tspan3706-4-8"
+           style="font-size:5.64444px;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.75"
+           x="-143.9021"
+           y="65.625481">Model 3</tspan></text>
+      <rect
+         style="fill:none;fill-rule:evenodd;stroke:#00d7fb;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-opacity:1;paint-order:stroke fill markers"
+         id="rect6252"
+         width="27.32258"
+         height="16.32258"
+         x="52.045494"
+         y="67.934799"
+         ry="1" />
+      <rect
+         style="fill:none;fill-rule:evenodd;stroke:#00d7fb;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-opacity:1;paint-order:stroke fill markers"
+         id="rect6252-8"
+         width="27.32258"
+         height="16.32258"
+         x="52.045494"
+         y="90.391411"
+         ry="1" />
+      <rect
+         style="fill:none;fill-rule:evenodd;stroke:#00d7fb;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-opacity:1;paint-order:stroke fill markers"
+         id="rect6252-2"
+         width="27.32258"
+         height="16.32258"
+         x="52.045494"
+         y="112.84802"
+         ry="1" />
+      <rect
+         style="fill:none;fill-rule:evenodd;stroke:#00d7fb;stroke-width:0.750001;stroke-linecap:round;stroke-linejoin:round;stroke-opacity:1;paint-order:stroke fill markers"
+         id="rect6252-6"
+         width="27.32258"
+         height="16.32258"
+         x="52.045494"
+         y="135.30464"
+         ry="1" />
+      <rect
+         style="fill:none;fill-opacity:1;fill-rule:evenodd;stroke:#00a6fb;stroke-width:0.399886;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:0.799773, 0.799773;stroke-dashoffset:0;stroke-opacity:1;paint-order:stroke fill markers"
+         id="rect1924-1-3"
+         width="43.035683"
+         height="94.487152"
+         x="44.188942"
+         y="62.536991"
+         ry="3.1694498" />
+      <g
+         id="g1010"
+         transform="translate(21.464467,-15.875004)">
+        <text
+           xml:space="preserve"
+           style="font-weight:bold;font-size:4.23333px;font-family:Arial;-inkscape-font-specification:'Arial Bold';text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:none;fill-rule:evenodd;stroke:#00d7fb;stroke-width:0.499999;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers"
+           x="-185.88483"
+           y="86.542366"
+           id="text15571"
+           transform="rotate(-90)"><tspan
+             sodipodi:role="line"
+             id="tspan15569"
+             style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.23333px;font-family:Arial;-inkscape-font-specification:Arial;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.5;stroke-opacity:1"
+             x="-185.88483"
+             y="86.542366">nb_worker_threads =  2 * MIN(nb_queue_pairs, (lcore_count - 1) / 2)</tspan></text>
+        <text
+           xml:space="preserve"
+           style="font-weight:bold;font-size:4.23333px;font-family:Arial;-inkscape-font-specification:'Arial Bold';text-align:center;writing-mode:tb-rl;text-anchor:middle;fill:none;fill-rule:evenodd;stroke:#00d7fb;stroke-width:0.499999;stroke-linecap:round;stroke-linejoin:round;stroke-dasharray:none;stroke-opacity:1;paint-order:stroke fill markers"
+           x="-178.43243"
+           y="90.903854"
+           id="text15571-3"
+           transform="rotate(-90)"><tspan
+             sodipodi:role="line"
+             id="tspan15569-9"
+             style="font-style:normal;font-variant:normal;font-weight:normal;font-stretch:normal;font-size:4.23333px;font-family:Arial;-inkscape-font-specification:Arial;fill:#000000;fill-opacity:1;stroke:none;stroke-width:0.5;stroke-opacity:1"
+             x="-178.43243"
+             y="90.903854">inferences_per_queue_pair = nb_models * (repetitions / nb_queue_pairs)</tspan></text>
+      </g>
+    </g>
+  </g>
+</svg>
diff --git a/doc/guides/tools/testmldev.rst b/doc/guides/tools/testmldev.rst
index 164fbca64f..1a1ab7d2bf 100644
--- a/doc/guides/tools/testmldev.rst
+++ b/doc/guides/tools/testmldev.rst
@@ -58,6 +58,7 @@  The following are the command-line options supported by the test application.
       **ML Inference Tests** ::
 
          inference_ordered
+         inference_interleave
 
 * ``--dev_id <n>``
 
@@ -280,6 +281,46 @@  Example command to run inference_ordered test:
         --test=inference_ordered --filelist model.bin,input.bin,output.bin
 
 
+INFERENCE_INTERLEAVE Test
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This is a stress test for validating the end-to-end inference execution on ML device. The test
+configures the ML device and queue pairs as per the queue-pair related options (queue_pairs
+and queue_size) specified by the user. Upon successful configuration of the device and queue
+pairs, all models specified through the filelist are loaded to the device. Inferences for multiple
+models are enqueued by a pool of worker threads in parallel. Inference execution by the device is
+interleaved between multiple models. Total number of inferences enqueued for a model are equal to
+the repetitions specified. An additional pool of threads would dequeue the inferences from the
+device. Models would be unloaded upon completion of inferences for all models loaded.
+
+
+.. _figure_mldev_inference_interleave:
+
+.. figure:: img/mldev_inference_interleave.*
+
+   Execution of inference_interleave on single model.
+
+
+Example
+^^^^^^^
+
+Example command to run inference_interleave test:
+
+.. code-block:: console
+
+    sudo <build_dir>/app/dpdk-test-mldev -c 0xf -a <PCI_ID> -- \
+        --test=inference_interleave --filelist model.bin,input.bin,output.bin
+
+
+Example command to run inference_interleave test with multiple models:
+
+.. code-block:: console
+
+    sudo <build_dir>/app/dpdk-test-mldev -c 0xf -a <PCI_ID> -- \
+        --test=inference_interleave --filelist model_A.bin,input_A.bin,output_A.bin \
+        --filelist model_B.bin,input_B.bin,output_B.bin
+
+
 Debug mode
 ----------