Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/129645/?format=api
{ "id": 129645, "url": "http://patchwork.dpdk.org/api/patches/129645/?format=api", "web_url": "http://patchwork.dpdk.org/project/dpdk/patch/20230719141303.33284-6-juraj.linkes@pantheon.tech/", "project": { "id": 1, "url": "http://patchwork.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20230719141303.33284-6-juraj.linkes@pantheon.tech>", "list_archive_url": "https://inbox.dpdk.org/dev/20230719141303.33284-6-juraj.linkes@pantheon.tech", "date": "2023-07-19T14:13:02", "name": "[v3,5/6] dts: scapy traffic generator implementation", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": true, "hash": "1052f30f93b2c4b422560c6ac6e158bc1aa8a926", "submitter": { "id": 1626, "url": "http://patchwork.dpdk.org/api/people/1626/?format=api", "name": "Juraj Linkeš", "email": "juraj.linkes@pantheon.tech" }, "delegate": { "id": 1, "url": "http://patchwork.dpdk.org/api/users/1/?format=api", "username": "tmonjalo", "first_name": "Thomas", "last_name": "Monjalon", "email": "thomas@monjalon.net" }, "mbox": "http://patchwork.dpdk.org/project/dpdk/patch/20230719141303.33284-6-juraj.linkes@pantheon.tech/mbox/", "series": [ { "id": 28973, "url": "http://patchwork.dpdk.org/api/series/28973/?format=api", "web_url": "http://patchwork.dpdk.org/project/dpdk/list/?series=28973", "date": "2023-07-19T14:12:57", "name": "dts: tg abstractions and scapy tg", "version": 3, "mbox": "http://patchwork.dpdk.org/series/28973/mbox/" } ], "comments": "http://patchwork.dpdk.org/api/patches/129645/comments/", "check": "success", "checks": "http://patchwork.dpdk.org/api/patches/129645/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 7704642EB9;\n\tWed, 19 Jul 2023 16:13:41 +0200 (CEST)", "from mails.dpdk.org (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 3F5AE42D2C;\n\tWed, 19 Jul 2023 16:13:13 +0200 (CEST)", "from mail-ed1-f51.google.com (mail-ed1-f51.google.com\n [209.85.208.51]) by mails.dpdk.org (Postfix) with ESMTP id 0CC1442C4D\n for <dev@dpdk.org>; Wed, 19 Jul 2023 16:13:11 +0200 (CEST)", "by mail-ed1-f51.google.com with SMTP id\n 4fb4d7f45d1cf-51e28b299adso9714318a12.2\n for <dev@dpdk.org>; Wed, 19 Jul 2023 07:13:11 -0700 (PDT)", "from jlinkes-PT-Latitude-5530.. (ip-46.34.247.144.o2inet.sk.\n [46.34.247.144]) by smtp.gmail.com with ESMTPSA id\n q8-20020a056402040800b0051e2809395bsm2721979edv.63.2023.07.19.07.13.09\n (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256);\n Wed, 19 Jul 2023 07:13:10 -0700 (PDT)" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n d=pantheon.tech; s=google; t=1689775990; x=1690380790;\n h=content-transfer-encoding:mime-version:references:in-reply-to\n :message-id:date:subject:cc:to:from:from:to:cc:subject:date\n :message-id:reply-to;\n bh=/ocCeGggALcPese7oorKm8tk1DZe7EJvyi6nf35oZQQ=;\n b=E8g/QpYnGpIqlp6yOCTeBbpTeXwfGPT1aA78c8402eTAcyXflW1XV3AqvcGncsePtP\n FFTRci+nab5QbP2L0enek320PAWefhh1IC36BSHu7fAt4pPnUYl10o4/zzvb/LUXgPAY\n mi2M/eCA7r2gnY5WAS8RuMv//eKguz8DdF8InoIc39Uy6FlOuLO/RVK0UTHa5jlsNUXs\n TJczqy7jqsoAy4cvn4T5cHIAHZRop+fMGWAA0RsdQ2OUsynUQtM7UAqBEJJ7HLyM1Jd4\n joZ9H61KTcrUUbg1fBd8bRd1vY8QD+yclMYE93aiVFPjCbCY5jpgYc0uf2/pyrZfroO7\n 2ppQ==", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n d=1e100.net; s=20221208; t=1689775990; x=1690380790;\n h=content-transfer-encoding:mime-version:references:in-reply-to\n :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc\n :subject:date:message-id:reply-to;\n bh=/ocCeGggALcPese7oorKm8tk1DZe7EJvyi6nf35oZQQ=;\n b=Ze8/zdMbI3zVZWoSkQH07blu5G4L0dMJcYujmjxMFFDIvo+cW9Ukm2L3+UASv+X2q0\n cbhrgBPpaAlANx8eI1i19q3KXqYaSP8/iUvjgLGQoAW/3xMl7PN9v0piBSQJqlUOfZpl\n mnZo6SxykFGZOm88Bs2XVTg0P9NAGgJe3WST4/0qR/dSqqEQ7GkHWhRQ0r5FtYTom97d\n acns5MqVVeczAatmSkM3ijdbkLNgBHsWmJ2jCamO9PWCOHMnTy+Rc9UAclr8y9/FiCUY\n mDY6Hi1z+YxKaVjQ/CSdZjEhXPceFiQsSxMe8urduKqn1i955WvidmoXssxtWfVuYL/q\n dSyg==", "X-Gm-Message-State": "ABy/qLZAxXbDYgmqtQhQ6LJIsV6ZhQaCZ2jQES4Rlmci+YoSpc/KUxo5\n 3hUNVIYRj/fG5plTzc0LKeQnMw==", "X-Google-Smtp-Source": "\n APBJJlG4/n5DP77Ru7HtmlhoUdu5RIfIiYMylT7/n5nAe5WzNiiMnFPZOXr3PyCBaSa1KK/705Vtbg==", "X-Received": "by 2002:aa7:d7d1:0:b0:51e:ebd:9f5b with SMTP id\n e17-20020aa7d7d1000000b0051e0ebd9f5bmr2437193eds.36.1689775990733;\n Wed, 19 Jul 2023 07:13:10 -0700 (PDT)", "From": "=?utf-8?q?Juraj_Linke=C5=A1?= <juraj.linkes@pantheon.tech>", "To": "thomas@monjalon.net, Honnappa.Nagarahalli@arm.com, lijuan.tu@intel.com,\n bruce.richardson@intel.com, jspewock@iol.unh.edu, probb@iol.unh.edu", "Cc": "dev@dpdk.org, =?utf-8?q?Juraj_Linke=C5=A1?= <juraj.linkes@pantheon.tech>", "Subject": "[PATCH v3 5/6] dts: scapy traffic generator implementation", "Date": "Wed, 19 Jul 2023 16:13:02 +0200", "Message-Id": "<20230719141303.33284-6-juraj.linkes@pantheon.tech>", "X-Mailer": "git-send-email 2.34.1", "In-Reply-To": "<20230719141303.33284-1-juraj.linkes@pantheon.tech>", "References": "<20230717110709.39220-1-juraj.linkes@pantheon.tech>\n <20230719141303.33284-1-juraj.linkes@pantheon.tech>", "MIME-Version": "1.0", "Content-Type": "text/plain; charset=UTF-8", "Content-Transfer-Encoding": "8bit", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org" }, "content": "Scapy is a traffic generator capable of sending and receiving traffic.\nSince it's a software traffic generator, it's not suitable for\nperformance testing, but it is suitable for functional testing.\n\nSigned-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>\n---\n dts/framework/remote_session/__init__.py | 1 +\n .../remote_session/remote/__init__.py | 1 +\n dts/framework/testbed_model/scapy.py | 224 +++++++++++++++++-\n 3 files changed, 222 insertions(+), 4 deletions(-)", "diff": "diff --git a/dts/framework/remote_session/__init__.py b/dts/framework/remote_session/__init__.py\nindex 1155dd8318..00b6d1f03a 100644\n--- a/dts/framework/remote_session/__init__.py\n+++ b/dts/framework/remote_session/__init__.py\n@@ -22,6 +22,7 @@\n CommandResult,\n InteractiveRemoteSession,\n InteractiveShell,\n+ PythonShell,\n RemoteSession,\n SSHSession,\n TestPmdDevice,\ndiff --git a/dts/framework/remote_session/remote/__init__.py b/dts/framework/remote_session/remote/__init__.py\nindex 1d29c3ea0d..06403691a5 100644\n--- a/dts/framework/remote_session/remote/__init__.py\n+++ b/dts/framework/remote_session/remote/__init__.py\n@@ -9,6 +9,7 @@\n \n from .interactive_remote_session import InteractiveRemoteSession\n from .interactive_shell import InteractiveShell\n+from .python_shell import PythonShell\n from .remote_session import CommandResult, RemoteSession\n from .ssh_session import SSHSession\n from .testpmd_shell import TestPmdDevice, TestPmdShell\ndiff --git a/dts/framework/testbed_model/scapy.py b/dts/framework/testbed_model/scapy.py\nindex 1a23dc9fa3..af0d4dbb25 100644\n--- a/dts/framework/testbed_model/scapy.py\n+++ b/dts/framework/testbed_model/scapy.py\n@@ -12,10 +12,21 @@\n a local server proxy.\n \"\"\"\n \n+import inspect\n+import marshal\n+import time\n+import types\n+import xmlrpc.client\n+from xmlrpc.server import SimpleXMLRPCServer\n+\n+import scapy.all # type: ignore[import]\n+from scapy.layers.l2 import Ether # type: ignore[import]\n from scapy.packet import Packet # type: ignore[import]\n \n from framework.config import OS, ScapyTrafficGeneratorConfig\n-from framework.logger import getLogger\n+from framework.logger import DTSLOG, getLogger\n+from framework.remote_session import PythonShell\n+from framework.settings import SETTINGS\n \n from .capturing_traffic_generator import (\n CapturingTrafficGenerator,\n@@ -24,6 +35,134 @@\n from .hw.port import Port\n from .tg_node import TGNode\n \n+\"\"\"\n+========= BEGIN RPC FUNCTIONS =========\n+\n+All of the functions in this section are intended to be exported to a python\n+shell which runs a scapy RPC server. These functions are made available via that\n+RPC server to the packet generator. To add a new function to the RPC server,\n+first write the function in this section. Then, if you need any imports, make sure to\n+add them to SCAPY_RPC_SERVER_IMPORTS as well. After that, add the function to the list\n+in EXPORTED_FUNCTIONS. Note that kwargs (keyword arguments) do not work via xmlrpc,\n+so you may need to construct wrapper functions around many scapy types.\n+\"\"\"\n+\n+\"\"\"\n+Add the line needed to import something in a normal python environment\n+as an entry to this array. It will be imported before any functions are\n+sent to the server.\n+\"\"\"\n+SCAPY_RPC_SERVER_IMPORTS = [\n+ \"from scapy.all import *\",\n+ \"import xmlrpc\",\n+ \"import sys\",\n+ \"from xmlrpc.server import SimpleXMLRPCServer\",\n+ \"import marshal\",\n+ \"import pickle\",\n+ \"import types\",\n+ \"import time\",\n+]\n+\n+\n+def scapy_send_packets_and_capture(\n+ xmlrpc_packets: list[xmlrpc.client.Binary],\n+ send_iface: str,\n+ recv_iface: str,\n+ duration: float,\n+) -> list[bytes]:\n+ \"\"\"RPC function to send and capture packets.\n+\n+ The function is meant to be executed on the remote TG node.\n+\n+ Args:\n+ xmlrpc_packets: The packets to send. These need to be converted to\n+ xmlrpc.client.Binary before sending to the remote server.\n+ send_iface: The logical name of the egress interface.\n+ recv_iface: The logical name of the ingress interface.\n+ duration: Capture for this amount of time, in seconds.\n+\n+ Returns:\n+ A list of bytes. Each item in the list represents one packet, which needs\n+ to be converted back upon transfer from the remote node.\n+ \"\"\"\n+ scapy_packets = [scapy.all.Packet(packet.data) for packet in xmlrpc_packets]\n+ sniffer = scapy.all.AsyncSniffer(\n+ iface=recv_iface,\n+ store=True,\n+ started_callback=lambda *args: scapy.all.sendp(scapy_packets, iface=send_iface),\n+ )\n+ sniffer.start()\n+ time.sleep(duration)\n+ return [scapy_packet.build() for scapy_packet in sniffer.stop(join=True)]\n+\n+\n+def scapy_send_packets(\n+ xmlrpc_packets: list[xmlrpc.client.Binary], send_iface: str\n+) -> None:\n+ \"\"\"RPC function to send packets.\n+\n+ The function is meant to be executed on the remote TG node.\n+ It doesn't return anything, only sends packets.\n+\n+ Args:\n+ xmlrpc_packets: The packets to send. These need to be converted to\n+ xmlrpc.client.Binary before sending to the remote server.\n+ send_iface: The logical name of the egress interface.\n+\n+ Returns:\n+ A list of bytes. Each item in the list represents one packet, which needs\n+ to be converted back upon transfer from the remote node.\n+ \"\"\"\n+ scapy_packets = [scapy.all.Packet(packet.data) for packet in xmlrpc_packets]\n+ scapy.all.sendp(scapy_packets, iface=send_iface, realtime=True, verbose=True)\n+\n+\n+\"\"\"\n+Functions to be exposed by the scapy RPC server.\n+\"\"\"\n+RPC_FUNCTIONS = [\n+ scapy_send_packets,\n+ scapy_send_packets_and_capture,\n+]\n+\n+\"\"\"\n+========= END RPC FUNCTIONS =========\n+\"\"\"\n+\n+\n+class QuittableXMLRPCServer(SimpleXMLRPCServer):\n+ \"\"\"Basic XML-RPC server that may be extended\n+ by functions serializable by the marshal module.\n+ \"\"\"\n+\n+ def __init__(self, *args, **kwargs):\n+ kwargs[\"allow_none\"] = True\n+ super().__init__(*args, **kwargs)\n+ self.register_introspection_functions()\n+ self.register_function(self.quit)\n+ self.register_function(self.add_rpc_function)\n+\n+ def quit(self) -> None:\n+ self._BaseServer__shutdown_request = True\n+ return None\n+\n+ def add_rpc_function(self, name: str, function_bytes: xmlrpc.client.Binary):\n+ \"\"\"Add a function to the server.\n+\n+ This is meant to be executed remotely.\n+\n+ Args:\n+ name: The name of the function.\n+ function_bytes: The code of the function.\n+ \"\"\"\n+ function_code = marshal.loads(function_bytes.data)\n+ function = types.FunctionType(function_code, globals(), name)\n+ self.register_function(function)\n+\n+ def serve_forever(self, poll_interval: float = 0.5) -> None:\n+ print(\"XMLRPC OK\")\n+ super().serve_forever(poll_interval)\n+\n \n class ScapyTrafficGenerator(CapturingTrafficGenerator):\n \"\"\"Provides access to scapy functions via an RPC interface.\n@@ -41,10 +180,19 @@ class ScapyTrafficGenerator(CapturingTrafficGenerator):\n Arguments:\n tg_node: The node where the traffic generator resides.\n config: The user configuration of the traffic generator.\n+\n+ Attributes:\n+ session: The exclusive interactive remote session created by the Scapy\n+ traffic generator where the XML-RPC server runs.\n+ rpc_server_proxy: The object used by clients to execute functions\n+ on the XML-RPC server.\n \"\"\"\n \n+ session: PythonShell\n+ rpc_server_proxy: xmlrpc.client.ServerProxy\n _config: ScapyTrafficGeneratorConfig\n _tg_node: TGNode\n+ _logger: DTSLOG\n \n def __init__(self, tg_node: TGNode, config: ScapyTrafficGeneratorConfig):\n self._config = config\n@@ -57,8 +205,58 @@ def __init__(self, tg_node: TGNode, config: ScapyTrafficGeneratorConfig):\n self._tg_node.config.os == OS.linux\n ), \"Linux is the only supported OS for scapy traffic generation\"\n \n+ self.session = self._tg_node.create_interactive_shell(\n+ PythonShell, timeout=5, privileged=True\n+ )\n+\n+ # import libs in remote python console\n+ for import_statement in SCAPY_RPC_SERVER_IMPORTS:\n+ self.session.send_command(import_statement)\n+\n+ # start the server\n+ xmlrpc_server_listen_port = 8000\n+ self._start_xmlrpc_server_in_remote_python(xmlrpc_server_listen_port)\n+\n+ # connect to the server\n+ server_url = (\n+ f\"http://{self._tg_node.config.hostname}:{xmlrpc_server_listen_port}\"\n+ )\n+ self.rpc_server_proxy = xmlrpc.client.ServerProxy(\n+ server_url, allow_none=True, verbose=SETTINGS.verbose\n+ )\n+\n+ # add functions to the server\n+ for function in RPC_FUNCTIONS:\n+ # A slightly hacky way to move a function to the remote server.\n+ # It is constructed from the name and code on the other side.\n+ # Pickle cannot handle functions, nor can any of the other serialization\n+ # frameworks aside from the libraries used to generate pyc files, which\n+ # are even more messy to work with.\n+ function_bytes = marshal.dumps(function.__code__)\n+ self.rpc_server_proxy.add_rpc_function(function.__name__, function_bytes)\n+\n+ def _start_xmlrpc_server_in_remote_python(self, listen_port: int):\n+ # load the source of the function\n+ src = inspect.getsource(QuittableXMLRPCServer)\n+ # Lines with only whitespace break the repl if in the middle of a function\n+ # or class, so strip all lines containing only whitespace\n+ src = \"\\n\".join(\n+ [line for line in src.splitlines() if not line.isspace() and line != \"\"]\n+ )\n+\n+ spacing = \"\\n\" * 4\n+\n+ # execute it in the python terminal\n+ self.session.send_command(spacing + src + spacing)\n+ self.session.send_command(\n+ f\"server = QuittableXMLRPCServer(('0.0.0.0', {listen_port}));\"\n+ f\"server.serve_forever()\",\n+ \"XMLRPC OK\",\n+ )\n+\n def _send_packets(self, packets: list[Packet], port: Port) -> None:\n- raise NotImplementedError()\n+ packets = [packet.build() for packet in packets]\n+ self.rpc_server_proxy.scapy_send_packets(packets, port.logical_name)\n \n def _send_packets_and_capture(\n self,\n@@ -68,7 +266,25 @@ def _send_packets_and_capture(\n duration: float,\n capture_name: str = _get_default_capture_name(),\n ) -> list[Packet]:\n- raise NotImplementedError()\n+ binary_packets = [packet.build() for packet in packets]\n+\n+ xmlrpc_packets: list[\n+ xmlrpc.client.Binary\n+ ] = self.rpc_server_proxy.scapy_send_packets_and_capture(\n+ binary_packets,\n+ send_port.logical_name,\n+ receive_port.logical_name,\n+ duration,\n+ ) # type: ignore[assignment]\n+\n+ scapy_packets = [Ether(packet.data) for packet in xmlrpc_packets]\n+ return scapy_packets\n \n def close(self):\n- pass\n+ try:\n+ self.rpc_server_proxy.quit()\n+ except ConnectionRefusedError:\n+ # Because the python instance closes, we get no RPC response.\n+ # Thus, this error is expected\n+ pass\n+ self.session.close()\n", "prefixes": [ "v3", "5/6" ] }