From patchwork Thu May 9 11:20:50 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Luca Vizzarro X-Patchwork-Id: 139999 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id CEA6843F7C; Thu, 9 May 2024 13:21:19 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id B60E64067A; Thu, 9 May 2024 13:21:13 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id DA8C7402B7 for ; Thu, 9 May 2024 13:21:10 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id D5E7112FC; Thu, 9 May 2024 04:21:35 -0700 (PDT) Received: from localhost.localdomain (unknown [10.1.194.74]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id A9F1A3F6A8; Thu, 9 May 2024 04:21:09 -0700 (PDT) From: Luca Vizzarro To: dev@dpdk.org Cc: =?utf-8?q?Juraj_Linke=C5=A1?= , Jeremy Spewock , Luca Vizzarro , Paul Szczepanek Subject: [PATCH v2 1/8] dts: add params manipulation module Date: Thu, 9 May 2024 12:20:50 +0100 Message-Id: <20240509112057.1167947-2-luca.vizzarro@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240509112057.1167947-1-luca.vizzarro@arm.com> References: <20240326190422.577028-1-luca.vizzarro@arm.com> <20240509112057.1167947-1-luca.vizzarro@arm.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org This commit introduces a new "params" module, which adds a new way to manage command line parameters. The provided Params dataclass is able to read the fields of its child class and produce a string representation to supply to the command line. Any data structure that is intended to represent command line parameters can inherit it. The main purpose is to make it easier to represent data structures that map to parameters. Aiding quicker development, while minimising code bloat. Signed-off-by: Luca Vizzarro Reviewed-by: Paul Szczepanek Tested-by: Nicholas Pratte Reviewed-by: Nicholas Pratte Reviewed-by: Jeremy Spewock --- dts/framework/params/__init__.py | 274 +++++++++++++++++++++++++++++++ 1 file changed, 274 insertions(+) create mode 100644 dts/framework/params/__init__.py diff --git a/dts/framework/params/__init__.py b/dts/framework/params/__init__.py new file mode 100644 index 0000000000..aa27e34357 --- /dev/null +++ b/dts/framework/params/__init__.py @@ -0,0 +1,274 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2024 Arm Limited + +"""Parameter manipulation module. + +This module provides :class:`Params` which can be used to model any data structure +that is meant to represent any command parameters. +""" + +from dataclasses import dataclass, fields +from enum import Flag +from typing import Any, Callable, Iterable, Literal, Reversible, TypedDict, cast + +from typing_extensions import Self + +#: Type for a function taking one argument. +FnPtr = Callable[[Any], Any] +#: Type for a switch parameter. +Switch = Literal[True, None] +#: Type for a yes/no switch parameter. +YesNoSwitch = Literal[True, False, None] + + +def _reduce_functions(funcs: Reversible[FnPtr]) -> FnPtr: + """Reduces an iterable of :attr:`FnPtr` from end to start to a composite function. + + If the iterable is empty, the created function just returns its fed value back. + """ + + def composite_function(value: Any): + for fn in reversed(funcs): + value = fn(value) + return value + + return composite_function + + +def convert_str(*funcs: FnPtr): + """Decorator that makes the ``__str__`` method a composite function created from its arguments. + + The :attr:`FnPtr`s fed to the decorator are executed from right to left + in the arguments list order. + + Example: + .. code:: python + + @convert_str(hex_from_flag_value) + class BitMask(enum.Flag): + A = auto() + B = auto() + + will allow ``BitMask`` to render as a hexadecimal value. + """ + + def _class_decorator(original_class): + original_class.__str__ = _reduce_functions(funcs) + return original_class + + return _class_decorator + + +def comma_separated(values: Iterable[Any]) -> str: + """Converts an iterable in a comma-separated string.""" + return ",".join([str(value).strip() for value in values if value is not None]) + + +def bracketed(value: str) -> str: + """Adds round brackets to the input.""" + return f"({value})" + + +def str_from_flag_value(flag: Flag) -> str: + """Returns the value from a :class:`enum.Flag` as a string.""" + return str(flag.value) + + +def hex_from_flag_value(flag: Flag) -> str: + """Returns the value from a :class:`enum.Flag` converted to hexadecimal.""" + return hex(flag.value) + + +class ParamsModifier(TypedDict, total=False): + """Params modifiers dict compatible with the :func:`dataclasses.field` metadata parameter.""" + + #: + Params_value_only: bool + #: + Params_short: str + #: + Params_long: str + #: + Params_multiple: bool + #: + Params_convert_value: Reversible[FnPtr] + + +@dataclass +class Params: + """Dataclass that renders its fields into command line arguments. + + The parameter name is taken from the field name by default. The following: + + .. code:: python + + name: str | None = "value" + + is rendered as ``--name=value``. + Through :func:`dataclasses.field` the resulting parameter can be manipulated by applying + this class' metadata modifier functions. + + To use fields as switches, set the value to ``True`` to render them. If you + use a yes/no switch you can also set ``False`` which would render a switch + prefixed with ``--no-``. Examples: + + .. code:: python + + interactive: Switch = True # renders --interactive + numa: YesNoSwitch = False # renders --no-numa + + Setting ``None`` will prevent it from being rendered. The :attr:`~Switch` type alias is provided + for regular switches, whereas :attr:`~YesNoSwitch` is offered for yes/no ones. + + An instance of a dataclass inheriting ``Params`` can also be assigned to an attribute, + this helps with grouping parameters together. + The attribute holding the dataclass will be ignored and the latter will just be rendered as + expected. + """ + + _suffix = "" + """Holder of the plain text value of Params when called directly. A suffix for child classes.""" + + """========= BEGIN FIELD METADATA MODIFIER FUNCTIONS ========""" + + @staticmethod + def value_only() -> ParamsModifier: + """Injects the value of the attribute as-is without flag. + + Metadata modifier for :func:`dataclasses.field`. + """ + return ParamsModifier(Params_value_only=True) + + @staticmethod + def short(name: str) -> ParamsModifier: + """Overrides any parameter name with the given short option. + + Metadata modifier for :func:`dataclasses.field`. + + Example: + .. code:: python + + logical_cores: str | None = field(default="1-4", metadata=Params.short("l")) + + will render as ``-l=1-4`` instead of ``--logical-cores=1-4``. + """ + return ParamsModifier(Params_short=name) + + @staticmethod + def long(name: str) -> ParamsModifier: + """Overrides the inferred parameter name to the specified one. + + Metadata modifier for :func:`dataclasses.field`. + + Example: + .. code:: python + + x_name: str | None = field(default="y", metadata=Params.long("x")) + + will render as ``--x=y``, but the field is accessed and modified through ``x_name``. + """ + return ParamsModifier(Params_long=name) + + @staticmethod + def multiple() -> ParamsModifier: + """Specifies that this parameter is set multiple times. Must be a list. + + Metadata modifier for :func:`dataclasses.field`. + + Example: + .. code:: python + + ports: list[int] | None = field( + default_factory=lambda: [0, 1, 2], + metadata=Params.multiple() | Params.long("port") + ) + + will render as ``--port=0 --port=1 --port=2``. Note that modifiers can be chained like + in this example. + """ + return ParamsModifier(Params_multiple=True) + + @classmethod + def convert_value(cls, *funcs: FnPtr) -> ParamsModifier: + """Takes in a variable number of functions to convert the value text representation. + + Metadata modifier for :func:`dataclasses.field`. + + The ``metadata`` keyword argument can be used to chain metadata modifiers together. + + Functions can be chained together, executed from right to left in the arguments list order. + + Example: + .. code:: python + + hex_bitmask: int | None = field( + default=0b1101, + metadata=Params.convert_value(hex) | Params.long("mask") + ) + + will render as ``--mask=0xd``. + """ + return ParamsModifier(Params_convert_value=funcs) + + """========= END FIELD METADATA MODIFIER FUNCTIONS ========""" + + def append_str(self, text: str) -> None: + """Appends a string at the end of the string representation.""" + self._suffix += text + + def __iadd__(self, text: str) -> Self: + """Appends a string at the end of the string representation.""" + self.append_str(text) + return self + + @classmethod + def from_str(cls, text: str) -> Self: + """Creates a plain Params object from a string.""" + obj = cls() + obj.append_str(text) + return obj + + @staticmethod + def _make_switch( + name: str, is_short: bool = False, is_no: bool = False, value: str | None = None + ) -> str: + prefix = f"{'-' if is_short else '--'}{'no-' if is_no else ''}" + name = name.replace("_", "-") + value = f"{' ' if is_short else '='}{value}" if value else "" + return f"{prefix}{name}{value}" + + def __str__(self) -> str: + """Returns a string of command-line-ready arguments from the class fields.""" + arguments: list[str] = [] + + for field in fields(self): + value = getattr(self, field.name) + modifiers = cast(ParamsModifier, field.metadata) + + if value is None: + continue + + value_only = modifiers.get("Params_value_only", False) + if isinstance(value, Params) or value_only: + arguments.append(str(value)) + continue + + # take the short modifier, or the long modifier, or infer from field name + switch_name = modifiers.get("Params_short", modifiers.get("Params_long", field.name)) + is_short = "Params_short" in modifiers + + if isinstance(value, bool): + arguments.append(self._make_switch(switch_name, is_short, is_no=(not value))) + continue + + convert = _reduce_functions(modifiers.get("Params_convert_value", [])) + multiple = modifiers.get("Params_multiple", False) + + values = value if multiple else [value] + for value in values: + arguments.append(self._make_switch(switch_name, is_short, value=convert(value))) + + if self._suffix: + arguments.append(self._suffix) + + return " ".join(arguments) From patchwork Thu May 9 11:20:51 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Luca Vizzarro X-Patchwork-Id: 140000 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 805FB43F7C; Thu, 9 May 2024 13:21:26 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 0573C4067D; Thu, 9 May 2024 13:21:15 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id AEB5B402B7 for ; Thu, 9 May 2024 13:21:11 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id BAD0F106F; Thu, 9 May 2024 04:21:36 -0700 (PDT) Received: from localhost.localdomain (unknown [10.1.194.74]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 903DE3F6A8; Thu, 9 May 2024 04:21:10 -0700 (PDT) From: Luca Vizzarro To: dev@dpdk.org Cc: =?utf-8?q?Juraj_Linke=C5=A1?= , Jeremy Spewock , Luca Vizzarro , Paul Szczepanek Subject: [PATCH v2 2/8] dts: use Params for interactive shells Date: Thu, 9 May 2024 12:20:51 +0100 Message-Id: <20240509112057.1167947-3-luca.vizzarro@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240509112057.1167947-1-luca.vizzarro@arm.com> References: <20240326190422.577028-1-luca.vizzarro@arm.com> <20240509112057.1167947-1-luca.vizzarro@arm.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Make it so that interactive shells accept an implementation of `Params` for app arguments. Convert EalParameters to use `Params` instead. String command line parameters can still be supplied by using the `Params.from_str()` method. Signed-off-by: Luca Vizzarro Reviewed-by: Paul Szczepanek Tested-by: Nicholas Pratte Reviewed-by: Nicholas Pratte Reviewed-by: Jeremy Spewock --- .../remote_session/interactive_shell.py | 12 +- dts/framework/remote_session/testpmd_shell.py | 11 +- dts/framework/testbed_model/node.py | 6 +- dts/framework/testbed_model/os_session.py | 4 +- dts/framework/testbed_model/sut_node.py | 124 ++++++++---------- dts/tests/TestSuite_pmd_buffer_scatter.py | 3 +- 6 files changed, 77 insertions(+), 83 deletions(-) diff --git a/dts/framework/remote_session/interactive_shell.py b/dts/framework/remote_session/interactive_shell.py index 074a541279..9da66d1c7e 100644 --- a/dts/framework/remote_session/interactive_shell.py +++ b/dts/framework/remote_session/interactive_shell.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2023 University of New Hampshire +# Copyright(c) 2024 Arm Limited """Common functionality for interactive shell handling. @@ -21,6 +22,7 @@ from paramiko import Channel, SSHClient, channel # type: ignore[import-untyped] from framework.logger import DTSLogger +from framework.params import Params from framework.settings import SETTINGS @@ -40,7 +42,7 @@ class InteractiveShell(ABC): _ssh_channel: Channel _logger: DTSLogger _timeout: float - _app_args: str + _app_params: Params #: Prompt to expect at the end of output when sending a command. #: This is often overridden by subclasses. @@ -63,7 +65,7 @@ def __init__( interactive_session: SSHClient, logger: DTSLogger, get_privileged_command: Callable[[str], str] | None, - app_args: str = "", + app_params: Params = Params(), timeout: float = SETTINGS.timeout, ) -> None: """Create an SSH channel during initialization. @@ -74,7 +76,7 @@ def __init__( get_privileged_command: A method for modifying a command to allow it to use elevated privileges. If :data:`None`, the application will not be started with elevated privileges. - app_args: The command line arguments to be passed to the application on startup. + app_params: The command line parameters to be passed to the application on startup. timeout: The timeout used for the SSH channel that is dedicated to this interactive shell. This timeout is for collecting output, so if reading from the buffer and no output is gathered within the timeout, an exception is thrown. @@ -87,7 +89,7 @@ def __init__( self._ssh_channel.set_combine_stderr(True) # combines stdout and stderr streams self._logger = logger self._timeout = timeout - self._app_args = app_args + self._app_params = app_params self._start_application(get_privileged_command) def _start_application(self, get_privileged_command: Callable[[str], str] | None) -> None: @@ -100,7 +102,7 @@ def _start_application(self, get_privileged_command: Callable[[str], str] | None get_privileged_command: A function (but could be any callable) that produces the version of the command with elevated privileges. """ - start_command = f"{self.path} {self._app_args}" + start_command = f"{self.path} {self._app_params}" if get_privileged_command is not None: start_command = get_privileged_command(start_command) self.send_command(start_command) diff --git a/dts/framework/remote_session/testpmd_shell.py b/dts/framework/remote_session/testpmd_shell.py index cb2ab6bd00..7eced27096 100644 --- a/dts/framework/remote_session/testpmd_shell.py +++ b/dts/framework/remote_session/testpmd_shell.py @@ -22,6 +22,7 @@ from framework.exception import InteractiveCommandExecutionError from framework.settings import SETTINGS +from framework.testbed_model.sut_node import EalParams from framework.utils import StrEnum from .interactive_shell import InteractiveShell @@ -118,8 +119,14 @@ def _start_application(self, get_privileged_command: Callable[[str], str] | None Also find the number of pci addresses which were allowed on the command line when the app was started. """ - self._app_args += " -i --mask-event intr_lsc" - self.number_of_ports = self._app_args.count("-a ") + self._app_params += " -i --mask-event intr_lsc" + + assert isinstance(self._app_params, EalParams) + + self.number_of_ports = ( + len(self._app_params.ports) if self._app_params.ports is not None else 0 + ) + super()._start_application(get_privileged_command) def start(self, verify: bool = True) -> None: diff --git a/dts/framework/testbed_model/node.py b/dts/framework/testbed_model/node.py index 74061f6262..6af4f25a3c 100644 --- a/dts/framework/testbed_model/node.py +++ b/dts/framework/testbed_model/node.py @@ -2,6 +2,7 @@ # Copyright(c) 2010-2014 Intel Corporation # Copyright(c) 2022-2023 PANTHEON.tech s.r.o. # Copyright(c) 2022-2023 University of New Hampshire +# Copyright(c) 2024 Arm Limited """Common functionality for node management. @@ -24,6 +25,7 @@ ) from framework.exception import ConfigurationError from framework.logger import DTSLogger, get_dts_logger +from framework.params import Params from framework.settings import SETTINGS from .cpu import ( @@ -199,7 +201,7 @@ def create_interactive_shell( shell_cls: Type[InteractiveShellType], timeout: float = SETTINGS.timeout, privileged: bool = False, - app_args: str = "", + app_params: Params = Params(), ) -> InteractiveShellType: """Factory for interactive session handlers. @@ -222,7 +224,7 @@ def create_interactive_shell( shell_cls, timeout, privileged, - app_args, + app_params, ) def filter_lcores( diff --git a/dts/framework/testbed_model/os_session.py b/dts/framework/testbed_model/os_session.py index d5bf7e0401..1a77aee532 100644 --- a/dts/framework/testbed_model/os_session.py +++ b/dts/framework/testbed_model/os_session.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2023 PANTHEON.tech s.r.o. # Copyright(c) 2023 University of New Hampshire +# Copyright(c) 2024 Arm Limited """OS-aware remote session. @@ -29,6 +30,7 @@ from framework.config import Architecture, NodeConfiguration, NodeInfo from framework.logger import DTSLogger +from framework.params import Params from framework.remote_session import ( CommandResult, InteractiveRemoteSession, @@ -134,7 +136,7 @@ def create_interactive_shell( shell_cls: Type[InteractiveShellType], timeout: float, privileged: bool, - app_args: str, + app_args: Params, ) -> InteractiveShellType: """Factory for interactive session handlers. diff --git a/dts/framework/testbed_model/sut_node.py b/dts/framework/testbed_model/sut_node.py index 97aa26d419..c886590979 100644 --- a/dts/framework/testbed_model/sut_node.py +++ b/dts/framework/testbed_model/sut_node.py @@ -2,6 +2,7 @@ # Copyright(c) 2010-2014 Intel Corporation # Copyright(c) 2023 PANTHEON.tech s.r.o. # Copyright(c) 2023 University of New Hampshire +# Copyright(c) 2024 Arm Limited """System under test (DPDK + hardware) node. @@ -14,8 +15,9 @@ import os import tarfile import time +from dataclasses import dataclass, field from pathlib import PurePath -from typing import Type +from typing import Literal, Type from framework.config import ( BuildTargetConfiguration, @@ -23,6 +25,7 @@ NodeInfo, SutNodeConfiguration, ) +from framework.params import Params, Switch from framework.remote_session import CommandResult from framework.settings import SETTINGS from framework.utils import MesonArgs @@ -34,62 +37,42 @@ from .virtual_device import VirtualDevice -class EalParameters(object): - """The environment abstraction layer parameters. - - The string representation can be created by converting the instance to a string. - """ +def _port_to_pci(port: Port) -> str: + return port.pci - def __init__( - self, - lcore_list: LogicalCoreList, - memory_channels: int, - prefix: str, - no_pci: bool, - vdevs: list[VirtualDevice], - ports: list[Port], - other_eal_param: str, - ): - """Initialize the parameters according to inputs. - - Process the parameters into the format used on the command line. - Args: - lcore_list: The list of logical cores to use. - memory_channels: The number of memory channels to use. - prefix: Set the file prefix string with which to start DPDK, e.g.: ``prefix='vf'``. - no_pci: Switch to disable PCI bus e.g.: ``no_pci=True``. - vdevs: Virtual devices, e.g.:: +@dataclass(kw_only=True) +class EalParams(Params): + """The environment abstraction layer parameters. - vdevs=[ - VirtualDevice('net_ring0'), - VirtualDevice('net_ring1') - ] - ports: The list of ports to allow. - other_eal_param: user defined DPDK EAL parameters, e.g.: + Attributes: + lcore_list: The list of logical cores to use. + memory_channels: The number of memory channels to use. + prefix: Set the file prefix string with which to start DPDK, e.g.: ``prefix="vf"``. + no_pci: Switch to disable PCI bus, e.g.: ``no_pci=True``. + vdevs: Virtual devices, e.g.:: + vdevs=[ + VirtualDevice('net_ring0'), + VirtualDevice('net_ring1') + ] + ports: The list of ports to allow. + other_eal_param: user defined DPDK EAL parameters, e.g.: ``other_eal_param='--single-file-segments'`` - """ - self._lcore_list = f"-l {lcore_list}" - self._memory_channels = f"-n {memory_channels}" - self._prefix = prefix - if prefix: - self._prefix = f"--file-prefix={prefix}" - self._no_pci = "--no-pci" if no_pci else "" - self._vdevs = " ".join(f"--vdev {vdev}" for vdev in vdevs) - self._ports = " ".join(f"-a {port.pci}" for port in ports) - self._other_eal_param = other_eal_param - - def __str__(self) -> str: - """Create the EAL string.""" - return ( - f"{self._lcore_list} " - f"{self._memory_channels} " - f"{self._prefix} " - f"{self._no_pci} " - f"{self._vdevs} " - f"{self._ports} " - f"{self._other_eal_param}" - ) + """ + + lcore_list: LogicalCoreList = field(metadata=Params.short("l")) + memory_channels: int = field(metadata=Params.short("n")) + prefix: str = field(metadata=Params.long("file-prefix")) + no_pci: Switch + vdevs: list[VirtualDevice] | None = field( + default=None, metadata=Params.multiple() | Params.long("vdev") + ) + ports: list[Port] | None = field( + default=None, + metadata=Params.convert_value(_port_to_pci) | Params.multiple() | Params.short("a"), + ) + other_eal_param: Params | None = None + _separator: Literal[True] = field(default=True, init=False, metadata=Params.short("-")) class SutNode(Node): @@ -350,11 +333,11 @@ def create_eal_parameters( ascending_cores: bool = True, prefix: str = "dpdk", append_prefix_timestamp: bool = True, - no_pci: bool = False, + no_pci: Switch = None, vdevs: list[VirtualDevice] | None = None, ports: list[Port] | None = None, other_eal_param: str = "", - ) -> "EalParameters": + ) -> EalParams: """Compose the EAL parameters. Process the list of cores and the DPDK prefix and pass that along with @@ -393,24 +376,21 @@ def create_eal_parameters( if prefix: self._dpdk_prefix_list.append(prefix) - if vdevs is None: - vdevs = [] - if ports is None: ports = self.ports - return EalParameters( + return EalParams( lcore_list=lcore_list, memory_channels=self.config.memory_channels, prefix=prefix, no_pci=no_pci, vdevs=vdevs, ports=ports, - other_eal_param=other_eal_param, + other_eal_param=Params.from_str(other_eal_param), ) def run_dpdk_app( - self, app_path: PurePath, eal_args: "EalParameters", timeout: float = 30 + self, app_path: PurePath, eal_params: EalParams, timeout: float = 30 ) -> CommandResult: """Run DPDK application on the remote node. @@ -419,14 +399,14 @@ def run_dpdk_app( Args: app_path: The remote path to the DPDK application. - eal_args: EAL parameters to run the DPDK application with. + eal_params: EAL parameters to run the DPDK application with. timeout: Wait at most this long in seconds for `command` execution to complete. Returns: The result of the DPDK app execution. """ return self.main_session.send_command( - f"{app_path} {eal_args}", timeout, privileged=True, verify=True + f"{app_path} {eal_params}", timeout, privileged=True, verify=True ) def configure_ipv4_forwarding(self, enable: bool) -> None: @@ -442,8 +422,8 @@ def create_interactive_shell( shell_cls: Type[InteractiveShellType], timeout: float = SETTINGS.timeout, privileged: bool = False, - app_parameters: str = "", - eal_parameters: EalParameters | None = None, + app_params: Params = Params(), + eal_params: EalParams | None = None, ) -> InteractiveShellType: """Extend the factory for interactive session handlers. @@ -459,26 +439,26 @@ def create_interactive_shell( reading from the buffer and don't receive any data within the timeout it will throw an error. privileged: Whether to run the shell with administrative privileges. - eal_parameters: List of EAL parameters to use to launch the app. If this + app_params: The parameters to be passed to the application. + eal_params: List of EAL parameters to use to launch the app. If this isn't provided or an empty string is passed, it will default to calling :meth:`create_eal_parameters`. - app_parameters: Additional arguments to pass into the application on the - command-line. Returns: An instance of the desired interactive application shell. """ # We need to append the build directory and add EAL parameters for DPDK apps if shell_cls.dpdk_app: - if not eal_parameters: - eal_parameters = self.create_eal_parameters() - app_parameters = f"{eal_parameters} -- {app_parameters}" + if eal_params is None: + eal_params = self.create_eal_parameters() + eal_params.append_str(str(app_params)) + app_params = eal_params shell_cls.path = self.main_session.join_remote_path( self.remote_dpdk_build_dir, shell_cls.path ) - return super().create_interactive_shell(shell_cls, timeout, privileged, app_parameters) + return super().create_interactive_shell(shell_cls, timeout, privileged, app_params) def bind_ports_to_driver(self, for_dpdk: bool = True) -> None: """Bind all ports on the SUT to a driver. diff --git a/dts/tests/TestSuite_pmd_buffer_scatter.py b/dts/tests/TestSuite_pmd_buffer_scatter.py index a020682e8d..c6e93839cb 100644 --- a/dts/tests/TestSuite_pmd_buffer_scatter.py +++ b/dts/tests/TestSuite_pmd_buffer_scatter.py @@ -22,6 +22,7 @@ from scapy.packet import Raw # type: ignore[import-untyped] from scapy.utils import hexstr # type: ignore[import-untyped] +from framework.params import Params from framework.remote_session.testpmd_shell import TestPmdForwardingModes, TestPmdShell from framework.test_suite import TestSuite @@ -103,7 +104,7 @@ def pmd_scatter(self, mbsize: int) -> None: """ testpmd = self.sut_node.create_interactive_shell( TestPmdShell, - app_parameters=( + app_params=Params.from_str( "--mbcache=200 " f"--mbuf-size={mbsize} " "--max-pkt-len=9000 " From patchwork Thu May 9 11:20:52 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Luca Vizzarro X-Patchwork-Id: 140001 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id A199043F7C; Thu, 9 May 2024 13:21:33 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 5F8364068E; Thu, 9 May 2024 13:21:16 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id 9E74C402EE for ; Thu, 9 May 2024 13:21:12 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 9E88212FC; Thu, 9 May 2024 04:21:37 -0700 (PDT) Received: from localhost.localdomain (unknown [10.1.194.74]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 780823F6A8; Thu, 9 May 2024 04:21:11 -0700 (PDT) From: Luca Vizzarro To: dev@dpdk.org Cc: =?utf-8?q?Juraj_Linke=C5=A1?= , Jeremy Spewock , Luca Vizzarro , Paul Szczepanek Subject: [PATCH v2 3/8] dts: refactor EalParams Date: Thu, 9 May 2024 12:20:52 +0100 Message-Id: <20240509112057.1167947-4-luca.vizzarro@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240509112057.1167947-1-luca.vizzarro@arm.com> References: <20240326190422.577028-1-luca.vizzarro@arm.com> <20240509112057.1167947-1-luca.vizzarro@arm.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Move EalParams to its own module to avoid circular dependencies. Signed-off-by: Luca Vizzarro Reviewed-by: Paul Szczepanek Tested-by: Nicholas Pratte Reviewed-by: Nicholas Pratte Reviewed-by: Jeremy Spewock --- dts/framework/params/eal.py | 50 +++++++++++++++++++ dts/framework/remote_session/testpmd_shell.py | 2 +- dts/framework/testbed_model/sut_node.py | 42 +--------------- 3 files changed, 53 insertions(+), 41 deletions(-) create mode 100644 dts/framework/params/eal.py diff --git a/dts/framework/params/eal.py b/dts/framework/params/eal.py new file mode 100644 index 0000000000..bbdbc8f334 --- /dev/null +++ b/dts/framework/params/eal.py @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2024 Arm Limited + +"""Module representing the DPDK EAL-related parameters.""" + +from dataclasses import dataclass, field +from typing import Literal + +from framework.params import Params, Switch +from framework.testbed_model.cpu import LogicalCoreList +from framework.testbed_model.port import Port +from framework.testbed_model.virtual_device import VirtualDevice + + +def _port_to_pci(port: Port) -> str: + return port.pci + + +@dataclass(kw_only=True) +class EalParams(Params): + """The environment abstraction layer parameters. + + Attributes: + lcore_list: The list of logical cores to use. + memory_channels: The number of memory channels to use. + prefix: Set the file prefix string with which to start DPDK, e.g.: ``prefix="vf"``. + no_pci: Switch to disable PCI bus, e.g.: ``no_pci=True``. + vdevs: Virtual devices, e.g.:: + vdevs=[ + VirtualDevice('net_ring0'), + VirtualDevice('net_ring1') + ] + ports: The list of ports to allow. + other_eal_param: user defined DPDK EAL parameters, e.g.: + ``other_eal_param='--single-file-segments'`` + """ + + lcore_list: LogicalCoreList = field(metadata=Params.short("l")) + memory_channels: int = field(metadata=Params.short("n")) + prefix: str = field(metadata=Params.long("file-prefix")) + no_pci: Switch = None + vdevs: list[VirtualDevice] | None = field( + default=None, metadata=Params.multiple() | Params.long("vdev") + ) + ports: list[Port] | None = field( + default=None, + metadata=Params.convert_value(_port_to_pci) | Params.multiple() | Params.short("a"), + ) + other_eal_param: Params | None = None + _separator: Literal[True] = field(default=True, init=False, metadata=Params.short("-")) diff --git a/dts/framework/remote_session/testpmd_shell.py b/dts/framework/remote_session/testpmd_shell.py index 7eced27096..841d456a2f 100644 --- a/dts/framework/remote_session/testpmd_shell.py +++ b/dts/framework/remote_session/testpmd_shell.py @@ -21,8 +21,8 @@ from typing import Callable, ClassVar from framework.exception import InteractiveCommandExecutionError +from framework.params.eal import EalParams from framework.settings import SETTINGS -from framework.testbed_model.sut_node import EalParams from framework.utils import StrEnum from .interactive_shell import InteractiveShell diff --git a/dts/framework/testbed_model/sut_node.py b/dts/framework/testbed_model/sut_node.py index c886590979..e1163106a3 100644 --- a/dts/framework/testbed_model/sut_node.py +++ b/dts/framework/testbed_model/sut_node.py @@ -15,9 +15,8 @@ import os import tarfile import time -from dataclasses import dataclass, field from pathlib import PurePath -from typing import Literal, Type +from typing import Type from framework.config import ( BuildTargetConfiguration, @@ -26,6 +25,7 @@ SutNodeConfiguration, ) from framework.params import Params, Switch +from framework.params.eal import EalParams from framework.remote_session import CommandResult from framework.settings import SETTINGS from framework.utils import MesonArgs @@ -37,44 +37,6 @@ from .virtual_device import VirtualDevice -def _port_to_pci(port: Port) -> str: - return port.pci - - -@dataclass(kw_only=True) -class EalParams(Params): - """The environment abstraction layer parameters. - - Attributes: - lcore_list: The list of logical cores to use. - memory_channels: The number of memory channels to use. - prefix: Set the file prefix string with which to start DPDK, e.g.: ``prefix="vf"``. - no_pci: Switch to disable PCI bus, e.g.: ``no_pci=True``. - vdevs: Virtual devices, e.g.:: - vdevs=[ - VirtualDevice('net_ring0'), - VirtualDevice('net_ring1') - ] - ports: The list of ports to allow. - other_eal_param: user defined DPDK EAL parameters, e.g.: - ``other_eal_param='--single-file-segments'`` - """ - - lcore_list: LogicalCoreList = field(metadata=Params.short("l")) - memory_channels: int = field(metadata=Params.short("n")) - prefix: str = field(metadata=Params.long("file-prefix")) - no_pci: Switch - vdevs: list[VirtualDevice] | None = field( - default=None, metadata=Params.multiple() | Params.long("vdev") - ) - ports: list[Port] | None = field( - default=None, - metadata=Params.convert_value(_port_to_pci) | Params.multiple() | Params.short("a"), - ) - other_eal_param: Params | None = None - _separator: Literal[True] = field(default=True, init=False, metadata=Params.short("-")) - - class SutNode(Node): """The system under test node. From patchwork Thu May 9 11:20:53 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Luca Vizzarro X-Patchwork-Id: 140002 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 0910C43F7C; Thu, 9 May 2024 13:21:43 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 47C16406B8; Thu, 9 May 2024 13:21:18 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id 78CB940649 for ; Thu, 9 May 2024 13:21:13 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 8322E106F; Thu, 9 May 2024 04:21:38 -0700 (PDT) Received: from localhost.localdomain (unknown [10.1.194.74]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 5B6043F6A8; Thu, 9 May 2024 04:21:12 -0700 (PDT) From: Luca Vizzarro To: dev@dpdk.org Cc: =?utf-8?q?Juraj_Linke=C5=A1?= , Jeremy Spewock , Luca Vizzarro , Paul Szczepanek Subject: [PATCH v2 4/8] dts: remove module-wide imports Date: Thu, 9 May 2024 12:20:53 +0100 Message-Id: <20240509112057.1167947-5-luca.vizzarro@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240509112057.1167947-1-luca.vizzarro@arm.com> References: <20240326190422.577028-1-luca.vizzarro@arm.com> <20240509112057.1167947-1-luca.vizzarro@arm.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Remove the imports in the testbed_model and remote_session modules init file, to avoid the initialisation of unneeded modules, thus removing or limiting the risk of circular dependencies. Signed-off-by: Luca Vizzarro Reviewed-by: Paul Szczepanek Tested-by: Nicholas Pratte Reviewed-by: Nicholas Pratte Reviewed-by: Jeremy Spewock --- dts/framework/remote_session/__init__.py | 5 +---- dts/framework/runner.py | 4 +++- dts/framework/test_suite.py | 5 ++++- dts/framework/testbed_model/__init__.py | 7 ------- dts/framework/testbed_model/os_session.py | 4 ++-- dts/framework/testbed_model/sut_node.py | 2 +- dts/framework/testbed_model/traffic_generator/scapy.py | 2 +- dts/tests/TestSuite_hello_world.py | 2 +- dts/tests/TestSuite_smoke_tests.py | 2 +- 9 files changed, 14 insertions(+), 19 deletions(-) diff --git a/dts/framework/remote_session/__init__.py b/dts/framework/remote_session/__init__.py index 1910c81c3c..29000a4642 100644 --- a/dts/framework/remote_session/__init__.py +++ b/dts/framework/remote_session/__init__.py @@ -18,11 +18,8 @@ from framework.logger import DTSLogger from .interactive_remote_session import InteractiveRemoteSession -from .interactive_shell import InteractiveShell -from .python_shell import PythonShell -from .remote_session import CommandResult, RemoteSession +from .remote_session import RemoteSession from .ssh_session import SSHSession -from .testpmd_shell import TestPmdShell def create_remote_session( diff --git a/dts/framework/runner.py b/dts/framework/runner.py index d74f1871db..e6c23af7c7 100644 --- a/dts/framework/runner.py +++ b/dts/framework/runner.py @@ -26,6 +26,9 @@ from types import FunctionType from typing import Iterable, Sequence +from framework.testbed_model.sut_node import SutNode +from framework.testbed_model.tg_node import TGNode + from .config import ( BuildTargetConfiguration, Configuration, @@ -51,7 +54,6 @@ TestSuiteWithCases, ) from .test_suite import TestSuite -from .testbed_model import SutNode, TGNode class DTSRunner: diff --git a/dts/framework/test_suite.py b/dts/framework/test_suite.py index 8768f756a6..9d3debb00f 100644 --- a/dts/framework/test_suite.py +++ b/dts/framework/test_suite.py @@ -20,9 +20,12 @@ from scapy.layers.l2 import Ether # type: ignore[import-untyped] from scapy.packet import Packet, Padding # type: ignore[import-untyped] +from framework.testbed_model.port import Port, PortLink +from framework.testbed_model.sut_node import SutNode +from framework.testbed_model.tg_node import TGNode + from .exception import TestCaseVerifyError from .logger import DTSLogger, get_dts_logger -from .testbed_model import Port, PortLink, SutNode, TGNode from .testbed_model.traffic_generator import PacketFilteringConfig from .utils import get_packet_summaries diff --git a/dts/framework/testbed_model/__init__.py b/dts/framework/testbed_model/__init__.py index 6086512ca2..4f8a58c039 100644 --- a/dts/framework/testbed_model/__init__.py +++ b/dts/framework/testbed_model/__init__.py @@ -19,10 +19,3 @@ """ # pylama:ignore=W0611 - -from .cpu import LogicalCoreCount, LogicalCoreCountFilter, LogicalCoreList -from .node import Node -from .port import Port, PortLink -from .sut_node import SutNode -from .tg_node import TGNode -from .virtual_device import VirtualDevice diff --git a/dts/framework/testbed_model/os_session.py b/dts/framework/testbed_model/os_session.py index 1a77aee532..e5f5fcbe0e 100644 --- a/dts/framework/testbed_model/os_session.py +++ b/dts/framework/testbed_model/os_session.py @@ -32,13 +32,13 @@ from framework.logger import DTSLogger from framework.params import Params from framework.remote_session import ( - CommandResult, InteractiveRemoteSession, - InteractiveShell, RemoteSession, create_interactive_session, create_remote_session, ) +from framework.remote_session.interactive_shell import InteractiveShell +from framework.remote_session.remote_session import CommandResult from framework.settings import SETTINGS from framework.utils import MesonArgs diff --git a/dts/framework/testbed_model/sut_node.py b/dts/framework/testbed_model/sut_node.py index e1163106a3..83ad06ae2d 100644 --- a/dts/framework/testbed_model/sut_node.py +++ b/dts/framework/testbed_model/sut_node.py @@ -26,7 +26,7 @@ ) from framework.params import Params, Switch from framework.params.eal import EalParams -from framework.remote_session import CommandResult +from framework.remote_session.remote_session import CommandResult from framework.settings import SETTINGS from framework.utils import MesonArgs diff --git a/dts/framework/testbed_model/traffic_generator/scapy.py b/dts/framework/testbed_model/traffic_generator/scapy.py index ed5467d825..7bc1c2cc08 100644 --- a/dts/framework/testbed_model/traffic_generator/scapy.py +++ b/dts/framework/testbed_model/traffic_generator/scapy.py @@ -25,7 +25,7 @@ from scapy.packet import Packet # type: ignore[import-untyped] from framework.config import OS, ScapyTrafficGeneratorConfig -from framework.remote_session import PythonShell +from framework.remote_session.python_shell import PythonShell from framework.settings import SETTINGS from framework.testbed_model.node import Node from framework.testbed_model.port import Port diff --git a/dts/tests/TestSuite_hello_world.py b/dts/tests/TestSuite_hello_world.py index fd7ff1534d..0d6995f260 100644 --- a/dts/tests/TestSuite_hello_world.py +++ b/dts/tests/TestSuite_hello_world.py @@ -8,7 +8,7 @@ """ from framework.test_suite import TestSuite -from framework.testbed_model import ( +from framework.testbed_model.cpu import ( LogicalCoreCount, LogicalCoreCountFilter, LogicalCoreList, diff --git a/dts/tests/TestSuite_smoke_tests.py b/dts/tests/TestSuite_smoke_tests.py index a553e89662..ca678f662d 100644 --- a/dts/tests/TestSuite_smoke_tests.py +++ b/dts/tests/TestSuite_smoke_tests.py @@ -15,7 +15,7 @@ import re from framework.config import PortConfig -from framework.remote_session import TestPmdShell +from framework.remote_session.testpmd_shell import TestPmdShell from framework.settings import SETTINGS from framework.test_suite import TestSuite from framework.utils import REGEX_FOR_PCI_ADDRESS From patchwork Thu May 9 11:20:54 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: Luca Vizzarro X-Patchwork-Id: 140003 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 1658143F7C; Thu, 9 May 2024 13:21:50 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 9A19640693; Thu, 9 May 2024 13:21:19 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id 68A964067D for ; Thu, 9 May 2024 13:21:14 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 6C7701474; Thu, 9 May 2024 04:21:39 -0700 (PDT) Received: from localhost.localdomain (unknown [10.1.194.74]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 400C43F6A8; Thu, 9 May 2024 04:21:13 -0700 (PDT) From: Luca Vizzarro To: dev@dpdk.org Cc: =?utf-8?q?Juraj_Linke=C5=A1?= , Jeremy Spewock , Luca Vizzarro , Paul Szczepanek Subject: [PATCH v2 5/8] dts: add testpmd shell params Date: Thu, 9 May 2024 12:20:54 +0100 Message-Id: <20240509112057.1167947-6-luca.vizzarro@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240509112057.1167947-1-luca.vizzarro@arm.com> References: <20240326190422.577028-1-luca.vizzarro@arm.com> <20240509112057.1167947-1-luca.vizzarro@arm.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Implement all the testpmd shell parameters into a data structure. Signed-off-by: Luca Vizzarro Reviewed-by: Paul Szczepanek Tested-by: Nicholas Pratte Reviewed-by: Nicholas Pratte Reviewed-by: Jeremy Spewock --- dts/framework/params/testpmd.py | 608 ++++++++++++++++++ dts/framework/remote_session/testpmd_shell.py | 42 +- dts/tests/TestSuite_pmd_buffer_scatter.py | 5 +- 3 files changed, 615 insertions(+), 40 deletions(-) create mode 100644 dts/framework/params/testpmd.py diff --git a/dts/framework/params/testpmd.py b/dts/framework/params/testpmd.py new file mode 100644 index 0000000000..f8f70320cf --- /dev/null +++ b/dts/framework/params/testpmd.py @@ -0,0 +1,608 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2024 Arm Limited + +"""Module containing all the TestPmd-related parameter classes.""" + +from dataclasses import dataclass, field +from enum import EnumMeta, Flag, auto, unique +from pathlib import PurePath +from typing import Literal, NamedTuple + +from framework.params import ( + Params, + Switch, + YesNoSwitch, + bracketed, + comma_separated, + convert_str, + hex_from_flag_value, + str_from_flag_value, +) +from framework.params.eal import EalParams +from framework.utils import StrEnum + + +class PortTopology(StrEnum): + """Enum representing the port topology.""" + + paired = auto() + """In paired mode, the forwarding is between pairs of ports, e.g.: (0,1), (2,3), (4,5).""" + chained = auto() + """In chained mode, the forwarding is to the next available port in the port mask, e.g.: + (0,1), (1,2), (2,0). + + The ordering of the ports can be changed using the portlist testpmd runtime function. + """ + loop = auto() + """In loop mode, ingress traffic is simply transmitted back on the same interface.""" + + +@convert_str(bracketed, comma_separated) +class PortNUMAConfig(NamedTuple): + """DPDK port to NUMA socket association tuple.""" + + #: + port: int + #: + socket: int + + +@convert_str(str_from_flag_value) +@unique +class FlowDirection(Flag): + """Flag indicating the direction of the flow. + + A bi-directional flow can be specified with the pipe: + + >>> TestPmdFlowDirection.RX | TestPmdFlowDirection.TX + + """ + + #: + RX = 1 << 0 + #: + TX = 1 << 1 + + +@convert_str(bracketed, comma_separated) +class RingNUMAConfig(NamedTuple): + """Tuple associating DPDK port, direction of the flow and NUMA socket.""" + + #: + port: int + #: + direction: FlowDirection + #: + socket: int + + +@convert_str(comma_separated) +class EthPeer(NamedTuple): + """Tuple associating a MAC address to the specified DPDK port.""" + + #: + port_no: int + #: + mac_address: str + + +@convert_str(comma_separated) +class TxIPAddrPair(NamedTuple): + """Tuple specifying the source and destination IPs for the packets.""" + + #: + source_ip: str + #: + dest_ip: str + + +@convert_str(comma_separated) +class TxUDPPortPair(NamedTuple): + """Tuple specifying the UDP source and destination ports for the packets. + + If leaving ``dest_port`` unspecified, ``source_port`` will be used for + the destination port as well. + """ + + #: + source_port: int + #: + dest_port: int | None = None + + +@dataclass +class DisableRSS(Params): + """Disables RSS (Receive Side Scaling).""" + + _disable_rss: Literal[True] = field( + default=True, init=False, metadata=Params.long("disable-rss") + ) + + +@dataclass +class SetRSSIPOnly(Params): + """Sets RSS (Receive Side Scaling) functions for IPv4/IPv6 only.""" + + _rss_ip: Literal[True] = field(default=True, init=False, metadata=Params.long("rss-ip")) + + +@dataclass +class SetRSSUDP(Params): + """Sets RSS (Receive Side Scaling) functions for IPv4/IPv6 and UDP.""" + + _rss_udp: Literal[True] = field(default=True, init=False, metadata=Params.long("rss-udp")) + + +class RSSSetting(EnumMeta): + """Enum representing a RSS setting. Each property is a class that needs to be initialised.""" + + #: + Disabled = DisableRSS + #: + SetIPOnly = SetRSSIPOnly + #: + SetUDP = SetRSSUDP + + +class SimpleForwardingModes(StrEnum): + r"""The supported packet forwarding modes for :class:`~TestPmdShell`\s.""" + + #: + io = auto() + #: + mac = auto() + #: + macswap = auto() + #: + rxonly = auto() + #: + csum = auto() + #: + icmpecho = auto() + #: + ieee1588 = auto() + #: + fivetswap = "5tswap" + #: + shared_rxq = "shared-rxq" + #: + recycle_mbufs = auto() + + +@dataclass(kw_only=True) +class TXOnlyForwardingMode(Params): + """Sets a TX-Only forwarding mode. + + Attributes: + multi_flow: Generates multiple flows if set to True. + segments_length: Sets TX segment sizes or total packet length. + """ + + _forward_mode: Literal["txonly"] = field( + default="txonly", init=False, metadata=Params.long("forward-mode") + ) + multi_flow: Switch = field(default=None, metadata=Params.long("txonly-multi-flow")) + segments_length: list[int] | None = field( + default=None, metadata=Params.long("txpkts") | Params.convert_value(comma_separated) + ) + + +@dataclass(kw_only=True) +class FlowGenForwardingMode(Params): + """Sets a flowgen forwarding mode. + + Attributes: + clones: Set the number of each packet clones to be sent. Sending clones reduces host CPU + load on creating packets and may help in testing extreme speeds or maxing out + Tx packet performance. N should be not zero, but less than ‘burst’ parameter. + flows: Set the number of flows to be generated, where 1 <= N <= INT32_MAX. + segments_length: Set TX segment sizes or total packet length. + """ + + _forward_mode: Literal["flowgen"] = field( + default="flowgen", init=False, metadata=Params.long("forward-mode") + ) + clones: int | None = field(default=None, metadata=Params.long("flowgen-clones")) + flows: int | None = field(default=None, metadata=Params.long("flowgen-flows")) + segments_length: list[int] | None = field( + default=None, metadata=Params.long("txpkts") | Params.convert_value(comma_separated) + ) + + +@dataclass(kw_only=True) +class NoisyForwardingMode(Params): + """Sets a noisy forwarding mode. + + Attributes: + forward_mode: Set the noisy VNF forwarding mode. + tx_sw_buffer_size: Set the maximum number of elements of the FIFO queue to be created for + buffering packets. + tx_sw_buffer_flushtime: Set the time before packets in the FIFO queue are flushed. + lkup_memory: Set the size of the noisy neighbor simulation memory buffer in MB to N. + lkup_num_reads: Set the size of the noisy neighbor simulation memory buffer in MB to N. + lkup_num_writes: Set the number of writes to be done in noisy neighbor simulation + memory buffer to N. + lkup_num_reads_writes: Set the number of r/w accesses to be done in noisy neighbor + simulation memory buffer to N. + """ + + _forward_mode: Literal["noisy"] = field( + default="noisy", init=False, metadata=Params.long("forward-mode") + ) + forward_mode: ( + Literal[ + SimpleForwardingModes.io, + SimpleForwardingModes.mac, + SimpleForwardingModes.macswap, + SimpleForwardingModes.fivetswap, + ] + | None + ) = field(default=SimpleForwardingModes.io, metadata=Params.long("noisy-forward-mode")) + tx_sw_buffer_size: int | None = field( + default=None, metadata=Params.long("noisy-tx-sw-buffer-size") + ) + tx_sw_buffer_flushtime: int | None = field( + default=None, metadata=Params.long("noisy-tx-sw-buffer-flushtime") + ) + lkup_memory: int | None = field(default=None, metadata=Params.long("noisy-lkup-memory")) + lkup_num_reads: int | None = field(default=None, metadata=Params.long("noisy-lkup-num-reads")) + lkup_num_writes: int | None = field(default=None, metadata=Params.long("noisy-lkup-num-writes")) + lkup_num_reads_writes: int | None = field( + default=None, metadata=Params.long("noisy-lkup-num-reads-writes") + ) + + +@convert_str(hex_from_flag_value) +@unique +class HairpinMode(Flag): + """Flag representing the hairpin mode.""" + + TWO_PORTS_LOOP = 1 << 0 + """Two hairpin ports loop.""" + TWO_PORTS_PAIRED = 1 << 1 + """Two hairpin ports paired.""" + EXPLICIT_TX_FLOW = 1 << 4 + """Explicit Tx flow rule.""" + FORCE_RX_QUEUE_MEM_SETTINGS = 1 << 8 + """Force memory settings of hairpin RX queue.""" + FORCE_TX_QUEUE_MEM_SETTINGS = 1 << 9 + """Force memory settings of hairpin TX queue.""" + RX_QUEUE_USE_LOCKED_DEVICE_MEMORY = 1 << 12 + """Hairpin RX queues will use locked device memory.""" + RX_QUEUE_USE_RTE_MEMORY = 1 << 13 + """Hairpin RX queues will use RTE memory.""" + TX_QUEUE_USE_LOCKED_DEVICE_MEMORY = 1 << 16 + """Hairpin TX queues will use locked device memory.""" + TX_QUEUE_USE_RTE_MEMORY = 1 << 18 + """Hairpin TX queues will use RTE memory.""" + + +@dataclass(kw_only=True) +class RXRingParams(Params): + """Sets the RX ring parameters. + + Attributes: + descriptors: Set the number of descriptors in the RX rings to N, where N > 0. + prefetch_threshold: Set the prefetch threshold register of RX rings to N, where N >= 0. + host_threshold: Set the host threshold register of RX rings to N, where N >= 0. + write_back_threshold: Set the write-back threshold register of RX rings to N, where N >= 0. + free_threshold: Set the free threshold of RX descriptors to N, + where 0 <= N < value of ``-–rxd``. + """ + + descriptors: int | None = field(default=None, metadata=Params.long("rxd")) + prefetch_threshold: int | None = field(default=None, metadata=Params.long("rxpt")) + host_threshold: int | None = field(default=None, metadata=Params.long("rxht")) + write_back_threshold: int | None = field(default=None, metadata=Params.long("rxwt")) + free_threshold: int | None = field(default=None, metadata=Params.long("rxfreet")) + + +@convert_str(hex_from_flag_value) +@unique +class RXMultiQueueMode(Flag): + """Flag representing the RX multi-queue mode.""" + + #: + RSS = 1 << 0 + #: + DCB = 1 << 1 + #: + VMDQ = 1 << 2 + + +@dataclass(kw_only=True) +class TXRingParams(Params): + """Sets the TX ring parameters. + + Attributes: + descriptors: Set the number of descriptors in the TX rings to N, where N > 0. + rs_bit_threshold: Set the transmit RS bit threshold of TX rings to N, + where 0 <= N <= value of ``--txd``. + prefetch_threshold: Set the prefetch threshold register of TX rings to N, where N >= 0. + host_threshold: Set the host threshold register of TX rings to N, where N >= 0. + write_back_threshold: Set the write-back threshold register of TX rings to N, where N >= 0. + free_threshold: Set the transmit free threshold of TX rings to N, + where 0 <= N <= value of ``--txd``. + """ + + descriptors: int | None = field(default=None, metadata=Params.long("txd")) + rs_bit_threshold: int | None = field(default=None, metadata=Params.long("txrst")) + prefetch_threshold: int | None = field(default=None, metadata=Params.long("txpt")) + host_threshold: int | None = field(default=None, metadata=Params.long("txht")) + write_back_threshold: int | None = field(default=None, metadata=Params.long("txwt")) + free_threshold: int | None = field(default=None, metadata=Params.long("txfreet")) + + +class Event(StrEnum): + """Enum representing a testpmd event.""" + + #: + unknown = auto() + #: + queue_state = auto() + #: + vf_mbox = auto() + #: + macsec = auto() + #: + intr_lsc = auto() + #: + intr_rmv = auto() + #: + intr_reset = auto() + #: + dev_probed = auto() + #: + dev_released = auto() + #: + flow_aged = auto() + #: + err_recovering = auto() + #: + recovery_success = auto() + #: + recovery_failed = auto() + #: + all = auto() + + +class SimpleMempoolAllocationMode(StrEnum): + """Enum representing simple mempool allocation modes.""" + + native = auto() + """Create and populate mempool using native DPDK memory.""" + xmem = auto() + """Create and populate mempool using externally and anonymously allocated area.""" + xmemhuge = auto() + """Create and populate mempool using externally and anonymously allocated hugepage area.""" + + +@dataclass(kw_only=True) +class AnonMempoolAllocationMode(Params): + """Create mempool using native DPDK memory, but populate using anonymous memory. + + Attributes: + no_iova_contig: Enables to create mempool which is not IOVA contiguous. + """ + + _mp_alloc: Literal["anon"] = field(default="anon", init=False, metadata=Params.long("mp-alloc")) + no_iova_contig: Switch = None + + +@dataclass(slots=True, kw_only=True) +class TestPmdParams(EalParams): + """The testpmd shell parameters. + + Attributes: + interactive_mode: Runs testpmd in interactive mode. + auto_start: Start forwarding on initialization. + tx_first: Start forwarding, after sending a burst of packets first. + stats_period: Display statistics every ``PERIOD`` seconds, if interactive mode is disabled. + The default value is 0, which means that the statistics will not be displayed. + + .. note:: This flag should be used only in non-interactive mode. + display_xstats: Display comma-separated list of extended statistics every ``PERIOD`` seconds + as specified in ``--stats-period`` or when used with interactive commands + that show Rx/Tx statistics (i.e. ‘show port stats’). + nb_cores: Set the number of forwarding cores, where 1 <= N <= “number of cores” or + ``RTE_MAX_LCORE`` from the configuration file. + coremask: Set the bitmask of the cores running the packet forwarding test. The main + lcore is reserved for command line parsing only and cannot be masked on for packet + forwarding. + nb_ports: Set the number of forwarding ports, where 1 <= N <= “number of ports” on the board + or ``RTE_MAX_ETHPORTS`` from the configuration file. The default value is the + number of ports on the board. + port_topology: Set port topology, where mode is paired (the default), chained or loop. + portmask: Set the bitmask of the ports used by the packet forwarding test. + portlist: Set the forwarding ports based on the user input used by the packet forwarding + test. ‘-‘ denotes a range of ports to set including the two specified port IDs ‘,’ + separates multiple port values. Possible examples like –portlist=0,1 or + –portlist=0-2 or –portlist=0,1-2 etc. + numa: Enable/disable NUMA-aware allocation of RX/TX rings and of RX memory buffers (mbufs). + socket_num: Set the socket from which all memory is allocated in NUMA mode, where + 0 <= N < number of sockets on the board. + port_numa_config: Specify the socket on which the memory pool to be used by the port will be + allocated. + ring_numa_config: Specify the socket on which the TX/RX rings for the port will be + allocated. Where flag is 1 for RX, 2 for TX, and 3 for RX and TX. + total_num_mbufs: Set the number of mbufs to be allocated in the mbuf pools, where N > 1024. + mbuf_size: Set the data size of the mbufs used to N bytes, where N < 65536. + If multiple mbuf-size values are specified the extra memory pools will be created + for allocating mbufs to receive packets with buffer splitting features. + mbcache: Set the cache of mbuf memory pools to N, where 0 <= N <= 512. + max_pkt_len: Set the maximum packet size to N bytes, where N >= 64. + eth_peers_configfile: Use a configuration file containing the Ethernet addresses of + the peer ports. + eth_peer: Set the MAC address XX:XX:XX:XX:XX:XX of the peer port N, + where 0 <= N < RTE_MAX_ETHPORTS. + tx_ip: Set the source and destination IP address used when doing transmit only test. + The defaults address values are source 198.18.0.1 and destination 198.18.0.2. + These are special purpose addresses reserved for benchmarking (RFC 5735). + tx_udp: Set the source and destination UDP port number for transmit test only test. + The default port is the port 9 which is defined for the discard protocol (RFC 863). + enable_lro: Enable large receive offload. + max_lro_pkt_size: Set the maximum LRO aggregated packet size to N bytes, where N >= 64. + disable_crc_strip: Disable hardware CRC stripping. + enable_scatter: Enable scatter (multi-segment) RX. + enable_hw_vlan: Enable hardware VLAN. + enable_hw_vlan_filter: Enable hardware VLAN filter. + enable_hw_vlan_strip: Enable hardware VLAN strip. + enable_hw_vlan_extend: Enable hardware VLAN extend. + enable_hw_qinq_strip: Enable hardware QINQ strip. + pkt_drop_enabled: Enable per-queue packet drop for packets with no descriptors. + rss: Receive Side Scaling setting. + forward_mode: Set the forwarding mode. + hairpin_mode: Set the hairpin port configuration. + hairpin_queues: Set the number of hairpin queues per port to N, where 1 <= N <= 65535. + burst: Set the number of packets per burst to N, where 1 <= N <= 512. + enable_rx_cksum: Enable hardware RX checksum offload. + rx_queues: Set the number of RX queues per port to N, where 1 <= N <= 65535. + rx_ring: Set the RX rings parameters. + no_flush_rx: Don’t flush the RX streams before starting forwarding. Used mainly with + the PCAP PMD. + rx_segments_offsets: Set the offsets of packet segments on receiving + if split feature is engaged. + rx_segments_length: Set the length of segments to scatter packets on receiving + if split feature is engaged. + multi_rx_mempool: Enable multiple mbuf pools per Rx queue. + rx_shared_queue: Create queues in shared Rx queue mode if device supports. Shared Rx queues + are grouped per X ports. X defaults to UINT32_MAX, implies all ports join + share group 1. Forwarding engine “shared-rxq” should be used for shared Rx + queues. This engine does Rx only and update stream statistics accordingly. + rx_offloads: Set the bitmask of RX queue offloads. + rx_mq_mode: Set the RX multi queue mode which can be enabled. + tx_queues: Set the number of TX queues per port to N, where 1 <= N <= 65535. + tx_ring: Set the TX rings params. + tx_offloads: Set the hexadecimal bitmask of TX queue offloads. + eth_link_speed: Set a forced link speed to the ethernet port. E.g. 1000 for 1Gbps. + disable_link_check: Disable check on link status when starting/stopping ports. + disable_device_start: Do not automatically start all ports. This allows testing + configuration of rx and tx queues before device is started + for the first time. + no_lsc_interrupt: Disable LSC interrupts for all ports, even those supporting it. + no_rmv_interrupt: Disable RMV interrupts for all ports, even those supporting it. + bitrate_stats: Set the logical core N to perform bitrate calculation. + latencystats: Set the logical core N to perform latency and jitter calculations. + print_events: Enable printing the occurrence of the designated events. + Using :attr:`TestPmdEvent.ALL` will enable all of them. + mask_events: Disable printing the occurrence of the designated events. + Using :attr:`TestPmdEvent.ALL` will disable all of them. + flow_isolate_all: Providing this parameter requests flow API isolated mode on all ports at + initialization time. It ensures all traffic is received through the + configured flow rules only (see flow command). Ports that do not support + this mode are automatically discarded. + disable_flow_flush: Disable port flow flush when stopping port. + This allows testing keep flow rules or shared flow objects across + restart. + hot_plug: Enable device event monitor mechanism for hotplug. + vxlan_gpe_port: Set the UDP port number of tunnel VXLAN-GPE to N. + geneve_parsed_port: Set the UDP port number that is used for parsing the GENEVE protocol + to N. HW may be configured with another tunnel Geneve port. + lock_all_memory: Enable/disable locking all memory. Disabled by default. + mempool_allocation_mode: Set mempool allocation mode. + record_core_cycles: Enable measurement of CPU cycles per packet. + record_burst_status: Enable display of RX and TX burst stats. + """ + + interactive_mode: Switch = field(default=True, metadata=Params.short("i")) + auto_start: Switch = field(default=None, metadata=Params.short("a")) + tx_first: Switch = None + stats_period: int | None = None + display_xstats: list[str] | None = field( + default=None, metadata=Params.convert_value(comma_separated) + ) + nb_cores: int | None = None + coremask: int | None = field(default=None, metadata=Params.convert_value(hex)) + nb_ports: int | None = None + port_topology: PortTopology | None = PortTopology.paired + portmask: int | None = field(default=None, metadata=Params.convert_value(hex)) + portlist: str | None = None # TODO: can be ranges 0,1-3 + + numa: YesNoSwitch = True + socket_num: int | None = None + port_numa_config: list[PortNUMAConfig] | None = field( + default=None, metadata=Params.convert_value(comma_separated) + ) + ring_numa_config: list[RingNUMAConfig] | None = field( + default=None, metadata=Params.convert_value(comma_separated) + ) + total_num_mbufs: int | None = None + mbuf_size: list[int] | None = field( + default=None, metadata=Params.convert_value(comma_separated) + ) + mbcache: int | None = None + max_pkt_len: int | None = None + eth_peers_configfile: PurePath | None = None + eth_peer: list[EthPeer] | None = field(default=None, metadata=Params.multiple()) + tx_ip: TxIPAddrPair | None = TxIPAddrPair(source_ip="198.18.0.1", dest_ip="198.18.0.2") + tx_udp: TxUDPPortPair | None = TxUDPPortPair(9) + enable_lro: Switch = None + max_lro_pkt_size: int | None = None + disable_crc_strip: Switch = None + enable_scatter: Switch = None + enable_hw_vlan: Switch = None + enable_hw_vlan_filter: Switch = None + enable_hw_vlan_strip: Switch = None + enable_hw_vlan_extend: Switch = None + enable_hw_qinq_strip: Switch = None + pkt_drop_enabled: Switch = field(default=None, metadata=Params.long("enable-drop-en")) + rss: RSSSetting | None = None + forward_mode: ( + SimpleForwardingModes + | FlowGenForwardingMode + | TXOnlyForwardingMode + | NoisyForwardingMode + | None + ) = SimpleForwardingModes.io + hairpin_mode: HairpinMode | None = HairpinMode(0) + hairpin_queues: int | None = field(default=None, metadata=Params.long("hairpinq")) + burst: int | None = None + enable_rx_cksum: Switch = None + + rx_queues: int | None = field(default=None, metadata=Params.long("rxq")) + rx_ring: RXRingParams | None = None + no_flush_rx: Switch = None + rx_segments_offsets: list[int] | None = field( + default=None, metadata=Params.long("rxoffs") | Params.convert_value(comma_separated) + ) + rx_segments_length: list[int] | None = field( + default=None, metadata=Params.long("rxpkts") | Params.convert_value(comma_separated) + ) + multi_rx_mempool: Switch = None + rx_shared_queue: Switch | int = field(default=None, metadata=Params.long("rxq-share")) + rx_offloads: int | None = field(default=None, metadata=Params.convert_value(hex)) + rx_mq_mode: RXMultiQueueMode | None = ( + RXMultiQueueMode.DCB | RXMultiQueueMode.RSS | RXMultiQueueMode.VMDQ + ) + + tx_queues: int | None = field(default=None, metadata=Params.long("txq")) + tx_ring: TXRingParams | None = None + tx_offloads: int | None = field(default=None, metadata=Params.convert_value(hex)) + + eth_link_speed: int | None = None + disable_link_check: Switch = None + disable_device_start: Switch = None + no_lsc_interrupt: Switch = None + no_rmv_interrupt: Switch = None + bitrate_stats: int | None = None + latencystats: int | None = None + print_events: list[Event] | None = field( + default=None, metadata=Params.multiple() | Params.long("print-event") + ) + mask_events: list[Event] | None = field( + default_factory=lambda: [Event.intr_lsc], + metadata=Params.multiple() | Params.long("mask-event"), + ) + + flow_isolate_all: Switch = None + disable_flow_flush: Switch = None + + hot_plug: Switch = None + vxlan_gpe_port: int | None = None + geneve_parsed_port: int | None = None + lock_all_memory: YesNoSwitch = field(default=False, metadata=Params.long("mlockall")) + mempool_allocation_mode: SimpleMempoolAllocationMode | AnonMempoolAllocationMode | None = field( + default=None, metadata=Params.long("mp-alloc") + ) + record_core_cycles: Switch = None + record_burst_status: Switch = None diff --git a/dts/framework/remote_session/testpmd_shell.py b/dts/framework/remote_session/testpmd_shell.py index 841d456a2f..ef3f23c582 100644 --- a/dts/framework/remote_session/testpmd_shell.py +++ b/dts/framework/remote_session/testpmd_shell.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2023 University of New Hampshire # Copyright(c) 2023 PANTHEON.tech s.r.o. +# Copyright(c) 2024 Arm Limited """Testpmd interactive shell. @@ -16,14 +17,12 @@ """ import time -from enum import auto from pathlib import PurePath from typing import Callable, ClassVar from framework.exception import InteractiveCommandExecutionError -from framework.params.eal import EalParams +from framework.params.testpmd import SimpleForwardingModes, TestPmdParams from framework.settings import SETTINGS -from framework.utils import StrEnum from .interactive_shell import InteractiveShell @@ -50,37 +49,6 @@ def __str__(self) -> str: return self.pci_address -class TestPmdForwardingModes(StrEnum): - r"""The supported packet forwarding modes for :class:`~TestPmdShell`\s.""" - - #: - io = auto() - #: - mac = auto() - #: - macswap = auto() - #: - flowgen = auto() - #: - rxonly = auto() - #: - txonly = auto() - #: - csum = auto() - #: - icmpecho = auto() - #: - ieee1588 = auto() - #: - noisy = auto() - #: - fivetswap = "5tswap" - #: - shared_rxq = "shared-rxq" - #: - recycle_mbufs = auto() - - class TestPmdShell(InteractiveShell): """Testpmd interactive shell. @@ -119,9 +87,7 @@ def _start_application(self, get_privileged_command: Callable[[str], str] | None Also find the number of pci addresses which were allowed on the command line when the app was started. """ - self._app_params += " -i --mask-event intr_lsc" - - assert isinstance(self._app_params, EalParams) + assert isinstance(self._app_params, TestPmdParams) self.number_of_ports = ( len(self._app_params.ports) if self._app_params.ports is not None else 0 @@ -213,7 +179,7 @@ def wait_link_status_up(self, port_id: int, timeout=SETTINGS.timeout) -> bool: self._logger.error(f"The link for port {port_id} did not come up in the given timeout.") return "Link status: up" in port_info - def set_forward_mode(self, mode: TestPmdForwardingModes, verify: bool = True): + def set_forward_mode(self, mode: SimpleForwardingModes, verify: bool = True): """Set packet forwarding mode. Args: diff --git a/dts/tests/TestSuite_pmd_buffer_scatter.py b/dts/tests/TestSuite_pmd_buffer_scatter.py index c6e93839cb..578b5a4318 100644 --- a/dts/tests/TestSuite_pmd_buffer_scatter.py +++ b/dts/tests/TestSuite_pmd_buffer_scatter.py @@ -23,7 +23,8 @@ from scapy.utils import hexstr # type: ignore[import-untyped] from framework.params import Params -from framework.remote_session.testpmd_shell import TestPmdForwardingModes, TestPmdShell +from framework.params.testpmd import SimpleForwardingModes +from framework.remote_session.testpmd_shell import TestPmdShell from framework.test_suite import TestSuite @@ -113,7 +114,7 @@ def pmd_scatter(self, mbsize: int) -> None: ), privileged=True, ) - testpmd.set_forward_mode(TestPmdForwardingModes.mac) + testpmd.set_forward_mode(SimpleForwardingModes.mac) testpmd.start() for offset in [-1, 0, 1, 4, 5]: From patchwork Thu May 9 11:20:55 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Luca Vizzarro X-Patchwork-Id: 140004 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 05E5F43F7C; Thu, 9 May 2024 13:21:57 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id E7D4840A67; Thu, 9 May 2024 13:21:20 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id 4839C40685 for ; Thu, 9 May 2024 13:21:15 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 4D75C106F; Thu, 9 May 2024 04:21:40 -0700 (PDT) Received: from localhost.localdomain (unknown [10.1.194.74]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 2817D3F6A8; Thu, 9 May 2024 04:21:14 -0700 (PDT) From: Luca Vizzarro To: dev@dpdk.org Cc: =?utf-8?q?Juraj_Linke=C5=A1?= , Jeremy Spewock , Luca Vizzarro , Paul Szczepanek Subject: [PATCH v2 6/8] dts: use testpmd params for scatter test suite Date: Thu, 9 May 2024 12:20:55 +0100 Message-Id: <20240509112057.1167947-7-luca.vizzarro@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240509112057.1167947-1-luca.vizzarro@arm.com> References: <20240326190422.577028-1-luca.vizzarro@arm.com> <20240509112057.1167947-1-luca.vizzarro@arm.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Update the buffer scatter test suite to use TestPmdParameters instead of the StrParams implementation. Signed-off-by: Luca Vizzarro Reviewed-by: Paul Szczepanek Tested-by: Nicholas Pratte Reviewed-by: Nicholas Pratte Reviewed-by: Jeremy Spewock --- dts/tests/TestSuite_pmd_buffer_scatter.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dts/tests/TestSuite_pmd_buffer_scatter.py b/dts/tests/TestSuite_pmd_buffer_scatter.py index 578b5a4318..6d206c1a40 100644 --- a/dts/tests/TestSuite_pmd_buffer_scatter.py +++ b/dts/tests/TestSuite_pmd_buffer_scatter.py @@ -16,14 +16,14 @@ """ import struct +from dataclasses import asdict from scapy.layers.inet import IP # type: ignore[import-untyped] from scapy.layers.l2 import Ether # type: ignore[import-untyped] from scapy.packet import Raw # type: ignore[import-untyped] from scapy.utils import hexstr # type: ignore[import-untyped] -from framework.params import Params -from framework.params.testpmd import SimpleForwardingModes +from framework.params.testpmd import SimpleForwardingModes, TestPmdParams from framework.remote_session.testpmd_shell import TestPmdShell from framework.test_suite import TestSuite @@ -105,16 +105,16 @@ def pmd_scatter(self, mbsize: int) -> None: """ testpmd = self.sut_node.create_interactive_shell( TestPmdShell, - app_params=Params.from_str( - "--mbcache=200 " - f"--mbuf-size={mbsize} " - "--max-pkt-len=9000 " - "--port-topology=paired " - "--tx-offloads=0x00008000" + app_params=TestPmdParams( + forward_mode=SimpleForwardingModes.mac, + mbcache=200, + mbuf_size=[mbsize], + max_pkt_len=9000, + tx_offloads=0x00008000, + **asdict(self.sut_node.create_eal_parameters()), ), privileged=True, ) - testpmd.set_forward_mode(SimpleForwardingModes.mac) testpmd.start() for offset in [-1, 0, 1, 4, 5]: From patchwork Thu May 9 11:20:56 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Luca Vizzarro X-Patchwork-Id: 140005 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 8AD2543F7C; Thu, 9 May 2024 13:22:05 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id B841840A7A; Thu, 9 May 2024 13:21:22 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id 22DD4402F1 for ; Thu, 9 May 2024 13:21:16 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 3771312FC; Thu, 9 May 2024 04:21:41 -0700 (PDT) Received: from localhost.localdomain (unknown [10.1.194.74]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 0BF7A3F6A8; Thu, 9 May 2024 04:21:14 -0700 (PDT) From: Luca Vizzarro To: dev@dpdk.org Cc: =?utf-8?q?Juraj_Linke=C5=A1?= , Jeremy Spewock , Luca Vizzarro , Paul Szczepanek Subject: [PATCH v2 7/8] dts: rework interactive shells Date: Thu, 9 May 2024 12:20:56 +0100 Message-Id: <20240509112057.1167947-8-luca.vizzarro@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240509112057.1167947-1-luca.vizzarro@arm.com> References: <20240326190422.577028-1-luca.vizzarro@arm.com> <20240509112057.1167947-1-luca.vizzarro@arm.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org The way nodes and interactive shells interact makes it difficult to develop for static type checking and hinting. The current system relies on a top-down approach, attempting to give a generic interface to the test developer, hiding the interaction of concrete shell classes as much as possible. When working with strong typing this approach is not ideal, as Python's implementation of generics is still rudimentary. This rework reverses the tests interaction to a bottom-up approach, allowing the test developer to call concrete shell classes directly, and let them ingest nodes independently. While also re-enforcing type checking and making the code easier to read. Signed-off-by: Luca Vizzarro Reviewed-by: Paul Szczepanek Tested-by: Nicholas Pratte Reviewed-by: Nicholas Pratte --- dts/framework/params/eal.py | 6 +- dts/framework/remote_session/dpdk_shell.py | 104 ++++++++++++++++ .../remote_session/interactive_shell.py | 75 +++++++----- dts/framework/remote_session/python_shell.py | 4 +- dts/framework/remote_session/testpmd_shell.py | 64 +++++----- dts/framework/testbed_model/node.py | 36 +----- dts/framework/testbed_model/os_session.py | 36 +----- dts/framework/testbed_model/sut_node.py | 112 +----------------- .../testbed_model/traffic_generator/scapy.py | 4 +- dts/tests/TestSuite_hello_world.py | 7 +- dts/tests/TestSuite_pmd_buffer_scatter.py | 21 ++-- dts/tests/TestSuite_smoke_tests.py | 2 +- 12 files changed, 201 insertions(+), 270 deletions(-) create mode 100644 dts/framework/remote_session/dpdk_shell.py diff --git a/dts/framework/params/eal.py b/dts/framework/params/eal.py index bbdbc8f334..8d7766fefc 100644 --- a/dts/framework/params/eal.py +++ b/dts/framework/params/eal.py @@ -35,9 +35,9 @@ class EalParams(Params): ``other_eal_param='--single-file-segments'`` """ - lcore_list: LogicalCoreList = field(metadata=Params.short("l")) - memory_channels: int = field(metadata=Params.short("n")) - prefix: str = field(metadata=Params.long("file-prefix")) + lcore_list: LogicalCoreList | None = field(default=None, metadata=Params.short("l")) + memory_channels: int | None = field(default=None, metadata=Params.short("n")) + prefix: str = field(default="dpdk", metadata=Params.long("file-prefix")) no_pci: Switch = None vdevs: list[VirtualDevice] | None = field( default=None, metadata=Params.multiple() | Params.long("vdev") diff --git a/dts/framework/remote_session/dpdk_shell.py b/dts/framework/remote_session/dpdk_shell.py new file mode 100644 index 0000000000..78caae36ea --- /dev/null +++ b/dts/framework/remote_session/dpdk_shell.py @@ -0,0 +1,104 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2024 Arm Limited + +"""DPDK-based interactive shell. + +Provides a base class to create interactive shells based on DPDK. +""" + + +from abc import ABC + +from framework.params.eal import EalParams +from framework.remote_session.interactive_shell import InteractiveShell +from framework.settings import SETTINGS +from framework.testbed_model.cpu import LogicalCoreCount, LogicalCoreList +from framework.testbed_model.sut_node import SutNode + + +def compute_eal_params( + node: SutNode, + params: EalParams | None = None, + lcore_filter_specifier: LogicalCoreCount | LogicalCoreList = LogicalCoreCount(), + ascending_cores: bool = True, + append_prefix_timestamp: bool = True, +) -> EalParams: + """Compute EAL parameters based on the node's specifications. + + Args: + node: The SUT node to compute the values for. + params: The EalParams object to amend, if set to None a new object is created and returned. + lcore_filter_specifier: A number of lcores/cores/sockets to use + or a list of lcore ids to use. + The default will select one lcore for each of two cores + on one socket, in ascending order of core ids. + ascending_cores: Sort cores in ascending order (lowest to highest IDs). + If :data:`False`, sort in descending order. + append_prefix_timestamp: If :data:`True`, will append a timestamp to DPDK file prefix. + """ + if params is None: + params = EalParams() + + if params.lcore_list is None: + params.lcore_list = LogicalCoreList( + node.filter_lcores(lcore_filter_specifier, ascending_cores) + ) + + prefix = params.prefix + if append_prefix_timestamp: + prefix = f"{prefix}_{node._dpdk_timestamp}" + prefix = node.main_session.get_dpdk_file_prefix(prefix) + if prefix: + node._dpdk_prefix_list.append(prefix) + params.prefix = prefix + + if params.ports is None: + params.ports = node.ports + + return params + + +class DPDKShell(InteractiveShell, ABC): + """The base class for managing DPDK-based interactive shells. + + This class shouldn't be instantiated directly, but instead be extended. + It automatically injects computed EAL parameters based on the node in the + supplied app parameters. + """ + + _node: SutNode + _app_params: EalParams + + _lcore_filter_specifier: LogicalCoreCount | LogicalCoreList + _ascending_cores: bool + _append_prefix_timestamp: bool + + def __init__( + self, + node: SutNode, + app_params: EalParams, + privileged: bool = True, + timeout: float = SETTINGS.timeout, + lcore_filter_specifier: LogicalCoreCount | LogicalCoreList = LogicalCoreCount(), + ascending_cores: bool = True, + append_prefix_timestamp: bool = True, + start_on_init: bool = True, + ) -> None: + """Overrides :meth:`~.interactive_shell.InteractiveShell.__init__`.""" + self._lcore_filter_specifier = lcore_filter_specifier + self._ascending_cores = ascending_cores + self._append_prefix_timestamp = append_prefix_timestamp + + super().__init__(node, app_params, privileged, timeout, start_on_init) + + def __post_init__(self): + """Computes EAL params based on the node capabilities before start.""" + self._app_params = compute_eal_params( + self._node, + self._app_params, + self._lcore_filter_specifier, + self._ascending_cores, + self._append_prefix_timestamp, + ) + + self._update_path(self._node.remote_dpdk_build_dir.joinpath(self.path)) diff --git a/dts/framework/remote_session/interactive_shell.py b/dts/framework/remote_session/interactive_shell.py index 9da66d1c7e..8163c8f247 100644 --- a/dts/framework/remote_session/interactive_shell.py +++ b/dts/framework/remote_session/interactive_shell.py @@ -17,13 +17,14 @@ from abc import ABC from pathlib import PurePath -from typing import Callable, ClassVar +from typing import ClassVar -from paramiko import Channel, SSHClient, channel # type: ignore[import-untyped] +from paramiko import Channel, channel # type: ignore[import-untyped] from framework.logger import DTSLogger from framework.params import Params from framework.settings import SETTINGS +from framework.testbed_model.node import Node class InteractiveShell(ABC): @@ -36,13 +37,14 @@ class InteractiveShell(ABC): session. """ - _interactive_session: SSHClient + _node: Node _stdin: channel.ChannelStdinFile _stdout: channel.ChannelFile _ssh_channel: Channel _logger: DTSLogger _timeout: float _app_params: Params + _privileged: bool #: Prompt to expect at the end of output when sending a command. #: This is often overridden by subclasses. @@ -56,57 +58,66 @@ class InteractiveShell(ABC): #: Path to the executable to start the interactive application. path: ClassVar[PurePath] - #: Whether this application is a DPDK app. If it is, the build directory - #: for DPDK on the node will be prepended to the path to the executable. - dpdk_app: ClassVar[bool] = False - def __init__( self, - interactive_session: SSHClient, - logger: DTSLogger, - get_privileged_command: Callable[[str], str] | None, + node: Node, app_params: Params = Params(), + privileged: bool = False, timeout: float = SETTINGS.timeout, + start_on_init: bool = True, ) -> None: """Create an SSH channel during initialization. Args: - interactive_session: The SSH session dedicated to interactive shells. - logger: The logger instance this session will use. - get_privileged_command: A method for modifying a command to allow it to use - elevated privileges. If :data:`None`, the application will not be started - with elevated privileges. + node: The node on which to run start the interactive shell. app_params: The command line parameters to be passed to the application on startup. + privileged: Enables the shell to run as superuser. timeout: The timeout used for the SSH channel that is dedicated to this interactive shell. This timeout is for collecting output, so if reading from the buffer and no output is gathered within the timeout, an exception is thrown. + start_on_init: Start interactive shell automatically after object initialisation. """ - self._interactive_session = interactive_session - self._ssh_channel = self._interactive_session.invoke_shell() + self._node = node + self._logger = node._logger + self._app_params = app_params + self._privileged = privileged + self._timeout = timeout + # Ensure path is properly formatted for the host + self._update_path(self._node.main_session.join_remote_path(self.path)) + + self.__post_init__() + + if start_on_init: + self.start_application() + + def __post_init__(self): + """Overridable. Method called after the object init and before application start.""" + pass + + def _setup_ssh_channel(self): + self._ssh_channel = self._node.main_session.interactive_session.session.invoke_shell() self._stdin = self._ssh_channel.makefile_stdin("w") self._stdout = self._ssh_channel.makefile("r") - self._ssh_channel.settimeout(timeout) + self._ssh_channel.settimeout(self._timeout) self._ssh_channel.set_combine_stderr(True) # combines stdout and stderr streams - self._logger = logger - self._timeout = timeout - self._app_params = app_params - self._start_application(get_privileged_command) - def _start_application(self, get_privileged_command: Callable[[str], str] | None) -> None: + def start_application(self) -> None: """Starts a new interactive application based on the path to the app. This method is often overridden by subclasses as their process for starting may look different. - - Args: - get_privileged_command: A function (but could be any callable) that produces - the version of the command with elevated privileges. """ - start_command = f"{self.path} {self._app_params}" - if get_privileged_command is not None: - start_command = get_privileged_command(start_command) + self._setup_ssh_channel() + + start_command = self._make_start_command() + if self._privileged: + start_command = self._node.main_session._get_privileged_command(start_command) self.send_command(start_command) + def _make_start_command(self) -> str: + """Makes the command that starts the interactive shell.""" + return f"{self.path} {self._app_params or ''}" + def send_command(self, command: str, prompt: str | None = None) -> str: """Send `command` and get all output before the expected ending string. @@ -150,3 +161,7 @@ def close(self) -> None: def __del__(self) -> None: """Make sure the session is properly closed before deleting the object.""" self.close() + + @classmethod + def _update_path(cls, path: PurePath) -> None: + cls.path = path diff --git a/dts/framework/remote_session/python_shell.py b/dts/framework/remote_session/python_shell.py index ccfd3783e8..953ed100df 100644 --- a/dts/framework/remote_session/python_shell.py +++ b/dts/framework/remote_session/python_shell.py @@ -6,9 +6,7 @@ Typical usage example in a TestSuite:: from framework.remote_session import PythonShell - python_shell = self.tg_node.create_interactive_shell( - PythonShell, timeout=5, privileged=True - ) + python_shell = PythonShell(self.tg_node, timeout=5, privileged=True) python_shell.send_command("print('Hello World')") python_shell.close() """ diff --git a/dts/framework/remote_session/testpmd_shell.py b/dts/framework/remote_session/testpmd_shell.py index ef3f23c582..92930d7fbb 100644 --- a/dts/framework/remote_session/testpmd_shell.py +++ b/dts/framework/remote_session/testpmd_shell.py @@ -7,9 +7,7 @@ Typical usage example in a TestSuite:: - testpmd_shell = self.sut_node.create_interactive_shell( - TestPmdShell, privileged=True - ) + testpmd_shell = TestPmdShell() devices = testpmd_shell.get_devices() for device in devices: print(device) @@ -18,13 +16,14 @@ import time from pathlib import PurePath -from typing import Callable, ClassVar +from typing import ClassVar from framework.exception import InteractiveCommandExecutionError from framework.params.testpmd import SimpleForwardingModes, TestPmdParams +from framework.remote_session.dpdk_shell import DPDKShell from framework.settings import SETTINGS - -from .interactive_shell import InteractiveShell +from framework.testbed_model.cpu import LogicalCoreCount, LogicalCoreList +from framework.testbed_model.sut_node import SutNode class TestPmdDevice(object): @@ -49,52 +48,48 @@ def __str__(self) -> str: return self.pci_address -class TestPmdShell(InteractiveShell): +class TestPmdShell(DPDKShell): """Testpmd interactive shell. The testpmd shell users should never use the :meth:`~.interactive_shell.InteractiveShell.send_command` method directly, but rather call specialized methods. If there isn't one that satisfies a need, it should be added. - - Attributes: - number_of_ports: The number of ports which were allowed on the command-line when testpmd - was started. """ - number_of_ports: int + _app_params: TestPmdParams #: The path to the testpmd executable. path: ClassVar[PurePath] = PurePath("app", "dpdk-testpmd") - #: Flag this as a DPDK app so that it's clear this is not a system app and - #: needs to be looked in a specific path. - dpdk_app: ClassVar[bool] = True - #: The testpmd's prompt. _default_prompt: ClassVar[str] = "testpmd>" #: This forces the prompt to appear after sending a command. _command_extra_chars: ClassVar[str] = "\n" - def _start_application(self, get_privileged_command: Callable[[str], str] | None) -> None: - """Overrides :meth:`~.interactive_shell._start_application`. - - Add flags for starting testpmd in interactive mode and disabling messages for link state - change events before starting the application. Link state is verified before starting - packet forwarding and the messages create unexpected newlines in the terminal which - complicates output collection. - - Also find the number of pci addresses which were allowed on the command line when the app - was started. - """ - assert isinstance(self._app_params, TestPmdParams) - - self.number_of_ports = ( - len(self._app_params.ports) if self._app_params.ports is not None else 0 + def __init__( + self, + node: SutNode, + privileged: bool = True, + timeout: float = SETTINGS.timeout, + lcore_filter_specifier: LogicalCoreCount | LogicalCoreList = LogicalCoreCount(), + ascending_cores: bool = True, + append_prefix_timestamp: bool = True, + start_on_init: bool = True, + **app_params, + ) -> None: + """Overrides :meth:`~.dpdk_shell.DPDKShell.__init__`. Changes app_params to kwargs.""" + super().__init__( + node, + TestPmdParams(**app_params), + privileged, + timeout, + lcore_filter_specifier, + ascending_cores, + append_prefix_timestamp, + start_on_init, ) - super()._start_application(get_privileged_command) - def start(self, verify: bool = True) -> None: """Start packet forwarding with the current configuration. @@ -114,7 +109,8 @@ def start(self, verify: bool = True) -> None: self._logger.debug(f"Failed to start packet forwarding: \n{start_cmd_output}") raise InteractiveCommandExecutionError("Testpmd failed to start packet forwarding.") - for port_id in range(self.number_of_ports): + number_of_ports = len(self._app_params.ports or []) + for port_id in range(number_of_ports): if not self.wait_link_status_up(port_id): raise InteractiveCommandExecutionError( "Not all ports came up after starting packet forwarding in testpmd." diff --git a/dts/framework/testbed_model/node.py b/dts/framework/testbed_model/node.py index 6af4f25a3c..88395faabe 100644 --- a/dts/framework/testbed_model/node.py +++ b/dts/framework/testbed_model/node.py @@ -15,7 +15,7 @@ from abc import ABC from ipaddress import IPv4Interface, IPv6Interface -from typing import Any, Callable, Type, Union +from typing import Any, Callable, Union from framework.config import ( OS, @@ -25,7 +25,6 @@ ) from framework.exception import ConfigurationError from framework.logger import DTSLogger, get_dts_logger -from framework.params import Params from framework.settings import SETTINGS from .cpu import ( @@ -36,7 +35,7 @@ lcore_filter, ) from .linux_session import LinuxSession -from .os_session import InteractiveShellType, OSSession +from .os_session import OSSession from .port import Port from .virtual_device import VirtualDevice @@ -196,37 +195,6 @@ def create_session(self, name: str) -> OSSession: self._other_sessions.append(connection) return connection - def create_interactive_shell( - self, - shell_cls: Type[InteractiveShellType], - timeout: float = SETTINGS.timeout, - privileged: bool = False, - app_params: Params = Params(), - ) -> InteractiveShellType: - """Factory for interactive session handlers. - - Instantiate `shell_cls` according to the remote OS specifics. - - Args: - shell_cls: The class of the shell. - timeout: Timeout for reading output from the SSH channel. If you are reading from - the buffer and don't receive any data within the timeout it will throw an error. - privileged: Whether to run the shell with administrative privileges. - app_args: The arguments to be passed to the application. - - Returns: - An instance of the desired interactive application shell. - """ - if not shell_cls.dpdk_app: - shell_cls.path = self.main_session.join_remote_path(shell_cls.path) - - return self.main_session.create_interactive_shell( - shell_cls, - timeout, - privileged, - app_params, - ) - def filter_lcores( self, filter_specifier: LogicalCoreCount | LogicalCoreList, diff --git a/dts/framework/testbed_model/os_session.py b/dts/framework/testbed_model/os_session.py index e5f5fcbe0e..e7e6c9d670 100644 --- a/dts/framework/testbed_model/os_session.py +++ b/dts/framework/testbed_model/os_session.py @@ -26,18 +26,16 @@ from collections.abc import Iterable from ipaddress import IPv4Interface, IPv6Interface from pathlib import PurePath -from typing import Type, TypeVar, Union +from typing import Union from framework.config import Architecture, NodeConfiguration, NodeInfo from framework.logger import DTSLogger -from framework.params import Params from framework.remote_session import ( InteractiveRemoteSession, RemoteSession, create_interactive_session, create_remote_session, ) -from framework.remote_session.interactive_shell import InteractiveShell from framework.remote_session.remote_session import CommandResult from framework.settings import SETTINGS from framework.utils import MesonArgs @@ -45,8 +43,6 @@ from .cpu import LogicalCore from .port import Port -InteractiveShellType = TypeVar("InteractiveShellType", bound=InteractiveShell) - class OSSession(ABC): """OS-unaware to OS-aware translation API definition. @@ -131,36 +127,6 @@ def send_command( return self.remote_session.send_command(command, timeout, verify, env) - def create_interactive_shell( - self, - shell_cls: Type[InteractiveShellType], - timeout: float, - privileged: bool, - app_args: Params, - ) -> InteractiveShellType: - """Factory for interactive session handlers. - - Instantiate `shell_cls` according to the remote OS specifics. - - Args: - shell_cls: The class of the shell. - timeout: Timeout for reading output from the SSH channel. If you are - reading from the buffer and don't receive any data within the timeout - it will throw an error. - privileged: Whether to run the shell with administrative privileges. - app_args: The arguments to be passed to the application. - - Returns: - An instance of the desired interactive application shell. - """ - return shell_cls( - self.interactive_session.session, - self._logger, - self._get_privileged_command if privileged else None, - app_args, - timeout, - ) - @staticmethod @abstractmethod def _get_privileged_command(command: str) -> str: diff --git a/dts/framework/testbed_model/sut_node.py b/dts/framework/testbed_model/sut_node.py index 83ad06ae2d..727170b7fc 100644 --- a/dts/framework/testbed_model/sut_node.py +++ b/dts/framework/testbed_model/sut_node.py @@ -16,7 +16,6 @@ import tarfile import time from pathlib import PurePath -from typing import Type from framework.config import ( BuildTargetConfiguration, @@ -24,17 +23,13 @@ NodeInfo, SutNodeConfiguration, ) -from framework.params import Params, Switch from framework.params.eal import EalParams from framework.remote_session.remote_session import CommandResult from framework.settings import SETTINGS from framework.utils import MesonArgs -from .cpu import LogicalCoreCount, LogicalCoreList from .node import Node -from .os_session import InteractiveShellType, OSSession -from .port import Port -from .virtual_device import VirtualDevice +from .os_session import OSSession class SutNode(Node): @@ -289,68 +284,6 @@ def kill_cleanup_dpdk_apps(self) -> None: self._dpdk_kill_session = self.create_session("dpdk_kill") self._dpdk_prefix_list = [] - def create_eal_parameters( - self, - lcore_filter_specifier: LogicalCoreCount | LogicalCoreList = LogicalCoreCount(), - ascending_cores: bool = True, - prefix: str = "dpdk", - append_prefix_timestamp: bool = True, - no_pci: Switch = None, - vdevs: list[VirtualDevice] | None = None, - ports: list[Port] | None = None, - other_eal_param: str = "", - ) -> EalParams: - """Compose the EAL parameters. - - Process the list of cores and the DPDK prefix and pass that along with - the rest of the arguments. - - Args: - lcore_filter_specifier: A number of lcores/cores/sockets to use - or a list of lcore ids to use. - The default will select one lcore for each of two cores - on one socket, in ascending order of core ids. - ascending_cores: Sort cores in ascending order (lowest to highest IDs). - If :data:`False`, sort in descending order. - prefix: Set the file prefix string with which to start DPDK, e.g.: ``prefix='vf'``. - append_prefix_timestamp: If :data:`True`, will append a timestamp to DPDK file prefix. - no_pci: Switch to disable PCI bus e.g.: ``no_pci=True``. - vdevs: Virtual devices, e.g.:: - - vdevs=[ - VirtualDevice('net_ring0'), - VirtualDevice('net_ring1') - ] - ports: The list of ports to allow. If :data:`None`, all ports listed in `self.ports` - will be allowed. - other_eal_param: user defined DPDK EAL parameters, e.g.: - ``other_eal_param='--single-file-segments'``. - - Returns: - An EAL param string, such as - ``-c 0xf -a 0000:88:00.0 --file-prefix=dpdk_1112_20190809143420``. - """ - lcore_list = LogicalCoreList(self.filter_lcores(lcore_filter_specifier, ascending_cores)) - - if append_prefix_timestamp: - prefix = f"{prefix}_{self._dpdk_timestamp}" - prefix = self.main_session.get_dpdk_file_prefix(prefix) - if prefix: - self._dpdk_prefix_list.append(prefix) - - if ports is None: - ports = self.ports - - return EalParams( - lcore_list=lcore_list, - memory_channels=self.config.memory_channels, - prefix=prefix, - no_pci=no_pci, - vdevs=vdevs, - ports=ports, - other_eal_param=Params.from_str(other_eal_param), - ) - def run_dpdk_app( self, app_path: PurePath, eal_params: EalParams, timeout: float = 30 ) -> CommandResult: @@ -379,49 +312,6 @@ def configure_ipv4_forwarding(self, enable: bool) -> None: """ self.main_session.configure_ipv4_forwarding(enable) - def create_interactive_shell( - self, - shell_cls: Type[InteractiveShellType], - timeout: float = SETTINGS.timeout, - privileged: bool = False, - app_params: Params = Params(), - eal_params: EalParams | None = None, - ) -> InteractiveShellType: - """Extend the factory for interactive session handlers. - - The extensions are SUT node specific: - - * The default for `eal_parameters`, - * The interactive shell path `shell_cls.path` is prepended with path to the remote - DPDK build directory for DPDK apps. - - Args: - shell_cls: The class of the shell. - timeout: Timeout for reading output from the SSH channel. If you are - reading from the buffer and don't receive any data within the timeout - it will throw an error. - privileged: Whether to run the shell with administrative privileges. - app_params: The parameters to be passed to the application. - eal_params: List of EAL parameters to use to launch the app. If this - isn't provided or an empty string is passed, it will default to calling - :meth:`create_eal_parameters`. - - Returns: - An instance of the desired interactive application shell. - """ - # We need to append the build directory and add EAL parameters for DPDK apps - if shell_cls.dpdk_app: - if eal_params is None: - eal_params = self.create_eal_parameters() - eal_params.append_str(str(app_params)) - app_params = eal_params - - shell_cls.path = self.main_session.join_remote_path( - self.remote_dpdk_build_dir, shell_cls.path - ) - - return super().create_interactive_shell(shell_cls, timeout, privileged, app_params) - def bind_ports_to_driver(self, for_dpdk: bool = True) -> None: """Bind all ports on the SUT to a driver. diff --git a/dts/framework/testbed_model/traffic_generator/scapy.py b/dts/framework/testbed_model/traffic_generator/scapy.py index 7bc1c2cc08..bf58ad1c5e 100644 --- a/dts/framework/testbed_model/traffic_generator/scapy.py +++ b/dts/framework/testbed_model/traffic_generator/scapy.py @@ -217,9 +217,7 @@ def __init__(self, tg_node: Node, config: ScapyTrafficGeneratorConfig): self._tg_node.config.os == OS.linux ), "Linux is the only supported OS for scapy traffic generation" - self.session = self._tg_node.create_interactive_shell( - PythonShell, timeout=5, privileged=True - ) + self.session = PythonShell(self._tg_node, timeout=5, privileged=True) # import libs in remote python console for import_statement in SCAPY_RPC_SERVER_IMPORTS: diff --git a/dts/tests/TestSuite_hello_world.py b/dts/tests/TestSuite_hello_world.py index 0d6995f260..d958f99030 100644 --- a/dts/tests/TestSuite_hello_world.py +++ b/dts/tests/TestSuite_hello_world.py @@ -7,6 +7,7 @@ No other EAL parameters apart from cores are used. """ +from framework.remote_session.dpdk_shell import compute_eal_params from framework.test_suite import TestSuite from framework.testbed_model.cpu import ( LogicalCoreCount, @@ -38,7 +39,7 @@ def test_hello_world_single_core(self) -> None: # get the first usable core lcore_amount = LogicalCoreCount(1, 1, 1) lcores = LogicalCoreCountFilter(self.sut_node.lcores, lcore_amount).filter() - eal_para = self.sut_node.create_eal_parameters(lcore_filter_specifier=lcore_amount) + eal_para = compute_eal_params(self.sut_node, lcore_filter_specifier=lcore_amount) result = self.sut_node.run_dpdk_app(self.app_helloworld_path, eal_para) self.verify( f"hello from core {int(lcores[0])}" in result.stdout, @@ -55,8 +56,8 @@ def test_hello_world_all_cores(self) -> None: "hello from core " """ # get the maximum logical core number - eal_para = self.sut_node.create_eal_parameters( - lcore_filter_specifier=LogicalCoreList(self.sut_node.lcores) + eal_para = compute_eal_params( + self.sut_node, lcore_filter_specifier=LogicalCoreList(self.sut_node.lcores) ) result = self.sut_node.run_dpdk_app(self.app_helloworld_path, eal_para, 50) for lcore in self.sut_node.lcores: diff --git a/dts/tests/TestSuite_pmd_buffer_scatter.py b/dts/tests/TestSuite_pmd_buffer_scatter.py index 6d206c1a40..43cf5c61eb 100644 --- a/dts/tests/TestSuite_pmd_buffer_scatter.py +++ b/dts/tests/TestSuite_pmd_buffer_scatter.py @@ -16,14 +16,13 @@ """ import struct -from dataclasses import asdict from scapy.layers.inet import IP # type: ignore[import-untyped] from scapy.layers.l2 import Ether # type: ignore[import-untyped] from scapy.packet import Raw # type: ignore[import-untyped] from scapy.utils import hexstr # type: ignore[import-untyped] -from framework.params.testpmd import SimpleForwardingModes, TestPmdParams +from framework.params.testpmd import SimpleForwardingModes from framework.remote_session.testpmd_shell import TestPmdShell from framework.test_suite import TestSuite @@ -103,17 +102,13 @@ def pmd_scatter(self, mbsize: int) -> None: Test: Start testpmd and run functional test with preset mbsize. """ - testpmd = self.sut_node.create_interactive_shell( - TestPmdShell, - app_params=TestPmdParams( - forward_mode=SimpleForwardingModes.mac, - mbcache=200, - mbuf_size=[mbsize], - max_pkt_len=9000, - tx_offloads=0x00008000, - **asdict(self.sut_node.create_eal_parameters()), - ), - privileged=True, + testpmd = TestPmdShell( + self.sut_node, + forward_mode=SimpleForwardingModes.mac, + mbcache=200, + mbuf_size=[mbsize], + max_pkt_len=9000, + tx_offloads=0x00008000, ) testpmd.start() diff --git a/dts/tests/TestSuite_smoke_tests.py b/dts/tests/TestSuite_smoke_tests.py index ca678f662d..eca27acfd8 100644 --- a/dts/tests/TestSuite_smoke_tests.py +++ b/dts/tests/TestSuite_smoke_tests.py @@ -99,7 +99,7 @@ def test_devices_listed_in_testpmd(self) -> None: Test: List all devices found in testpmd and verify the configured devices are among them. """ - testpmd_driver = self.sut_node.create_interactive_shell(TestPmdShell, privileged=True) + testpmd_driver = TestPmdShell(self.sut_node) dev_list = [str(x) for x in testpmd_driver.get_devices()] for nic in self.nics_in_node: self.verify( From patchwork Thu May 9 11:20:57 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Luca Vizzarro X-Patchwork-Id: 140006 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 9386F43F7C; Thu, 9 May 2024 13:22:12 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 47BFD40A89; Thu, 9 May 2024 13:21:24 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id 08804402F1 for ; Thu, 9 May 2024 13:21:17 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id 1BEF4106F; Thu, 9 May 2024 04:21:42 -0700 (PDT) Received: from localhost.localdomain (unknown [10.1.194.74]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id E7E933F6A8; Thu, 9 May 2024 04:21:15 -0700 (PDT) From: Luca Vizzarro To: dev@dpdk.org Cc: =?utf-8?q?Juraj_Linke=C5=A1?= , Jeremy Spewock , Luca Vizzarro , Paul Szczepanek Subject: [PATCH v2 8/8] dts: use Unpack for type checking and hinting Date: Thu, 9 May 2024 12:20:57 +0100 Message-Id: <20240509112057.1167947-9-luca.vizzarro@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240509112057.1167947-1-luca.vizzarro@arm.com> References: <20240326190422.577028-1-luca.vizzarro@arm.com> <20240509112057.1167947-1-luca.vizzarro@arm.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Interactive shells that inherit DPDKShell initialise their params classes from a kwargs dict. Therefore, static type checking is disabled. This change uses the functionality of Unpack added in PEP 692 to re-enable it. The disadvantage is that this functionality has been implemented only with TypedDict, forcing the creation of TypedDict mirrors of the Params classes. Signed-off-by: Luca Vizzarro Reviewed-by: Paul Szczepanek Tested-by: Nicholas Pratte Reviewed-by: Nicholas Pratte Reviewed-by: Jeremy Spewock --- dts/framework/params/types.py | 133 ++++++++++++++++++ dts/framework/remote_session/testpmd_shell.py | 5 +- 2 files changed, 137 insertions(+), 1 deletion(-) create mode 100644 dts/framework/params/types.py diff --git a/dts/framework/params/types.py b/dts/framework/params/types.py new file mode 100644 index 0000000000..e668f658d8 --- /dev/null +++ b/dts/framework/params/types.py @@ -0,0 +1,133 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2024 Arm Limited + +"""Module containing TypeDict-equivalents of Params classes for static typing and hinting. + +TypedDicts can be used in conjunction with Unpack and kwargs for type hinting on function calls. + +Example: + ..code:: python + def create_testpmd(**kwargs: Unpack[TestPmdParamsDict]): + params = TestPmdParams(**kwargs) +""" + +from pathlib import PurePath +from typing import TypedDict + +from framework.params import Switch, YesNoSwitch +from framework.params.testpmd import ( + AnonMempoolAllocationMode, + EthPeer, + Event, + FlowGenForwardingMode, + HairpinMode, + NoisyForwardingMode, + Params, + PortNUMAConfig, + PortTopology, + RingNUMAConfig, + RSSSetting, + RXMultiQueueMode, + RXRingParams, + SimpleForwardingModes, + SimpleMempoolAllocationMode, + TxIPAddrPair, + TXOnlyForwardingMode, + TXRingParams, + TxUDPPortPair, +) +from framework.testbed_model.cpu import LogicalCoreList +from framework.testbed_model.port import Port +from framework.testbed_model.virtual_device import VirtualDevice + + +class EalParamsDict(TypedDict, total=False): + """:class:`TypedDict` equivalent of :class:`~.eal.EalParams`.""" + + lcore_list: LogicalCoreList | None + memory_channels: int | None + prefix: str + no_pci: Switch + vdevs: list[VirtualDevice] | None + ports: list[Port] | None + other_eal_param: Params | None + + +class TestPmdParamsDict(EalParamsDict, total=False): + """:class:`TypedDict` equivalent of :class:`~.testpmd.TestPmdParams`.""" + + interactive_mode: Switch + auto_start: Switch + tx_first: Switch + stats_period: int | None + display_xstats: list[str] | None + nb_cores: int | None + coremask: int | None + nb_ports: int | None + port_topology: PortTopology | None + portmask: int | None + portlist: str | None + numa: YesNoSwitch + socket_num: int | None + port_numa_config: list[PortNUMAConfig] | None + ring_numa_config: list[RingNUMAConfig] | None + total_num_mbufs: int | None + mbuf_size: list[int] | None + mbcache: int | None + max_pkt_len: int | None + eth_peers_configfile: PurePath | None + eth_peer: list[EthPeer] | None + tx_ip: TxIPAddrPair | None + tx_udp: TxUDPPortPair | None + enable_lro: Switch + max_lro_pkt_size: int | None + disable_crc_strip: Switch + enable_scatter: Switch + enable_hw_vlan: Switch + enable_hw_vlan_filter: Switch + enable_hw_vlan_strip: Switch + enable_hw_vlan_extend: Switch + enable_hw_qinq_strip: Switch + pkt_drop_enabled: Switch + rss: RSSSetting | None + forward_mode: ( + SimpleForwardingModes + | FlowGenForwardingMode + | TXOnlyForwardingMode + | NoisyForwardingMode + | None + ) + hairpin_mode: HairpinMode | None + hairpin_queues: int | None + burst: int | None + enable_rx_cksum: Switch + rx_queues: int | None + rx_ring: RXRingParams | None + no_flush_rx: Switch + rx_segments_offsets: list[int] | None + rx_segments_length: list[int] | None + multi_rx_mempool: Switch + rx_shared_queue: Switch | int + rx_offloads: int | None + rx_mq_mode: RXMultiQueueMode | None + tx_queues: int | None + tx_ring: TXRingParams | None + tx_offloads: int | None + eth_link_speed: int | None + disable_link_check: Switch + disable_device_start: Switch + no_lsc_interrupt: Switch + no_rmv_interrupt: Switch + bitrate_stats: int | None + latencystats: int | None + print_events: list[Event] | None + mask_events: list[Event] | None + flow_isolate_all: Switch + disable_flow_flush: Switch + hot_plug: Switch + vxlan_gpe_port: int | None + geneve_parsed_port: int | None + lock_all_memory: YesNoSwitch + mempool_allocation_mode: SimpleMempoolAllocationMode | AnonMempoolAllocationMode | None + record_core_cycles: Switch + record_burst_status: Switch diff --git a/dts/framework/remote_session/testpmd_shell.py b/dts/framework/remote_session/testpmd_shell.py index 92930d7fbb..5b3a7bb9ab 100644 --- a/dts/framework/remote_session/testpmd_shell.py +++ b/dts/framework/remote_session/testpmd_shell.py @@ -18,8 +18,11 @@ from pathlib import PurePath from typing import ClassVar +from typing_extensions import Unpack + from framework.exception import InteractiveCommandExecutionError from framework.params.testpmd import SimpleForwardingModes, TestPmdParams +from framework.params.types import TestPmdParamsDict from framework.remote_session.dpdk_shell import DPDKShell from framework.settings import SETTINGS from framework.testbed_model.cpu import LogicalCoreCount, LogicalCoreList @@ -76,7 +79,7 @@ def __init__( ascending_cores: bool = True, append_prefix_timestamp: bool = True, start_on_init: bool = True, - **app_params, + **app_params: Unpack[TestPmdParamsDict], ) -> None: """Overrides :meth:`~.dpdk_shell.DPDKShell.__init__`. Changes app_params to kwargs.""" super().__init__(