@@ -5,12 +5,14 @@
test_runs:
# define one test run environment
- dpdk_build:
- arch: x86_64
- os: linux
- cpu: native
- # the combination of the following two makes CC="ccache gcc"
- compiler: gcc
- compiler_wrapper: ccache
+ tarball: "" # define path to DPDK tarball
+ build:
+ arch: x86_64
+ os: linux
+ cpu: native
+ # the combination of the following two makes CC="ccache gcc"
+ compiler: gcc
+ compiler_wrapper: ccache
perf: false # disable performance testing
func: true # enable functional testing
skip_smoke_tests: false # optional
@@ -47,6 +47,7 @@
from framework.config.types import (
ConfigurationDict,
DPDKBuildConfigDict,
+ DPDKSetupDict,
NodeConfigDict,
PortConfigDict,
TestRunConfigDict,
@@ -380,6 +381,67 @@ def from_dict(cls, d: DPDKBuildConfigDict) -> Self:
)
+@dataclass(slots=True, frozen=True)
+class DPDKLocation:
+ """DPDK location.
+
+ The path to the DPDK sources, build dir and type of location.
+
+ Attributes:
+ dpdk_tree: The path to the DPDK tree.
+ tarball: The path to the DPDK tarball.
+ remote: If :data:`True`, `dpdk_tree` or `tarball` is on the SUT node.
+ build_dir: A directory name, which would be located in the `dpdk tree` or `tarball`.
+ """
+
+ dpdk_tree: str | None
+ tarball: str | None
+ remote: bool
+ build_dir: str | None
+
+ @classmethod
+ def from_dict(cls, d: DPDKSetupDict) -> Self | None:
+ """A convenience method that processes and validate the inputs before creating an instance.
+
+ Ensures that either `dpdk_tree` or `tarball` is provided and, if local
+ (`remote` is False), verifies their existence. Constructs and returns
+ a `DPDKLocation` object with the provided parameters if validation is
+ successful, or `None` if neither `dpdk_tree` nor `tarball` is given.
+
+ Args:
+ d: The configuration dictionary.
+
+ Returns:
+ A DPDK location if construction is successful, otherwise None.
+
+ Raises:
+ ConfigurationError: If `dpdk_tree` or `tarball` not found in local filesystem.
+ """
+ dpdk_tree = d.get("dpdk_tree")
+ tarball = d.get("tarball")
+ remote = d.get("remote", False)
+
+ if dpdk_tree or tarball:
+ if not remote:
+ if dpdk_tree and not Path(dpdk_tree).is_dir():
+ raise ConfigurationError(
+ f"DPDK tree '{dpdk_tree}' not found in local filesystem."
+ )
+ if tarball and not Path(tarball).is_file():
+ raise ConfigurationError(
+ f"DPDK tarball '{tarball}' not found in local filesystem."
+ )
+
+ return cls(
+ dpdk_tree=dpdk_tree,
+ tarball=tarball,
+ remote=remote,
+ build_dir=d.get("dir_name"),
+ )
+
+ return None
+
+
@dataclass(slots=True, frozen=True)
class DPDKBuildInfo:
"""Various versions and other information about a DPDK build.
@@ -389,8 +451,8 @@ class DPDKBuildInfo:
compiler_version: The version of the compiler used to build DPDK.
"""
- dpdk_version: str
- compiler_version: str
+ dpdk_version: str | None
+ compiler_version: str | None
@dataclass(slots=True, frozen=True)
@@ -437,7 +499,8 @@ class TestRunConfiguration:
and with what DPDK build.
Attributes:
- dpdk_build: A DPDK build to test.
+ dpdk_location: The target source of the DPDK tree.
+ dpdk_build_config: A DPDK build configuration to test.
perf: Whether to run performance tests.
func: Whether to run functional tests.
skip_smoke_tests: Whether to skip smoke tests.
@@ -447,7 +510,8 @@ class TestRunConfiguration:
vdevs: The names of virtual devices to test.
"""
- dpdk_build: DPDKBuildConfiguration
+ dpdk_location: DPDKLocation | None
+ dpdk_build_config: DPDKBuildConfiguration | None
perf: bool
func: bool
skip_smoke_tests: bool
@@ -475,6 +539,18 @@ def from_dict(
Returns:
The test run configuration instance.
"""
+ dpdk_location = None
+ dpdk_build_config = None
+
+ dpdk_build_dct = d.get("dpdk_build")
+ if dpdk_build_dct:
+ dpdk_location = DPDKLocation.from_dict(dpdk_build_dct)
+ dpdk_build_config = (
+ DPDKBuildConfiguration.from_dict(dpdk_build_dct["build"])
+ if dpdk_build_dct.get("build")
+ else None
+ )
+
test_suites: list[TestSuiteConfig] = list(map(TestSuiteConfig.from_dict, d["test_suites"]))
sut_name = d["system_under_test_node"]["node_name"]
skip_smoke_tests = d.get("skip_smoke_tests", False)
@@ -495,7 +571,8 @@ def from_dict(
d["system_under_test_node"]["vdevs"] if "vdevs" in d["system_under_test_node"] else []
)
return cls(
- dpdk_build=DPDKBuildConfiguration.from_dict(d["dpdk_build"]),
+ dpdk_location=dpdk_location,
+ dpdk_build_config=dpdk_build_config,
perf=d["perf"],
func=d["func"],
skip_smoke_tests=skip_smoke_tests,
@@ -110,9 +110,9 @@
"mscv"
]
},
- "dpdk_build": {
+ "build": {
"type": "object",
- "description": "DPDK build configuration supported by DTS.",
+ "description": "DPDK build configuration supported by DTS. Either this or `dir_name` can be defined, but not both.",
"properties": {
"arch": {
"type": "string",
@@ -146,6 +146,43 @@
"compiler"
]
},
+ "dpdk_build": {
+ "type":"object",
+ "description": "DPDK source and build configuration. Optional.",
+ "properties": {
+ "dpdk_tree": {
+ "type": "string",
+ "description": "Path to the DPDK source code. Either this or `tarball` can be defined, but not both."
+ },
+ "tarball": {
+ "type": "string",
+ "description": "Path to the DPDK tarball. Either this or `dpdk_tree` can be defined, but not both."
+ },
+ "remote": {
+ "type": "boolean",
+ "description": "If present, `dpdk_tree` or `tarball` is on the SUT node."
+ },
+ "dir_name": {
+ "type": "string",
+ "description": "A directory name, which would be located in the `dpdk tree` or `tarball`. Either this or `build` can be defined, but not both."
+ },
+ "build": {
+ "$ref": "#/definitions/build"
+ }
+ },
+ "allOf": [
+ {
+ "not": {
+ "required": ["dpdk_tree", "tarball"]
+ }
+ },
+ {
+ "not": {
+ "required": ["dir_name", "build"]
+ }
+ }
+ ]
+ },
"hugepages_2mb": {
"type": "object",
"description": "Optional hugepage configuration. If not specified, hugepages won't be configured and DTS will use system configuration.",
@@ -86,6 +86,21 @@ class DPDKBuildConfigDict(TypedDict):
compiler_wrapper: str
+class DPDKSetupDict(TypedDict):
+ """Allowed keys and values."""
+
+ #:
+ dpdk_tree: str | None
+ #:
+ tarball: str | None
+ #:
+ remote: bool
+ #:
+ dir_name: str | None
+ #:
+ build: DPDKBuildConfigDict
+
+
class TestSuiteConfigDict(TypedDict):
"""Allowed keys and values."""
@@ -108,7 +123,7 @@ class TestRunConfigDict(TypedDict):
"""Allowed keys and values."""
#:
- dpdk_build: DPDKBuildConfigDict
+ dpdk_build: DPDKSetupDict
#:
perf: bool
#:
@@ -184,8 +184,8 @@ class InteractiveCommandExecutionError(DTSError):
severity: ClassVar[ErrorSeverity] = ErrorSeverity.REMOTE_CMD_EXEC_ERR
-class RemoteDirectoryExistsError(DTSError):
- """A directory that exists on a remote node."""
+class RemoteFileNotFoundError(DTSError):
+ """A remote file or directory is requested but doesn’t exist."""
#:
severity: ClassVar[ErrorSeverity] = ErrorSeverity.REMOTE_CMD_EXEC_ERR
@@ -104,4 +104,4 @@ def _update_real_path(self, path: PurePath) -> None:
Adds the remote DPDK build directory to the path.
"""
- super()._update_real_path(self._node.remote_dpdk_build_dir.joinpath(path))
+ super()._update_real_path(PurePath(self._node.remote_dpdk_build_dir).joinpath(path))
@@ -412,15 +412,27 @@ def _run_test_run(
test_run_config: A test run configuration.
test_run_result: The test run's result.
test_suites_with_cases: The test suites with test cases to run.
+
+ Raises:
+ ConfigurationError: If the DPDK sources or build is not set up from config or settings.
"""
self._logger.info(
f"Running test run with SUT '{test_run_config.system_under_test_node.name}'."
)
test_run_result.add_sut_info(sut_node.node_info)
try:
- sut_node.set_up_test_run(test_run_config)
+ dpdk_location = SETTINGS.dpdk_location or test_run_config.dpdk_location
+ if not dpdk_location:
+ raise ConfigurationError("DPDK sources is not set up from config or settings.")
+ elif not (dpdk_location.build_dir or test_run_config.dpdk_build_config):
+ raise ConfigurationError(
+ "Either DPDK build config is not set up from config or DPDK build dir is not "
+ "sets up from config or settings."
+ )
+
+ sut_node.set_up_test_run(test_run_config, dpdk_location)
test_run_result.add_dpdk_build_info(sut_node.get_dpdk_build_info())
- tg_node.set_up_test_run(test_run_config)
+ tg_node.set_up_test_run(test_run_config, dpdk_location)
test_run_result.update_setup(Result.PASS)
except Exception as e:
self._logger.exception("Test run setup failed.")
@@ -39,10 +39,10 @@
Set to any value to enable logging everything to the console.
-.. option:: -s, --skip-setup
-.. envvar:: DTS_SKIP_SETUP
+.. option:: --dpdk-tree
+.. envvar:: DTS_DPDK_TREE
- Set to any value to skip building DPDK.
+ Path to DPDK source code tree to test.
.. option:: --tarball, --snapshot
.. envvar:: DTS_DPDK_TARBALL
@@ -55,10 +55,20 @@
Git revision ID to test. Could be commit, tag, tree ID etc.
To test local changes, first commit them, then use their commit ID.
+.. option:: --remote-source
+.. envvar:: DTS_REMOTE_SOURCE
+
+ Set when the DPDK source tree or tarball is located on the SUT node.
+
+.. option:: --build-dir
+.. envvar:: DTS_BUILD_DIR
+
+ A directory name, which would be located in the `dpdk tree` or `tarball`.
+
.. option:: -f, --force
.. envvar:: DTS_FORCE
- Specify to remove an already existing dpdk tarball before copying/extracting a new one.
+ Specify to remove an already existing DPDK tarball or tree before copying/extracting a new one.
.. option:: --test-suite
.. envvar:: DTS_TEST_SUITES
@@ -90,7 +100,7 @@
from pathlib import Path
from typing import Callable
-from .config import TestSuiteConfig
+from .config import DPDKLocation, TestSuiteConfig
from .exception import ConfigurationError
from .utils import DPDKGitTarball, get_commit_id
@@ -111,9 +121,7 @@ class Settings:
#:
verbose: bool = False
#:
- skip_setup: bool = False
- #:
- dpdk_tarball_path: Path | str = ""
+ dpdk_location: DPDKLocation | None = None
#:
force: bool = False
#:
@@ -241,14 +249,6 @@ def _get_help_string(self, action):
return help
-def _parse_tarball_path(file_path: str) -> Path:
- """Validate whether `file_path` is valid and return a Path object."""
- path = Path(file_path)
- if not path.exists() or not path.is_file():
- raise argparse.ArgumentTypeError("The file path provided is not a valid file")
- return path
-
-
def _parse_revision_id(rev_id: str) -> str:
"""Validate revision ID and retrieve corresponding commit ID."""
try:
@@ -257,6 +257,48 @@ def _parse_revision_id(rev_id: str) -> str:
raise argparse.ArgumentTypeError("The Git revision ID supplied is invalid or ambiguous")
+def _required_with_one_of(parser: _DTSArgumentParser, action: Action, *required_dests: str) -> None:
+ """Verify that `action` is listed together with `required_dests`.
+
+ Verify that a specific action is included in the command-line arguments or environment variables
+ if at least one of the required destination is already defined in the command-line arguments or
+ environment variables.
+
+ Args:
+ parser: The custom ArgumentParser object which contains `action`.
+ action: The action to be verified.
+ *required_dests: Destination variable names of the required arguments.
+
+ Raises:
+ argparse.ArgumentTypeError: If the action is not included when one
+ of the required arguments is present.
+
+ Example:
+ For etc. if the `--option1` argument is provided, then the `--option2` argument
+ must also be included too. Only one of the required_dests needs to be provided for
+ the check to be applied.
+
+ parser = _DTSArgumentParser()
+ option1_arg = parser.add_argument('--option1', dest='option1', action='store_true')
+ option2_arg = arser.add_argument('--option2', dest='option2', action='store_true')
+
+ _required_with_one_of(parser, option1_arg, 'option2')
+
+ """
+ if _is_action_in_args(action):
+ for required_dest in required_dests:
+ required_action = parser.find_action(required_dest)
+ if required_action is None:
+ continue
+
+ if _is_action_in_args(required_action):
+ return None
+
+ raise argparse.ArgumentTypeError(
+ f"The '{action.dest}' is required at least with one of '{', '.join(required_dests)}'."
+ )
+
+
def _get_parser() -> _DTSArgumentParser:
"""Create the argument parser for DTS.
@@ -311,21 +353,19 @@ def _get_parser() -> _DTSArgumentParser:
)
_add_env_var_to_action(action)
- action = parser.add_argument(
- "-s",
- "--skip-setup",
- action="store_true",
- default=SETTINGS.skip_setup,
- help="Specify to skip all setup steps on SUT and TG nodes.",
- )
- _add_env_var_to_action(action)
+ dpdk_source = parser.add_mutually_exclusive_group()
- dpdk_source = parser.add_mutually_exclusive_group(required=True)
+ action = dpdk_source.add_argument(
+ "--dpdk-tree",
+ help="Path to DPDK source code tree to test.",
+ metavar="DIR_PATH",
+ dest="dpdk_tree_path",
+ )
+ _add_env_var_to_action(action, "DPDK_TREE")
action = dpdk_source.add_argument(
"--tarball",
"--snapshot",
- type=_parse_tarball_path,
help="Path to DPDK source code tarball to test.",
metavar="FILE_PATH",
dest="dpdk_tarball_path",
@@ -344,6 +384,23 @@ def _get_parser() -> _DTSArgumentParser:
)
_add_env_var_to_action(action)
+ action = parser.add_argument(
+ "--remote-source",
+ action="store_true",
+ default=False,
+ help="Set when the DPDK source tree or tarball is located on the SUT node.",
+ )
+ _add_env_var_to_action(action)
+ _required_with_one_of(parser, action, "dpdk_tarball_path", "dpdk_tree_path")
+
+ action = parser.add_argument(
+ "--build-dir",
+ help="A directory name, which would be located in the `dpdk tree` or `tarball`.",
+ metavar="DIR_NAME",
+ )
+ _add_env_var_to_action(action)
+ _required_with_one_of(parser, action, "dpdk_tarball_path", "dpdk_tree_path")
+
action = parser.add_argument(
"-f",
"--force",
@@ -395,6 +452,49 @@ def _get_parser() -> _DTSArgumentParser:
return parser
+def _process_dpdk_location(
+ dpdk_tree: str | None,
+ tarball: str | None,
+ remote: bool,
+ build_dir: str | None,
+):
+ """Process and validate DPDK build arguments.
+
+ Ensures that either `dpdk_tree` or `tarball` is provided and, if local
+ (`remote` is False), verifies their existence. Constructs and returns
+ a `DPDKLocation` object with the provided parameters if validation is
+ successful, or `None` if neither `dpdk_tree` nor `tarball` is given.
+
+ Args:
+ dpdk_tree: The path to the DPDK tree.
+ tarball: The path to the DPDK tarball.
+ remote: If :data:`True`, `dpdk_tree` or `tarball` is on the SUT node.
+ build_dir: A directory name, which would be located in the `dpdk tree` or `tarball`.
+
+ Returns:
+ A DPDK location if construction is successful, otherwise None.
+
+ Raises:
+ argparse.ArgumentTypeError: If `dpdk_tree` or `tarball` not found in local filesystem.
+ """
+ if dpdk_tree or tarball:
+ if not remote:
+ if dpdk_tree and not Path(dpdk_tree).is_dir():
+ raise argparse.ArgumentTypeError(
+ f"DPDK tree '{dpdk_tree}' not found in local filesystem."
+ )
+ if tarball and not Path(tarball).is_file():
+ raise argparse.ArgumentTypeError(
+ f"DPDK tarball '{tarball}' not found in local filesystem."
+ )
+
+ return DPDKLocation(
+ dpdk_tree=dpdk_tree, tarball=tarball, remote=remote, build_dir=build_dir
+ )
+
+ return None
+
+
def _process_test_suites(
parser: _DTSArgumentParser, args: list[list[str]]
) -> list[TestSuiteConfig]:
@@ -424,16 +524,14 @@ def get_settings() -> Settings:
The new settings object.
"""
parser = _get_parser()
-
- if len(sys.argv) == 1:
- parser.print_help()
- sys.exit(1)
-
args = parser.parse_args()
if args.dpdk_revision_id:
args.dpdk_tarball_path = Path(DPDKGitTarball(args.dpdk_revision_id, args.output_dir))
+ args.dpdk_location = _process_dpdk_location(
+ args.dpdk_tree_path, args.dpdk_tarball_path, args.remote_source, args.build_dir
+ )
args.test_suites = _process_test_suites(parser, args.test_suites)
kwargs = {k: v for k, v in vars(args).items() if hasattr(SETTINGS, k)}
@@ -29,16 +29,7 @@
from types import FunctionType
from typing import Union
-from .config import (
- OS,
- Architecture,
- Compiler,
- CPUType,
- DPDKBuildInfo,
- NodeInfo,
- TestRunConfiguration,
- TestSuiteConfig,
-)
+from .config import DPDKBuildInfo, NodeInfo, TestRunConfiguration, TestSuiteConfig
from .exception import DTSError, ErrorSeverity
from .logger import DTSLogger
from .settings import SETTINGS
@@ -220,8 +211,8 @@ def add_stats(self, statistics: "Statistics") -> None:
class DTSResult(BaseResult):
"""Stores environment information and test results from a DTS run.
- * Test run level information, such as testbed, compiler, target OS and cpu and
- the test suite list,
+ * Test run level information, such as testbed, compiler version, dpdk version
+ and the test suite list,
* Test suite and test case results,
* All errors that are caught and recorded during DTS execution.
@@ -318,10 +309,6 @@ class TestRunResult(BaseResult):
The internal list stores the results of all test suites in a given test run.
Attributes:
- arch: The DPDK build architecture.
- os: The DPDK build operating system.
- cpu: The DPDK build CPU.
- compiler: The DPDK build compiler.
compiler_version: The DPDK build compiler version.
dpdk_version: The built DPDK version.
sut_os_name: The operating system of the SUT node.
@@ -329,10 +316,6 @@ class TestRunResult(BaseResult):
sut_kernel_version: The operating system kernel version of the SUT node.
"""
- arch: Architecture
- os: OS
- cpu: CPUType
- compiler: Compiler
compiler_version: str | None
dpdk_version: str | None
sut_os_name: str
@@ -348,10 +331,6 @@ def __init__(self, test_run_config: TestRunConfiguration):
test_run_config: A test run configuration.
"""
super().__init__()
- self.arch = test_run_config.dpdk_build.arch
- self.os = test_run_config.dpdk_build.os
- self.cpu = test_run_config.dpdk_build.cpu
- self.compiler = test_run_config.dpdk_build.compiler
self.compiler_version = None
self.dpdk_version = None
self._config = test_run_config
@@ -15,12 +15,11 @@
from abc import ABC
from ipaddress import IPv4Interface, IPv6Interface
-from typing import Any, Callable, Union
+from typing import Union
-from framework.config import OS, NodeConfiguration, TestRunConfiguration
+from framework.config import OS, DPDKLocation, NodeConfiguration, TestRunConfiguration
from framework.exception import ConfigurationError
from framework.logger import DTSLogger, get_dts_logger
-from framework.settings import SETTINGS
from .cpu import (
LogicalCore,
@@ -95,7 +94,9 @@ def _init_ports(self) -> None:
for port in self.ports:
self.configure_port_state(port)
- def set_up_test_run(self, test_run_config: TestRunConfiguration) -> None:
+ def set_up_test_run(
+ self, test_run_config: TestRunConfiguration, dpdk_location: DPDKLocation
+ ) -> None:
"""Test run setup steps.
Configure hugepages on all DTS node types. Additional steps can be added by
@@ -104,6 +105,7 @@ def set_up_test_run(self, test_run_config: TestRunConfiguration) -> None:
Args:
test_run_config: A test run configuration according to which
the setup steps will be taken.
+ dpdk_location: The target source of the DPDK tree.
"""
self._setup_hugepages()
@@ -216,18 +218,6 @@ def close(self) -> None:
for session in self._other_sessions:
session.close()
- @staticmethod
- def skip_setup(func: Callable[..., Any]) -> Callable[..., Any]:
- """Skip the decorated function.
-
- The :option:`--skip-setup` command line argument and the :envvar:`DTS_SKIP_SETUP`
- environment variable enable the decorator.
- """
- if SETTINGS.skip_setup:
- return lambda *args: None
- else:
- return func
-
def create_session(node_config: NodeConfiguration, name: str, logger: DTSLogger) -> OSSession:
"""Factory for OS-aware sessions.
@@ -25,7 +25,7 @@
from abc import ABC, abstractmethod
from collections.abc import Iterable
from ipaddress import IPv4Interface, IPv6Interface
-from pathlib import Path, PurePath
+from pathlib import Path, PurePath, PurePosixPath
from typing import Union
from framework.config import Architecture, NodeConfiguration, NodeInfo
@@ -137,17 +137,6 @@ def _get_privileged_command(command: str) -> str:
The modified command that executes with administrative privileges.
"""
- @abstractmethod
- def guess_dpdk_remote_dir(self, remote_dir: str | PurePath) -> PurePath:
- """Try to find DPDK directory in `remote_dir`.
-
- The directory is the one which is created after the extraction of the tarball. The files
- are usually extracted into a directory starting with ``dpdk-``.
-
- Returns:
- The absolute path of the DPDK remote directory, empty path if not found.
- """
-
@abstractmethod
def get_remote_tmp_dir(self) -> PurePath:
"""Get the path of the temporary directory of the remote OS.
@@ -177,6 +166,17 @@ def join_remote_path(self, *args: str | PurePath) -> PurePath:
The resulting joined path.
"""
+ @abstractmethod
+ def remote_path_exists(self, remote_path: str | PurePath) -> bool:
+ """Check whether a path exists on the remote system.
+
+ Args:
+ remote_path: The path to check.
+
+ Returns:
+ True if the path exists, False otherwise.
+ """
+
@abstractmethod
def copy_from(
self, source_file: str | PurePath, destination_dir: str | Path, force: bool = SETTINGS.force
@@ -321,6 +321,25 @@ def extract_remote_tarball(
extracting to prevent overwriting data.
"""
+ @abstractmethod
+ def get_tarball_top_dir(
+ self, remote_tarball_path: str | PurePath
+ ) -> str | PurePosixPath | None:
+ """Get the top directory of the remote tarball.
+
+ It examines the contents of a tarball located at the given `remote_tarball_path` and
+ determines the top-level directory. If all files and directories in the tarball share
+ the same top-level directory, that directory name is returned. If the tarball contains
+ multiple top-level directories or is empty, the method return None.
+
+ Args:
+ remote_tarball_path: The path to the remote tarball.
+
+ Returns:
+ The top directory of the tarball, if there are not multiple top directories
+ otherwise None.
+ """
+
@abstractmethod
def build_dpdk(
self,
@@ -91,6 +91,11 @@ def join_remote_path(self, *args: str | PurePath) -> PurePosixPath:
"""Overrides :meth:`~.os_session.OSSession.join_remote_path`."""
return PurePosixPath(*args)
+ def remote_path_exists(self, remote_path: str | PurePath) -> bool:
+ """Overrides :meth:`~.os_session.OSSession.remote_path_exists`."""
+ result = self.send_command(f"test -e {remote_path}")
+ return not result.return_code
+
def copy_from(
self, source_file: str | PurePath, destination_dir: str | Path, force: bool = SETTINGS.force
) -> None:
@@ -216,6 +221,16 @@ def extract_remote_tarball(
if expected_dir:
self.send_command(f"ls {expected_dir}", verify=True)
+ def get_tarball_top_dir(
+ self, remote_tarball_path: str | PurePath
+ ) -> str | PurePosixPath | None:
+ """Overrides :meth:`~.os_session.OSSession.get_tarball_top_dir`."""
+ members = self.send_command(f"tar tf {remote_tarball_path}").stdout.split()
+ top_dirs = [PurePosixPath(member).parts[0] for member in members if member]
+ if len(set(top_dirs)) == 1:
+ return top_dirs[0]
+ return None
+
def build_dpdk(
self,
env_vars: dict,
@@ -321,7 +336,7 @@ def _get_dpdk_pids(self, dpdk_runtime_dirs: Iterable[str | PurePath]) -> list[in
pid_regex = r"p(\d+)"
for dpdk_runtime_dir in dpdk_runtime_dirs:
dpdk_config_file = PurePosixPath(dpdk_runtime_dir, "config")
- if self._remote_files_exists(dpdk_config_file):
+ if self.remote_path_exists(dpdk_config_file):
out = self.send_command(f"lsof -Fp {dpdk_config_file}").stdout
if out and "No such file or directory" not in out:
for out_line in out.splitlines():
@@ -330,10 +345,6 @@ def _get_dpdk_pids(self, dpdk_runtime_dirs: Iterable[str | PurePath]) -> list[in
pids.append(int(match.group(1)))
return pids
- def _remote_files_exists(self, remote_path: PurePath) -> bool:
- result = self.send_command(f"test -e {remote_path}")
- return not result.return_code
-
def _check_dpdk_hugepages(self, dpdk_runtime_dirs: Iterable[str | PurePath]) -> None:
"""Check there aren't any leftover hugepages.
@@ -345,7 +356,7 @@ def _check_dpdk_hugepages(self, dpdk_runtime_dirs: Iterable[str | PurePath]) ->
"""
for dpdk_runtime_dir in dpdk_runtime_dirs:
hugepage_info = PurePosixPath(dpdk_runtime_dir, "hugepage_info")
- if self._remote_files_exists(hugepage_info):
+ if self.remote_path_exists(hugepage_info):
out = self.send_command(f"lsof -Fp {hugepage_info}").stdout
if out and "No such file or directory" not in out:
self._logger.warning("Some DPDK processes did not free hugepages.")
@@ -13,20 +13,20 @@
import os
-import tarfile
import time
from pathlib import PurePath
from framework.config import (
DPDKBuildConfiguration,
DPDKBuildInfo,
+ DPDKLocation,
NodeInfo,
SutNodeConfiguration,
TestRunConfiguration,
)
+from framework.exception import RemoteFileNotFoundError
from framework.params.eal import EalParams
from framework.remote_session.remote_session import CommandResult
-from framework.settings import SETTINGS
from framework.utils import MesonArgs
from .node import Node
@@ -39,14 +39,27 @@ class SutNode(Node):
The SUT node extends :class:`Node` with DPDK specific features:
- * DPDK build,
+ * Managing DPDK source tree on the remote SUT,
+ * Building the DPDK from source or using a pre-built version,
* Gathering of DPDK build info,
* The running of DPDK apps, interactively or one-time execution,
* DPDK apps cleanup.
- The :option:`--tarball` command line argument and the :envvar:`DTS_DPDK_TARBALL`
- environment variable configure the path to the DPDK tarball
- or the git commit ID, tag ID or tree ID to test.
+ The :option:`--tarball` command line argument, :envvar:`DTS_DPDK_TARBALL` environment variable
+ and `tarball` inside `dpdk_build` from configuration, set the path to the DPDK tarball.
+
+ The :option:`--dpdk-tree` command line argument, :envvar:`DTS_DPDK_TREE` environment variable
+ and `dpdk-tree` inside `dpdk_build` from configuration, set the path to the DPDK tree.
+
+ The :option:`--remote-source` command line argument, :envvar:`DTS_REMOTE_SOURCE` environment
+ variable and `remote` inside `dpdk_build` from configuration, set when the `dpdk_tree` or
+ `tarball` is located on the SUT node.
+
+ The :option:`--build-dir` command line argument, :envvar:`DTS_BUILD_DIR` environment
+ variable and `dir_name` inside `dpdk_build` from configuration, set a directory name,
+ which would be located in the `dpdk_tree` or `tarball`.
+
+ Building DPDK from source uses `build` configuration inside `dpdk_build` of configuration.
Attributes:
config: The SUT node configuration.
@@ -57,10 +70,10 @@ class SutNode(Node):
virtual_devices: list[VirtualDevice]
dpdk_prefix_list: list[str]
dpdk_timestamp: str
- _dpdk_build_config: DPDKBuildConfiguration | None
_env_vars: dict
_remote_tmp_dir: PurePath
- __remote_dpdk_dir: PurePath | None
+ __remote_dpdk_tree_path: str | PurePath | None
+ _remote_dpdk_build_dir: PurePath | None
_app_compile_timeout: float
_dpdk_kill_session: OSSession | None
_dpdk_version: str | None
@@ -77,10 +90,10 @@ def __init__(self, node_config: SutNodeConfiguration):
super().__init__(node_config)
self.virtual_devices = []
self.dpdk_prefix_list = []
- self._dpdk_build_config = None
self._env_vars = {}
self._remote_tmp_dir = self.main_session.get_remote_tmp_dir()
- self.__remote_dpdk_dir = None
+ self.__remote_dpdk_tree_path = None
+ self._remote_dpdk_build_dir = None
self._app_compile_timeout = 90
self._dpdk_kill_session = None
self.dpdk_timestamp = (
@@ -93,40 +106,34 @@ def __init__(self, node_config: SutNodeConfiguration):
self._logger.info(f"Created node: {self.name}")
@property
- def _remote_dpdk_dir(self) -> PurePath:
- """The remote DPDK dir.
-
- This internal property should be set after extracting the DPDK tarball. If it's not set,
- that implies the DPDK setup step has been skipped, in which case we can guess where
- a previous build was located.
- """
- if self.__remote_dpdk_dir is None:
- self.__remote_dpdk_dir = self._guess_dpdk_remote_dir()
- return self.__remote_dpdk_dir
-
- @_remote_dpdk_dir.setter
- def _remote_dpdk_dir(self, value: PurePath) -> None:
- self.__remote_dpdk_dir = value
+ def _remote_dpdk_tree_path(self) -> str | PurePath:
+ """The remote DPDK tree path."""
+ if self.__remote_dpdk_tree_path:
+ return self.__remote_dpdk_tree_path
+
+ self._logger.warning(
+ "Failed to get remote dpdk tree path because we don't know the "
+ "location on the SUT node."
+ )
+ return ""
@property
- def remote_dpdk_build_dir(self) -> PurePath:
- """The remote DPDK build directory.
-
- This is the directory where DPDK was built.
- We assume it was built in a subdirectory of the extracted tarball.
- """
- if self._dpdk_build_config:
- return self.main_session.join_remote_path(
- self._remote_dpdk_dir, self._dpdk_build_config.name
- )
- else:
- return self.main_session.join_remote_path(self._remote_dpdk_dir, "build")
+ def remote_dpdk_build_dir(self) -> str | PurePath:
+ """The remote DPDK build dir path."""
+ if self._remote_dpdk_build_dir:
+ return self._remote_dpdk_build_dir
+
+ self._logger.warning(
+ "Failed to get remote dpdk build dir because we don't know the "
+ "location on the SUT node."
+ )
+ return ""
@property
- def dpdk_version(self) -> str:
+ def dpdk_version(self) -> str | None:
"""Last built DPDK version."""
if self._dpdk_version is None:
- self._dpdk_version = self.main_session.get_dpdk_version(self._remote_dpdk_dir)
+ self._dpdk_version = self.main_session.get_dpdk_version(self._remote_dpdk_tree_path)
return self._dpdk_version
@property
@@ -137,26 +144,25 @@ def node_info(self) -> NodeInfo:
return self._node_info
@property
- def compiler_version(self) -> str:
+ def compiler_version(self) -> str | None:
"""The node's compiler version."""
- if self._compiler_version is None:
- if self._dpdk_build_config is not None:
- self._compiler_version = self.main_session.get_compiler_version(
- self._dpdk_build_config.compiler.name
- )
- else:
- self._logger.warning(
- "Failed to get compiler version because _dpdk_build_config is None."
- )
- return ""
return self._compiler_version
+ @compiler_version.setter
+ def compiler_version(self, value: str) -> None:
+ """Set the compiler version used on the SUT.
+
+ Args:
+ value: The node's compiler version.
+ """
+ self._compiler_version = value
+
@property
- def path_to_devbind_script(self) -> PurePath:
+ def path_to_devbind_script(self) -> PurePath | str:
"""The path to the dpdk-devbind.py script on the node."""
if self._path_to_devbind_script is None:
self._path_to_devbind_script = self.main_session.join_remote_path(
- self._remote_dpdk_dir, "usertools", "dpdk-devbind.py"
+ self._remote_dpdk_tree_path, "usertools", "dpdk-devbind.py"
)
return self._path_to_devbind_script
@@ -168,101 +174,209 @@ def get_dpdk_build_info(self) -> DPDKBuildInfo:
"""
return DPDKBuildInfo(dpdk_version=self.dpdk_version, compiler_version=self.compiler_version)
- def _guess_dpdk_remote_dir(self) -> PurePath:
- return self.main_session.guess_dpdk_remote_dir(self._remote_tmp_dir)
+ def set_up_test_run(
+ self, test_run_config: TestRunConfiguration, dpdk_location: DPDKLocation
+ ) -> None:
+ """Extend the test run setup with vdev config and DPDK build set up.
- def set_up_test_run(self, test_run_config: TestRunConfiguration) -> None:
- """Extend the test run setup with vdev config.
+ This method extends the setup process by configuring virtual devices and preparing the DPDK
+ environment based on the provided configuration.
Args:
test_run_config: A test run configuration according to which
the setup steps will be taken.
+ dpdk_location: The target source of the DPDK tree.
"""
- super().set_up_test_run(test_run_config)
+ super().set_up_test_run(test_run_config, dpdk_location)
for vdev in test_run_config.vdevs:
self.virtual_devices.append(VirtualDevice(vdev))
- self._set_up_dpdk(test_run_config.dpdk_build)
+ self._set_up_dpdk(dpdk_location, test_run_config.dpdk_build_config)
def tear_down_test_run(self) -> None:
- """Extend the test run teardown with virtual device teardown."""
+ """Extend the test run teardown with virtual device teardown and DPDK teardown."""
super().tear_down_test_run()
self.virtual_devices = []
self._tear_down_dpdk()
- def _set_up_dpdk(self, dpdk_build_config: DPDKBuildConfiguration) -> None:
+ def _set_up_dpdk(
+ self, dpdk_location: DPDKLocation, dpdk_build_config: DPDKBuildConfiguration | None
+ ) -> None:
"""Set up DPDK the SUT node and bind ports.
- DPDK setup includes setting all internals needed for the build, the copying of DPDK tarball
- and then building DPDK. The drivers are bound to those that DPDK needs.
+ DPDK setup includes setting all internals needed for the build, the copying of DPDK
+ sources and then building DPDK or used the exist ones from the `dpdk_location`. The drivers
+ are bound to those that DPDK needs.
Args:
+ dpdk_location: The target source of the DPDK tree.
dpdk_build_config: The DPDK build test run configuration according to which
the setup steps will be taken.
"""
- self._configure_dpdk_build(dpdk_build_config)
- self._copy_dpdk_tarball()
- self._build_dpdk()
+ self._set_remote_dpdk_tree_path(dpdk_location)
+ if not self._remote_dpdk_tree_path:
+ if dpdk_location.dpdk_tree:
+ self._copy_dpdk_tree(dpdk_location.dpdk_tree)
+ elif dpdk_location.tarball:
+ self._prepare_and_extract_dpdk_tarball(dpdk_location.tarball, dpdk_location.remote)
+
+ self._set_remote_dpdk_build_dir(dpdk_location.build_dir)
+ if not self.remote_dpdk_build_dir and dpdk_build_config:
+ self._configure_dpdk_build(dpdk_build_config)
+ self._build_dpdk()
+
self.bind_ports_to_driver()
def _tear_down_dpdk(self) -> None:
"""Reset DPDK variables and bind port driver to the OS driver."""
self._env_vars = {}
- self._dpdk_build_config = None
- self.__remote_dpdk_dir = None
+ self.__remote_dpdk_tree_path = None
+ self._remote_dpdk_build_dir = None
self._dpdk_version = None
self._compiler_version = None
self.bind_ports_to_driver(for_dpdk=False)
+ def _set_remote_dpdk_tree_path(self, dpdk_location: DPDKLocation):
+ """Set the path to the remote DPDK source tree based on the provided DPDK location.
+
+ Verifies DPDK source tree existence on the SUT node and sets the `_remote_dpdk_tree_path`
+ property.
+
+ Args:
+ dpdk_location: The target source of the DPDK tree.
+
+ Raises:
+ RemoteFileNotFoundError: If the DPDK source tree is expected to be on the SUT node but
+ is not found.
+ """
+ if dpdk_location.remote and dpdk_location.dpdk_tree:
+ if self.main_session.remote_path_exists(dpdk_location.dpdk_tree):
+ self.__remote_dpdk_tree_path = PurePath(dpdk_location.dpdk_tree)
+ else:
+ raise RemoteFileNotFoundError(
+ f"Remote DPDK source tree '{dpdk_location.dpdk_tree}' not found in SUT node."
+ )
+
+ def _copy_dpdk_tree(self, dpdk_tree_path: str) -> None:
+ """Copy the DPDK source tree to the SUT.
+
+ Args:
+ dpdk_tree_path: The path to DPDK source tree on local filesystem.
+ """
+ self._logger.info(
+ f"Copying DPDK source tree to SUT: '{dpdk_tree_path}' into '{self._remote_tmp_dir}'."
+ )
+ self.main_session.copy_dir_to(dpdk_tree_path, self._remote_tmp_dir, exclude=".git")
+
+ self.__remote_dpdk_tree_path = self.main_session.join_remote_path(
+ self._remote_tmp_dir, PurePath(dpdk_tree_path).name
+ )
+
+ def _prepare_and_extract_dpdk_tarball(self, dpdk_tarball: str, remote: bool) -> None:
+ """Ensure the DPDK tarball is available on the SUT node and extract it.
+
+ This method ensures that the DPDK source tree tarball is available on the
+ SUT node. If the `dpdk_tarball` is local, it is copied to the SUT node. If the
+ `dpdk_tarball` is already on the SUT node, it verifies its existence.
+ The `dpdk_tarball` is then extracted on the SUT node.
+
+ This method sets the `_remote_dpdk_tree_path` property to the path of the
+ extracted DPDK tree on the SUT node.
+
+ Args:
+ dpdk_tarball: The path to the DPDK tarball, either locally or on the SUT node.
+ remote: Indicates whether the `dpdk_tarball` is already on the SUT node.
+
+ Raises:
+ RemoteFileNotFoundError: If the `dpdk_tarball` is expected to be on the SUT node but
+ is not found.
+ """
+ if remote:
+ if not self.main_session.remote_path_exists(dpdk_tarball):
+ raise RemoteFileNotFoundError(
+ f"Remote DPDK tarball '{dpdk_tarball}' not found in SUT."
+ )
+
+ remote_tarball_path = PurePath(dpdk_tarball)
+ else:
+ self._logger.info(
+ f"Copying DPDK tarball to SUT: '{dpdk_tarball}' into '{self._remote_tmp_dir}'."
+ )
+ self.main_session.copy_to(dpdk_tarball, self._remote_tmp_dir)
+
+ remote_tarball_path = self.main_session.join_remote_path(
+ self._remote_tmp_dir, PurePath(dpdk_tarball).name
+ )
+
+ tarball_top_dir = self.main_session.get_tarball_top_dir(remote_tarball_path)
+ self.__remote_dpdk_tree_path = self.main_session.join_remote_path(
+ PurePath(remote_tarball_path).parent,
+ tarball_top_dir or PurePath(remote_tarball_path).stem,
+ )
+
+ self._logger.info(
+ "Extracting DPDK tarball on SUT: "
+ f"'{remote_tarball_path}' into '{self._remote_dpdk_tree_path}'."
+ )
+ self.main_session.extract_remote_tarball(
+ remote_tarball_path,
+ self._remote_dpdk_tree_path,
+ )
+
+ def _set_remote_dpdk_build_dir(self, build_dir: str | None):
+ """Set the `remote_dpdk_build_dir` on the SUT.
+
+ Args:
+ build_dir: A directory name, which is located inside `_remote_dpdk_tree_path`.
+
+ Raises:
+ RemoteFileNotFoundError: If the `build_dir` does not exist on the SUT node.
+ """
+ if build_dir:
+ remote_dpdk_build_dir = self.main_session.join_remote_path(
+ self._remote_dpdk_tree_path, build_dir
+ )
+ if not self.main_session.remote_path_exists(remote_dpdk_build_dir):
+ raise RemoteFileNotFoundError(
+ f"Remote DPDK build dir '{remote_dpdk_build_dir}' not found in SUT node."
+ )
+
+ self._remote_dpdk_build_dir = PurePath(remote_dpdk_build_dir)
+
def _configure_dpdk_build(self, dpdk_build_config: DPDKBuildConfiguration) -> None:
- """Populate common environment variables and set DPDK build config."""
+ """Populate common environment variables and set the DPDK build related properties.
+
+ This method sets `compiler_version` for additional information and `remote_dpdk_build_dir`
+ from DPDK build config name.
+
+ Args:
+ dpdk_build_config: A DPDK build configuration to test.
+ """
self._env_vars = {}
- self._dpdk_build_config = dpdk_build_config
self._env_vars.update(self.main_session.get_dpdk_build_env_vars(dpdk_build_config.arch))
self._env_vars["CC"] = dpdk_build_config.compiler.name
if dpdk_build_config.compiler_wrapper:
- self._env_vars["CC"] = f"'{self._dpdk_build_config.compiler_wrapper} "
- f"{self._dpdk_build_config.compiler.name}'"
-
- @Node.skip_setup
- def _copy_dpdk_tarball(self) -> None:
- """Copy to and extract DPDK tarball on the SUT node."""
- self._logger.info("Copying DPDK tarball to SUT.")
- self.main_session.copy_to(SETTINGS.dpdk_tarball_path, self._remote_tmp_dir)
-
- # construct remote tarball path
- # the basename is the same on local host and on remote Node
- remote_tarball_path = self.main_session.join_remote_path(
- self._remote_tmp_dir, os.path.basename(SETTINGS.dpdk_tarball_path)
- )
+ self._env_vars[
+ "CC"
+ ] = f"'{dpdk_build_config.compiler_wrapper} {dpdk_build_config.compiler.name}'"
- # construct remote path after extracting
- with tarfile.open(SETTINGS.dpdk_tarball_path) as dpdk_tar:
- dpdk_top_dir = dpdk_tar.getnames()[0]
- self._remote_dpdk_dir = self.main_session.join_remote_path(
- self._remote_tmp_dir, dpdk_top_dir
+ self.compiler_version = self.main_session.get_compiler_version(
+ dpdk_build_config.compiler.name
)
- self._logger.info(
- f"Extracting DPDK tarball on SUT: "
- f"'{remote_tarball_path}' into '{self._remote_dpdk_dir}'."
+ self._remote_dpdk_build_dir = self.main_session.join_remote_path(
+ self._remote_dpdk_tree_path, dpdk_build_config.name
)
- # clean remote path where we're extracting
- self.main_session.remove_remote_dir(self._remote_dpdk_dir)
-
- # then extract to remote path
- self.main_session.extract_remote_tarball(remote_tarball_path, self._remote_dpdk_dir)
- @Node.skip_setup
def _build_dpdk(self) -> None:
"""Build DPDK.
- Uses the already configured target. Assumes that the tarball has
- already been copied to and extracted on the SUT node.
+ Uses the already configured DPDK build configuration. Assumes that the
+ `_remote_dpdk_tree_path` has already been sets on the SUT node.
"""
self.main_session.build_dpdk(
self._env_vars,
MesonArgs(default_library="static", enable_kmods=True, libdir="lib"),
- self._remote_dpdk_dir,
+ self._remote_dpdk_tree_path,
self.remote_dpdk_build_dir,
)
@@ -285,7 +399,7 @@ def build_dpdk_app(self, app_name: str, **meson_dpdk_args: str | bool) -> PurePa
self._env_vars,
MesonArgs(examples=app_name, **meson_dpdk_args), # type: ignore [arg-type]
# ^^ https://github.com/python/mypy/issues/11583
- self._remote_dpdk_dir,
+ self._remote_dpdk_tree_path,
self.remote_dpdk_build_dir,
rebuild=True,
timeout=self._app_compile_timeout,