@@ -2,6 +2,7 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright(c) 2022 University of New Hampshire
# Copyright(c) 2023 PANTHEON.tech s.r.o.
+# Copyright(c) 2024 Arm Limited
usage() {
echo "Usage: $(basename $0) [options] [directory]"
@@ -13,15 +14,19 @@ usage() {
format=true
lint=true
typecheck=true
+generate=true
# Comments after args serve as documentation; must be present
-while getopts "hflt" arg; do
+while getopts "hgflt" arg; do
case $arg in
h) # Display this message
- echo 'Run formatting and linting programs for DTS.'
+ echo 'Run generating, formatting and linting programs for DTS.'
usage
exit 0
;;
+ g) # Don't run code generator
+ generate=false
+ ;;
f) # Don't run formatters
format=false
;;
@@ -48,7 +53,22 @@ heading() {
errors=0
+if $generate; then
+ heading "Generating test suites to configuration mappings"
+ if command -v python3 > /dev/null; then
+ ../devtools/dts-generate-tests-mappings.py
+ errors=$((errors + $?))
+ else
+ echo "python3 not found, unable to run generator"
+ errros=$((errors + 1))
+ fi
+fi
+
if $format; then
+ if $generate; then
+ echo
+ fi
+
if command -v git > /dev/null; then
if git rev-parse --is-inside-work-tree >&-; then
heading "Formatting in $directory/"
@@ -85,7 +105,7 @@ if $format; then
fi
if $lint; then
- if $format; then
+ if $generate || $format; then
echo
fi
heading "Linting in $directory/"
@@ -99,7 +119,7 @@ if $lint; then
fi
if $typecheck; then
- if $format || $lint; then
+ if $generate || $format || $lint; then
echo
fi
heading "Checking types in $directory/"
new file mode 100755
@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2024 Arm Limited
+
+"""DTS Test Suites to Configuration mappings generation script."""
+
+import os
+import sys
+from collections import defaultdict
+from pathlib import Path
+from textwrap import indent
+from typing import Iterable
+
+DTS_DIR = Path(__file__).parent.joinpath("..", "dts").resolve()
+SCRIPT_FILE_NAME = Path(__file__).relative_to(Path(__file__).parent.parent)
+
+sys.path.append(str(DTS_DIR))
+
+from framework.config.test_suite import BaseTestSuitesConfigs, TestSuiteConfig
+from framework.exception import InternalError
+from framework.test_suite import AVAILABLE_TEST_SUITES, TestSuiteSpec
+
+FRAMEWORK_IMPORTS = [BaseTestSuitesConfigs, TestSuiteConfig]
+
+RELATIVE_PATH_TO_GENERATED_FILE = "framework/config/generated.py"
+SMOKE_TESTS_SUITE_NAME = "smoke_tests"
+CUSTOM_CONFIG_TYPES_VAR_NAME = "CUSTOM_CONFIG_TYPES"
+CUSTOM_CONFIG_TYPES_VAR_DOCSTRING = [
+ "#: Mapping of test suites to their corresponding custom configuration objects if any."
+]
+TEST_SUITES_CONFIG_CLASS_NAME = "TestSuitesConfigs"
+TEST_SUITES_CONFIG_CLASS_DOCSTRING = [
+ '"""Configuration mapping class to select and configure the test suites."""',
+]
+
+
+GENERATED_FILE_HEADER = f"""# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2024 The DPDK contributors
+# This file is automatically generated by {SCRIPT_FILE_NAME}.
+# Do NOT modify this file manually.
+
+\"\"\"Generated file containing the links between the test suites and the configuration.\"\"\"
+"""
+
+
+def join(lines: Iterable[str]) -> str:
+ """Join list of strings into text lines."""
+ return "\n".join(lines)
+
+
+def join_and_indent(lines: Iterable[str], indentation_level=1, indentation_spaces=4) -> str:
+ """Join list of strings into indented text lines."""
+ return "\n".join([indent(line, " " * indentation_level * indentation_spaces) for line in lines])
+
+
+def format_attributes_types(test_suite_spec: TestSuiteSpec):
+ """Format the config type into the respective configuration class field attribute type."""
+ config_type = test_suite_spec.config_obj.__name__
+ return f"Optional[{config_type}]"
+
+
+try:
+ framework_imports: dict[str, list[str]] = defaultdict(list)
+ for _import in FRAMEWORK_IMPORTS:
+ framework_imports[_import.__module__].append(_import.__name__)
+ formatted_framework_imports = sorted(
+ [
+ f"from {module} import {', '.join(sorted(imports))}"
+ for module, imports in framework_imports.items()
+ ]
+ )
+
+ test_suites = [
+ test_suite_spec
+ for test_suite_spec in AVAILABLE_TEST_SUITES
+ if test_suite_spec.name != SMOKE_TESTS_SUITE_NAME
+ ]
+
+ custom_configs = [t for t in test_suites if t.config_obj is not TestSuiteConfig]
+
+ custom_config_imports = [
+ f"from {t.config_obj.__module__} import {t.config_obj.__name__}" for t in custom_configs
+ ]
+
+ test_suites_attributes = [f"{t.name}: {format_attributes_types(t)} = None" for t in test_suites]
+
+ custom_config_mappings = [f'"{t.name}": {t.config_obj.__name__},' for t in custom_configs]
+
+ generated_file_contents = f"""{GENERATED_FILE_HEADER}
+from typing import Optional
+
+{join(formatted_framework_imports)}
+
+{join(custom_config_imports)}
+
+{join(CUSTOM_CONFIG_TYPES_VAR_DOCSTRING)}
+{CUSTOM_CONFIG_TYPES_VAR_NAME}: dict[str, type[{TestSuiteConfig.__name__}]] = {'{'}
+{join_and_indent(custom_config_mappings)}
+{'}'}
+
+
+class {TEST_SUITES_CONFIG_CLASS_NAME}({BaseTestSuitesConfigs.__name__}):
+{join_and_indent(TEST_SUITES_CONFIG_CLASS_DOCSTRING)}
+
+{join_and_indent(test_suites_attributes)}
+"""
+
+ path = os.path.join(DTS_DIR, RELATIVE_PATH_TO_GENERATED_FILE)
+
+ with open(path, "w") as generated_file:
+ generated_file.write(generated_file_contents)
+
+ print("Test suites to configuration mappings generated successfully!")
+except Exception as e:
+ raise InternalError(
+ "Failed to generate test suites to configuration mappings."
+ ) from e
new file mode 100644
@@ -0,0 +1,8 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+
+generated - Generated Test Suite Configurations
+===============================================
+
+.. automodule:: framework.config.generated
+ :members:
+ :show-inheritance:
@@ -6,3 +6,10 @@ config - Configuration Package
.. automodule:: framework.config
:members:
:show-inheritance:
+
+.. toctree::
+ :hidden:
+ :maxdepth: 1
+
+ framework.config.generated
+ framework.config.test_suite
new file mode 100644
@@ -0,0 +1,8 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+
+test_suite - Test Suite Configuration Definitions
+=================================================
+
+.. automodule:: framework.config.test_suite
+ :members:
+ :show-inheritance:
new file mode 100644
@@ -0,0 +1,9 @@
+.. SPDX-License-Identifier: BSD-3-Clause
+
+Test Suites Configurations
+==========================
+
+.. automodule:: tests.config
+ :members:
+ :show-inheritance:
+
@@ -11,6 +11,7 @@ tests - Test Suites Package
:hidden:
:maxdepth: 1
+ tests.config
tests.TestSuite_hello_world
tests.TestSuite_os_udp
tests.TestSuite_pmd_buffer_scatter
@@ -408,6 +408,29 @@ There are four types of methods that comprise a test suite:
should be implemented in the ``SutNode`` class (and the underlying classes that ``SutNode`` uses)
and used by the test suite via the ``sut_node`` field.
+The test suites can also implement their own custom configuration fields. This can be achieved by
+creating a new test suite config file which inherits from ``TestSuiteConfig`` defined in
+``dts/framework/config/test_suite.py``. So that this new custom configuration class is used, the
+test suite class must override the ``config`` attribute annotation with your new class, for example:
+
+.. code:: python
+
+ # place this under tests/config.py to avoid circular dependencies
+ class CustomConfig(TestSuiteConfig):
+ my_custom_field: int = 10
+
+ # place this under tests/TestSuite_my_new_test_suite.py
+ class TestMyNewTestSuite(TestSuite):
+ config: CustomConfig
+
+Finally, the test suites and the custom configuration files need to linked in the global configuration.
+This can be easily achieved by running the ``dts/generate-test-mappings.py``, e.g.:
+
+.. code-block:: console
+
+ $ poetry shell
+ (dts-py3.10) $ ./generate-test-mappings.py
+
.. _dts_dev_tools:
@@ -28,8 +28,8 @@ test_runs:
func: true # enable functional testing
skip_smoke_tests: false # optional
test_suites: # the following test suites will be run in their entirety
- - hello_world
- - os_udp
+ hello_world: all
+ os_udp: all
# The machine running the DPDK test executable
system_under_test_node:
node_name: "SUT 1"
@@ -32,11 +32,13 @@
and makes it thread safe should we ever want to move in that direction.
"""
+# pylama:ignore=W0611
+
import tarfile
from enum import Enum, auto, unique
from functools import cached_property
from pathlib import Path, PurePath
-from typing import TYPE_CHECKING, Annotated, Any, Literal, NamedTuple
+from typing import Annotated, Literal, NamedTuple
import yaml
from pydantic import (
@@ -52,8 +54,7 @@
from framework.exception import ConfigurationError
from framework.utils import REGEX_FOR_PCI_ADDRESS, StrEnum
-if TYPE_CHECKING:
- from framework.test_suite import TestSuiteSpec
+from .generated import TestSuitesConfigs
class FrozenModel(BaseModel):
@@ -382,69 +383,6 @@ class DPDKUncompiledBuildConfiguration(BaseDPDKBuildConfiguration):
DPDKBuildConfiguration = DPDKPrecompiledBuildConfiguration | DPDKUncompiledBuildConfiguration
-class TestSuiteConfig(FrozenModel):
- """Test suite configuration.
-
- Information about a single test suite to be executed. This can also be represented as a string
- instead of a mapping, example:
-
- .. code:: yaml
-
- test_runs:
- - test_suites:
- # As string representation:
- - hello_world # test all of `hello_world`, or
- - hello_world hello_world_single_core # test only `hello_world_single_core`
- # or as model fields:
- - test_suite: hello_world
- test_cases: [hello_world_single_core] # without this field all test cases are run
- """
-
- #: The name of the test suite module without the starting ``TestSuite_``.
- test_suite_name: str = Field(alias="test_suite")
- #: The names of test cases from this test suite to execute. If empty, all test cases will be
- #: executed.
- test_cases_names: list[str] = Field(default_factory=list, alias="test_cases")
-
- @cached_property
- def test_suite_spec(self) -> "TestSuiteSpec":
- """The specification of the requested test suite."""
- from framework.test_suite import find_by_name
-
- test_suite_spec = find_by_name(self.test_suite_name)
- assert (
- test_suite_spec is not None
- ), f"{self.test_suite_name} is not a valid test suite module name."
- return test_suite_spec
-
- @model_validator(mode="before")
- @classmethod
- def convert_from_string(cls, data: Any) -> Any:
- """Convert the string representation of the model into a valid mapping."""
- if isinstance(data, str):
- [test_suite, *test_cases] = data.split()
- return dict(test_suite=test_suite, test_cases=test_cases)
- return data
-
- @model_validator(mode="after")
- def validate_names(self) -> Self:
- """Validate the supplied test suite and test cases names.
-
- This validator relies on the cached property `test_suite_spec` to run for the first
- time in this call, therefore triggering the assertions if needed.
- """
- available_test_cases = map(
- lambda t: t.name, self.test_suite_spec.class_obj.get_test_cases()
- )
- for requested_test_case in self.test_cases_names:
- assert requested_test_case in available_test_cases, (
- f"{requested_test_case} is not a valid test case "
- f"of test suite {self.test_suite_name}."
- )
-
- return self
-
-
class TestRunSUTNodeConfiguration(FrozenModel):
"""The SUT node configuration of a test run."""
@@ -469,8 +407,8 @@ class TestRunConfiguration(FrozenModel):
func: bool
#: Whether to skip smoke tests.
skip_smoke_tests: bool = False
- #: The names of test suites and/or test cases to execute.
- test_suites: list[TestSuiteConfig] = Field(min_length=1)
+ #: The test suites to be selected and/or configured.
+ test_suites: TestSuitesConfigs
#: The SUT node configuration to use in this test run.
system_under_test_node: TestRunSUTNodeConfiguration
#: The TG node name to use in this test run.
@@ -602,6 +540,6 @@ def load_config(config_file_path: Path) -> Configuration:
config_data = yaml.safe_load(f)
try:
- return Configuration.model_validate(config_data)
+ return Configuration.model_validate(config_data, context={})
except ValidationError as e:
raise ConfigurationError("failed to load the supplied configuration") from e
new file mode 100644
@@ -0,0 +1,25 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2024 The DPDK contributors
+# This file is automatically generated by devtools/dts-generate-tests-mappings.py.
+# Do NOT modify this file manually.
+
+"""Generated file containing the links between the test suites and the configuration."""
+
+from typing import Optional
+
+from framework.config.test_suite import BaseTestSuitesConfigs, TestSuiteConfig
+from tests.config import HelloWorldConfig
+
+#: Mapping of test suites to their corresponding custom configuration objects if any.
+CUSTOM_CONFIG_TYPES: dict[str, type[TestSuiteConfig]] = {
+ "hello_world": HelloWorldConfig,
+}
+
+
+class TestSuitesConfigs(BaseTestSuitesConfigs):
+ """Configuration mapping class to select and configure the test suites."""
+
+ hello_world: Optional[HelloWorldConfig] = None
+ os_udp: Optional[TestSuiteConfig] = None
+ pmd_buffer_scatter: Optional[TestSuiteConfig] = None
+ vlan: Optional[TestSuiteConfig] = None
new file mode 100644
@@ -0,0 +1,154 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2024 Arm Limited
+
+"""Test suites configuration module.
+
+Test suites can inherit :class:`TestSuiteConfig` to create their own custom configuration.
+By doing so, the test suite class must also override the annotation of the field
+`~framework.test_suite.TestSuite.config` to use their custom configuration type.
+"""
+
+from functools import cached_property
+from typing import TYPE_CHECKING, Any, Iterable
+
+from pydantic import (
+ BaseModel,
+ ConfigDict,
+ Field,
+ ValidationInfo,
+ field_validator,
+ model_validator,
+)
+from typing_extensions import Self
+
+if TYPE_CHECKING:
+ from framework.test_suite import TestSuiteSpec
+
+
+class TestSuiteConfig(BaseModel):
+ """Test suite configuration base model.
+
+ By default the configuration of a generic test suite does not contain any attributes. Any test
+ suite should inherit this class to create their own custom configuration. Finally override the
+ type of the :attr:`~TestSuite.config` to use the newly created one.
+
+ If no custom fields require setting, this can also be represented as a string instead of
+ a mapping, example:
+
+ .. code:: yaml
+
+ test_runs:
+ - test_suites:
+ # As string representation:
+ hello_world: all # test all of `hello_world`, or
+ hello_world: hello_world_single_core # test only `hello_world_single_core`
+ # or as a mapping of the model's fields:
+ hello_world:
+ test_cases: [hello_world_single_core] # without this field all test cases are run
+
+ .. warning::
+
+ This class sets `protected_namespaces` to an empty tuple as a workaround for autodoc.
+ Due to autodoc loading this class first before any other child ones, it causes the Pydantic
+ fields in the protected namespace ``model_`` to be set on the parent. Leading any child
+ classes to inherit these protected fields as user-defined ones, finally triggering their
+ instances to complain about the presence of protected fields.
+
+ Because any class inheriting this class will therefore have protected namespaces disabled,
+ you won't be blocked to create fields starting with ``model_``. Nonetheless, you **must**
+ refrain from doing so as this is not the intended behavior.
+ """
+
+ model_config = ConfigDict(frozen=True, extra="forbid", protected_namespaces=())
+
+ #: The name of the test suite module without the starting ``TestSuite_``. This field **cannot**
+ #: be used in the configuration file. The name will be inherited from the mapping key instead.
+ test_suite_name: str
+ #: The names of test cases from this test suite to execute. If empty, all test cases will be
+ #: executed.
+ test_cases_names: list[str] = Field(default_factory=list, alias="test_cases")
+
+ @cached_property
+ def test_suite_spec(self) -> "TestSuiteSpec":
+ """The specification of the requested test suite."""
+ from framework.test_suite import find_by_name
+
+ test_suite_spec = find_by_name(self.test_suite_name)
+ assert (
+ test_suite_spec is not None
+ ), f"{self.test_suite_name} is not a valid test suite module name."
+ return test_suite_spec
+
+ @model_validator(mode="before")
+ @classmethod
+ def load_test_suite_name_from_context(cls, data: Any, info: ValidationInfo) -> dict:
+ """Load the test suite name from the validation context, if any."""
+ assert isinstance(data, dict), "The test suite configuration value is invalid."
+ name = data.get("test_suite_name")
+ # If the context is carrying the test suite name, then use it instead.
+ if info.context is not None and (test_suite_name := info.context.get("test_suite_name")):
+ assert not name, "The test suite name cannot be set manually."
+ data["test_suite_name"] = test_suite_name
+ return data
+
+ @model_validator(mode="before")
+ @classmethod
+ def convert_from_string(cls, data: Any) -> dict:
+ """Convert the string representation of the model into a valid mapping."""
+ if isinstance(data, str):
+ test_cases = [] if data == "all" else data.split()
+ return dict(test_cases=test_cases)
+ return data
+
+ @model_validator(mode="after")
+ def validate_names(self) -> Self:
+ """Validate the supplied test suite and test cases names.
+
+ This validator relies on the cached property `test_suite_spec` to run for the first
+ time in this call, therefore triggering the assertions if needed.
+ """
+ available_test_cases = map(
+ lambda t: t.name, self.test_suite_spec.class_obj.get_test_cases()
+ )
+ for requested_test_case in self.test_cases_names:
+ assert requested_test_case in available_test_cases, (
+ f"{requested_test_case} is not a valid test case "
+ f"of test suite {self.test_suite_name}."
+ )
+
+ return self
+
+
+class BaseTestSuitesConfigs(BaseModel):
+ """Base class for test suites configs."""
+
+ model_config = ConfigDict(frozen=True, extra="forbid")
+
+ def __contains__(self, key) -> bool:
+ """Check if the provided test suite name has been selected and/or configured."""
+ return key in self.model_fields_set
+
+ def __getitem__(self, key) -> TestSuiteConfig:
+ """Get test suite configuration."""
+ return self.__getattribute__(key)
+
+ def get_configs(self) -> Iterable[TestSuiteConfig]:
+ """Get all the test suite configurations."""
+ return map(lambda t: self[t], self.model_fields_set)
+
+ @classmethod
+ def available_test_suites(cls) -> Iterable[str]:
+ """List all the available test suites."""
+ return cls.model_fields.keys()
+
+ @field_validator("*", mode="before")
+ @classmethod
+ def pass_test_suite_name_to_config(cls, field_value: Any, info: ValidationInfo) -> Any:
+ """Before validating any :class:`TestSuiteConfig`, pass the test suite name via context."""
+ test_suite_name = info.field_name
+ assert test_suite_name is not None
+
+ assert info.context is not None, "A context dictionary is required to load test suites."
+ info.context.update({"test_suite_name": test_suite_name})
+
+ return field_value
@@ -25,6 +25,8 @@
from types import MethodType
from typing import Iterable
+from pydantic import ValidationError
+
from framework.testbed_model.capability import Capability, get_supported_capabilities
from framework.testbed_model.sut_node import SutNode
from framework.testbed_model.tg_node import TGNode
@@ -34,11 +36,17 @@
DPDKPrecompiledBuildConfiguration,
SutNodeConfiguration,
TestRunConfiguration,
- TestSuiteConfig,
TGNodeConfiguration,
load_config,
)
-from .exception import BlockingTestSuiteError, SSHTimeoutError, TestCaseVerifyError
+from .config.generated import CUSTOM_CONFIG_TYPES
+from .config.test_suite import TestSuiteConfig
+from .exception import (
+ BlockingTestSuiteError,
+ ConfigurationError,
+ SSHTimeoutError,
+ TestCaseVerifyError,
+)
from .logger import DTSLogger, DtsStage, get_dts_logger
from .settings import SETTINGS
from .test_result import (
@@ -141,12 +149,7 @@ def run(self) -> None:
self._logger.info(f"Running test run with SUT '{sut_node_config.name}'.")
self._init_random_seed(test_run_config)
test_run_result = self._result.add_test_run(test_run_config)
- # we don't want to modify the original config, so create a copy
- test_run_test_suites = list(
- SETTINGS.test_suites if SETTINGS.test_suites else test_run_config.test_suites
- )
- if not test_run_config.skip_smoke_tests:
- test_run_test_suites[:0] = [TestSuiteConfig(test_suite="smoke_tests")]
+ test_run_test_suites = self._prepare_test_suites(test_run_config)
try:
test_suites_with_cases = self._get_test_suites_with_cases(
test_run_test_suites, test_run_config.func, test_run_config.perf
@@ -203,6 +206,46 @@ def _check_dts_python_version(self) -> None:
)
self._logger.warning("Please use Python >= 3.10 instead.")
+ def _prepare_test_suites(self, test_run_config: TestRunConfiguration) -> list[TestSuiteConfig]:
+ if SETTINGS.test_suites:
+ test_suites_configs = []
+ for selected_test_suite, selected_test_cases in SETTINGS.test_suites:
+ if selected_test_suite in test_run_config.test_suites:
+ config = test_run_config.test_suites[selected_test_suite].model_copy(
+ update={"test_cases_names": selected_test_cases}
+ )
+ else:
+ try:
+ config = CUSTOM_CONFIG_TYPES[selected_test_suite](
+ test_suite_name=selected_test_suite, test_cases=selected_test_cases
+ )
+ except AssertionError as e:
+ raise ConfigurationError(
+ "Invalid test cases were selected "
+ f"for test suite {selected_test_suite}."
+ ) from e
+ except ValidationError as e:
+ raise ConfigurationError(
+ f"Test suite {selected_test_suite} needs to be explicitly configured "
+ "in order to be selected."
+ ) from e
+ except KeyError:
+ # not a custom configuration
+ config = TestSuiteConfig(
+ test_suite_name=selected_test_suite, test_cases=selected_test_cases
+ )
+ test_suites_configs.append(config)
+ else:
+ # we don't want to modify the original config, so create a copy
+ test_suites_configs = [
+ config.model_copy() for config in test_run_config.test_suites.get_configs()
+ ]
+
+ if not test_run_config.skip_smoke_tests:
+ test_suites_configs[:0] = [TestSuiteConfig(test_suite_name="smoke_tests")]
+
+ return test_suites_configs
+
def _get_test_suites_with_cases(
self,
test_suite_configs: list[TestSuiteConfig],
@@ -236,7 +279,11 @@ def _get_test_suites_with_cases(
test_cases.extend(perf_test_cases)
test_suites_with_cases.append(
- TestSuiteWithCases(test_suite_class=test_suite_class, test_cases=test_cases)
+ TestSuiteWithCases(
+ test_suite_class=test_suite_class,
+ test_cases=test_cases,
+ config=test_suite_config,
+ )
)
return test_suites_with_cases
@@ -453,7 +500,9 @@ def _run_test_suite(
self._logger.set_stage(
DtsStage.test_suite_setup, Path(SETTINGS.output_dir, test_suite_name)
)
- test_suite = test_suite_with_cases.test_suite_class(sut_node, tg_node, topology)
+ test_suite = test_suite_with_cases.test_suite_class(
+ sut_node, tg_node, topology, test_suite_with_cases.config
+ )
try:
self._logger.info(f"Starting test suite setup: {test_suite_name}")
test_suite.set_up_suite()
@@ -107,7 +107,7 @@
LocalDPDKTreeLocation,
RemoteDPDKTarballLocation,
RemoteDPDKTreeLocation,
- TestSuiteConfig,
+ TestSuitesConfigs,
)
@@ -133,7 +133,7 @@ class Settings:
#:
compile_timeout: float = 1200
#:
- test_suites: list[TestSuiteConfig] = field(default_factory=list)
+ test_suites: list[tuple[str, list[str]]] = field(default_factory=list)
#:
re_run: int = 0
#:
@@ -508,7 +508,7 @@ def _process_dpdk_location(
def _process_test_suites(
parser: _DTSArgumentParser, args: list[list[str]]
-) -> list[TestSuiteConfig]:
+) -> list[tuple[str, list[str]]]:
"""Process the given argument to a list of :class:`TestSuiteConfig` to execute.
Args:
@@ -524,19 +524,17 @@ def _process_test_suites(
# Environment variable in the form of "SUITE1 CASE1 CASE2, SUITE2 CASE1, SUITE3, ..."
args = [suite_with_cases.split() for suite_with_cases in args[0][0].split(",")]
- try:
- return [
- TestSuiteConfig(test_suite=test_suite, test_cases=test_cases)
- for [test_suite, *test_cases] in args
- ]
- except ValidationError as e:
- print(
- "An error has occurred while validating the test suites supplied in the "
- f"{'environment variable' if action else 'arguments'}:",
- file=sys.stderr,
- )
- print(e, file=sys.stderr)
- sys.exit(1)
+ available_test_suites = TestSuitesConfigs.available_test_suites()
+ for test_suite_name, *_ in args:
+ if test_suite_name not in available_test_suites:
+ print(
+ f"The test suite {test_suite_name} supplied in the "
+ f"{'environment variable' if action else 'arguments'} is invalid.",
+ file=sys.stderr,
+ )
+ sys.exit(1)
+
+ return [(test_suite, test_cases) for test_suite, *test_cases in args]
def get_settings() -> Settings:
@@ -30,7 +30,8 @@
from framework.testbed_model.capability import Capability
-from .config import TestRunConfiguration, TestSuiteConfig
+from .config import TestRunConfiguration
+from .config.test_suite import TestSuiteConfig
from .exception import DTSError, ErrorSeverity
from .logger import DTSLogger
from .settings import SETTINGS
@@ -59,23 +60,13 @@ class is to hold a subset of test cases (which could be all test cases) because
test_suite_class: type[TestSuite]
test_cases: list[type[TestCase]]
required_capabilities: set[Capability] = field(default_factory=set, init=False)
+ config: TestSuiteConfig
def __post_init__(self):
"""Gather the required capabilities of the test suite and all test cases."""
for test_object in [self.test_suite_class] + self.test_cases:
self.required_capabilities.update(test_object.required_capabilities)
- def create_config(self) -> TestSuiteConfig:
- """Generate a :class:`TestSuiteConfig` from the stored test suite with test cases.
-
- Returns:
- The :class:`TestSuiteConfig` representation.
- """
- return TestSuiteConfig(
- test_suite=self.test_suite_class.__name__,
- test_cases=[test_case.__name__ for test_case in self.test_cases],
- )
-
def mark_skip_unsupported(self, supported_capabilities: set[Capability]) -> None:
"""Mark the test suite and test cases to be skipped.
@@ -24,13 +24,14 @@
from ipaddress import IPv4Interface, IPv6Interface, ip_interface
from pkgutil import iter_modules
from types import ModuleType
-from typing import ClassVar, Protocol, TypeVar, Union, cast
+from typing import ClassVar, Protocol, TypeVar, Union, cast, get_type_hints
from scapy.layers.inet import IP # type: ignore[import-untyped]
from scapy.layers.l2 import Ether # type: ignore[import-untyped]
from scapy.packet import Packet, Padding, raw # type: ignore[import-untyped]
from typing_extensions import Self
+from framework.config.test_suite import TestSuiteConfig
from framework.testbed_model.capability import TestProtocol
from framework.testbed_model.port import Port
from framework.testbed_model.sut_node import SutNode
@@ -80,6 +81,7 @@ class TestSuite(TestProtocol):
#: Whether the test suite is blocking. A failure of a blocking test suite
#: will block the execution of all subsequent test suites in the current test run.
is_blocking: ClassVar[bool] = False
+ config: TestSuiteConfig
_logger: DTSLogger
_sut_port_ingress: Port
_sut_port_egress: Port
@@ -95,6 +97,7 @@ def __init__(
sut_node: SutNode,
tg_node: TGNode,
topology: Topology,
+ config: TestSuiteConfig,
):
"""Initialize the test suite testbed information and basic configuration.
@@ -105,9 +108,11 @@ def __init__(
sut_node: The SUT node where the test suite will run.
tg_node: The TG node where the test suite will run.
topology: The topology where the test suite will run.
+ config: The test suite configuration.
"""
self.sut_node = sut_node
self.tg_node = tg_node
+ self.config = config
self._logger = get_dts_logger(self.__class__.__name__)
self._tg_port_egress = topology.tg_port_egress
self._sut_port_ingress = topology.sut_port_ingress
@@ -663,6 +668,21 @@ def is_test_suite(obj) -> bool:
f"Expected class {self.class_name} not found in module {self.module_name}."
)
+ @cached_property
+ def config_obj(self) -> type[TestSuiteConfig]:
+ """A reference to the test suite's configuration type."""
+ fields = get_type_hints(self.class_obj)
+ config_obj = fields.get("config")
+ if config_obj is None:
+ raise InternalError(
+ "Test suite class {self.class_name} is missing the `config` attribute."
+ )
+ if not issubclass(config_obj, TestSuiteConfig):
+ raise InternalError(
+ f"Test suite class {self.class_name} has an invalid configuration type assigned."
+ )
+ return config_obj
+
@classmethod
def discover_all(
cls, package_name: str | None = None, module_prefix: str | None = None
@@ -15,12 +15,15 @@
LogicalCoreCountFilter,
LogicalCoreList,
)
+from tests.config import HelloWorldConfig
@requires(topology_type=TopologyType.no_link)
class TestHelloWorld(TestSuite):
"""DPDK hello world app test suite."""
+ config: HelloWorldConfig
+
def set_up_suite(self) -> None:
"""Set up the test suite.
@@ -63,7 +66,7 @@ def hello_world_all_cores(self) -> None:
eal_para = compute_eal_params(
self.sut_node, lcore_filter_specifier=LogicalCoreList(self.sut_node.lcores)
)
- result = self.sut_node.run_dpdk_app(self.app_helloworld_path, eal_para, 50)
+ result = self.sut_node.run_dpdk_app(self.app_helloworld_path, eal_para, self.config.timeout)
for lcore in self.sut_node.lcores:
self.verify(
f"hello from core {int(lcore)}" in result.stdout,
new file mode 100644
@@ -0,0 +1,7 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2024 Arm Limited
+
+"""Test suites.
+
+This package contains all the available test suites in DTS.
+"""
new file mode 100644
@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2024 Arm Limited
+
+"""Module for test suites custom configurations.
+
+Any test suite that requires custom configuration fields should create a new config class inheriting
+:class:`~framework.config.test_suite.TestSuiteConfig`, while respecting the parents' frozen state.
+Any custom fields can be added in this class.
+
+The custom configuration classes can be stored in this module.
+"""
+
+from framework.config.test_suite import TestSuiteConfig
+
+
+class HelloWorldConfig(TestSuiteConfig):
+ """Example custom configuration for the `TestHelloWorld` test suite."""
+
+ #: Timeout for the DPDK apps.
+ timeout: int = 50