@@ -4,13 +4,13 @@
test_runs:
# define one test run environment
- - dpdk_builds:
- - arch: x86_64
- os: linux
- cpu: native
- # the combination of the following two makes CC="ccache gcc"
- compiler: gcc
- compiler_wrapper: ccache
+ - dpdk_build:
+ arch: x86_64
+ os: linux
+ cpu: native
+ # the combination of the following two makes CC="ccache gcc"
+ compiler: gcc
+ compiler_wrapper: ccache
perf: false # disable performance testing
func: true # enable functional testing
skip_smoke_tests: false # optional
@@ -437,7 +437,7 @@ class TestRunConfiguration:
and with what DPDK build.
Attributes:
- dpdk_builds: A list of DPDK builds to test.
+ dpdk_build: A DPDK build to test.
perf: Whether to run performance tests.
func: Whether to run functional tests.
skip_smoke_tests: Whether to skip smoke tests.
@@ -447,7 +447,7 @@ class TestRunConfiguration:
vdevs: The names of virtual devices to test.
"""
- dpdk_builds: list[DPDKBuildConfiguration]
+ dpdk_build: DPDKBuildConfiguration
perf: bool
func: bool
skip_smoke_tests: bool
@@ -475,9 +475,6 @@ def from_dict(
Returns:
The test run configuration instance.
"""
- dpdk_builds: list[DPDKBuildConfiguration] = list(
- map(DPDKBuildConfiguration.from_dict, d["dpdk_builds"])
- )
test_suites: list[TestSuiteConfig] = list(map(TestSuiteConfig.from_dict, d["test_suites"]))
sut_name = d["system_under_test_node"]["node_name"]
skip_smoke_tests = d.get("skip_smoke_tests", False)
@@ -498,7 +495,7 @@ def from_dict(
d["system_under_test_node"]["vdevs"] if "vdevs" in d["system_under_test_node"] else []
)
return cls(
- dpdk_builds=dpdk_builds,
+ dpdk_build=DPDKBuildConfiguration.from_dict(d["dpdk_build"]),
perf=d["perf"],
func=d["func"],
skip_smoke_tests=skip_smoke_tests,
@@ -327,12 +327,8 @@
"items": {
"type": "object",
"properties": {
- "dpdk_builds": {
- "type": "array",
- "items": {
- "$ref": "#/definitions/dpdk_build"
- },
- "minimum": 1
+ "dpdk_build": {
+ "$ref": "#/definitions/dpdk_build"
},
"perf": {
"type": "boolean",
@@ -383,7 +379,7 @@
},
"additionalProperties": false,
"required": [
- "dpdk_builds",
+ "dpdk_build",
"perf",
"func",
"test_suites",
@@ -108,7 +108,7 @@ class TestRunConfigDict(TypedDict):
"""Allowed keys and values."""
#:
- dpdk_builds: list[DPDKBuildConfigDict]
+ dpdk_build: DPDKBuildConfigDict
#:
perf: bool
#:
@@ -33,16 +33,12 @@ class DtsStage(StrEnum):
#:
test_run_setup = auto()
#:
- dpdk_build_setup = auto()
- #:
test_suite_setup = auto()
#:
test_suite = auto()
#:
test_suite_teardown = auto()
#:
- dpdk_build_teardown = auto()
- #:
test_run_teardown = auto()
#:
post_run = auto()
@@ -12,7 +12,7 @@
#. Test suite stage,
#. Test case stage.
-The test run and DPDK build stages set up the environment before running test suites.
+The test run stage sets up the environment before running test suites.
The test suite stage sets up steps common to all test cases
and the test case stage runs test cases individually.
"""
@@ -29,13 +29,7 @@
from framework.testbed_model.sut_node import SutNode
from framework.testbed_model.tg_node import TGNode
-from .config import (
- Configuration,
- DPDKBuildConfiguration,
- TestRunConfiguration,
- TestSuiteConfig,
- load_config,
-)
+from .config import Configuration, TestRunConfiguration, TestSuiteConfig, load_config
from .exception import (
BlockingTestSuiteError,
ConfigurationError,
@@ -45,7 +39,6 @@
from .logger import DTSLogger, DtsStage, get_dts_logger
from .settings import SETTINGS
from .test_result import (
- DPDKBuildResult,
DTSResult,
Result,
TestCaseResult,
@@ -69,9 +62,9 @@ class DTSRunner:
:class:`~.framework.exception.DTSError`\s.
Example:
- An error occurs in a DPDK build setup. The current DPDK build is aborted,
- all test suites and their test cases are marked as blocked and the run continues
- with the next DPDK build. If the errored DPDK build was the last one in the
+ An error occurs in a test suite setup. The current test suite is aborted,
+ all its test cases are marked as blocked and the run continues
+ with the next test suite. If the errored test suite was the last one in the
given test run, the next test run begins.
"""
@@ -97,16 +90,16 @@ def __init__(self):
self._perf_test_case_regex = r"test_perf_"
def run(self) -> None:
- """Run all DPDK build in all test runs from the test run configuration.
+ """Run all test runs from the test run configuration.
- Before running test suites, test runs and DPDK builds are first set up.
- The test runs and DPDK builds defined in the test run configuration are iterated over.
- The test runs define which tests to run and where to run them and DPDK builds define
- the DPDK build setup.
+ Before running test suites, test runs are first set up.
+ The test runs defined in the test run configuration are iterated over.
+ The test runs define which tests to run and where to run them.
- The tests suites are set up for each test run/DPDK build tuple and each discovered
+ The test suites are set up for each test run and each discovered
test case within the test suite is set up, executed and torn down. After all test cases
- have been executed, the test suite is torn down and the next DPDK build will be tested.
+ have been executed, the test suite is torn down and the next test suite will be run. Once
+ all test suites have been run, the test test run will be tested.
In order to properly mark test suites and test cases as blocked in case of a failure,
we need to have discovered which test suites and test cases to run before any failures
@@ -116,17 +109,13 @@ def run(self) -> None:
#. Test run setup
- #. DPDK build setup
-
- #. Test suite setup
+ #. Test suite setup
- #. Test case setup
- #. Test case logic
- #. Test case teardown
+ #. Test case setup
+ #. Test case logic
+ #. Test case teardown
- #. Test suite teardown
-
- #. DPDK build teardown
+ #. Test suite teardown
#. Test run teardown
@@ -414,7 +403,7 @@ def _run_test_run(
) -> None:
"""Run the given test run.
- This involves running the test run setup as well as running all DPDK builds
+ This involves running the test run setup as well as running all test suites
in the given test run. After that, the test run teardown is run.
Args:
@@ -430,6 +419,7 @@ def _run_test_run(
test_run_result.add_sut_info(sut_node.node_info)
try:
sut_node.set_up_test_run(test_run_config)
+ test_run_result.add_dpdk_build_info(sut_node.get_dpdk_build_info())
tg_node.set_up_test_run(test_run_config)
test_run_result.update_setup(Result.PASS)
except Exception as e:
@@ -437,15 +427,7 @@ def _run_test_run(
test_run_result.update_setup(Result.FAIL, e)
else:
- for dpdk_build_config in test_run_config.dpdk_builds:
- dpdk_build_result = test_run_result.add_dpdk_build(dpdk_build_config)
- self._run_dpdk_build(
- sut_node,
- tg_node,
- dpdk_build_config,
- dpdk_build_result,
- test_suites_with_cases,
- )
+ self._run_test_suites(sut_node, tg_node, test_run_result, test_suites_with_cases)
finally:
try:
@@ -457,82 +439,35 @@ def _run_test_run(
self._logger.exception("Test run teardown failed.")
test_run_result.update_teardown(Result.FAIL, e)
- def _run_dpdk_build(
- self,
- sut_node: SutNode,
- tg_node: TGNode,
- dpdk_build_config: DPDKBuildConfiguration,
- dpdk_build_result: DPDKBuildResult,
- test_suites_with_cases: Iterable[TestSuiteWithCases],
- ) -> None:
- """Run the given DPDK build.
-
- This involves running the DPDK build setup as well as running all test suites
- of the DPDK build's test run.
- After that, DPDK build teardown is run.
-
- Args:
- sut_node: The test run's sut node.
- tg_node: The test run's tg node.
- dpdk_build_config: A DPDK build's test run configuration.
- dpdk_build_result: The DPDK build level result object associated
- with the current DPDK build.
- test_suites_with_cases: The test suites with test cases to run.
- """
- self._logger.set_stage(DtsStage.dpdk_build_setup)
- self._logger.info(f"Running DPDK build '{dpdk_build_config.name}'.")
-
- try:
- sut_node.set_up_dpdk(dpdk_build_config)
- self._result.dpdk_version = sut_node.dpdk_version
- dpdk_build_result.add_dpdk_build_info(sut_node.get_dpdk_build_info())
- dpdk_build_result.update_setup(Result.PASS)
- except Exception as e:
- self._logger.exception("DPDK build setup failed.")
- dpdk_build_result.update_setup(Result.FAIL, e)
-
- else:
- self._run_test_suites(sut_node, tg_node, dpdk_build_result, test_suites_with_cases)
-
- finally:
- try:
- self._logger.set_stage(DtsStage.dpdk_build_teardown)
- sut_node.tear_down_dpdk()
- dpdk_build_result.update_teardown(Result.PASS)
- except Exception as e:
- self._logger.exception("DPDK build teardown failed.")
- dpdk_build_result.update_teardown(Result.FAIL, e)
-
def _run_test_suites(
self,
sut_node: SutNode,
tg_node: TGNode,
- dpdk_build_result: DPDKBuildResult,
+ test_run_result: TestRunResult,
test_suites_with_cases: Iterable[TestSuiteWithCases],
) -> None:
- """Run `test_suites_with_cases` with the current DPDK build.
+ """Run `test_suites_with_cases` with the current test run.
The method assumes the DPDK we're testing has already been built on the SUT node.
If a blocking test suite (such as the smoke test suite) fails, the rest of the test suites
- in the current DPDK build won't be executed.
+ in the current test run won't be executed.
Args:
sut_node: The test run's SUT node.
tg_node: The test run's TG node.
- dpdk_build_result: The DPDK build level result object associated
- with the current DPDK build.
+ test_run_result: The test run's result.
test_suites_with_cases: The test suites with test cases to run.
"""
end_dpdk_build = False
for test_suite_with_cases in test_suites_with_cases:
- test_suite_result = dpdk_build_result.add_test_suite(test_suite_with_cases)
+ test_suite_result = test_run_result.add_test_suite(test_suite_with_cases)
try:
self._run_test_suite(sut_node, tg_node, test_suite_result, test_suite_with_cases)
except BlockingTestSuiteError as e:
self._logger.exception(
f"An error occurred within {test_suite_with_cases.test_suite_class.__name__}. "
- "Skipping DPDK build ..."
+ "Skipping the rest of the test suites in this test run."
)
self._result.add_error(e)
end_dpdk_build = True
@@ -8,12 +8,11 @@
* :class:`DTSResult` contains
* :class:`TestRunResult` contains
- * :class:`DPDKBuildResult` contains
* :class:`TestSuiteResult` contains
* :class:`TestCaseResult`
Each result may contain multiple lower level results, e.g. there are multiple
-:class:`TestSuiteResult`\s in a :class:`DPDKBuildResult`.
+:class:`TestSuiteResult`\s in an :class:`TestRunResult`.
The results have common parts, such as setup and teardown results, captured in :class:`BaseResult`,
which also defines some common behaviors in its methods.
@@ -35,7 +34,6 @@
Architecture,
Compiler,
CPUType,
- DPDKBuildConfiguration,
DPDKBuildInfo,
NodeInfo,
TestRunConfiguration,
@@ -138,7 +136,7 @@ class BaseResult:
Stores the results of the setup and teardown portions of the corresponding stage.
The hierarchical nature of DTS results is captured recursively in an internal list.
A stage is each level in this particular hierarchy (pre-run or the top-most level,
- test run, DPDK build, test suite and test case.)
+ test run, test suite and test case.)
Attributes:
setup_result: The result of the setup of the particular stage.
@@ -222,8 +220,8 @@ def add_stats(self, statistics: "Statistics") -> None:
class DTSResult(BaseResult):
"""Stores environment information and test results from a DTS run.
- * Test run level information, such as testbed and the test suite list,
- * DPDK build level information, such as compiler, target OS and cpu,
+ * Test run level information, such as testbed, compiler, target OS and cpu and
+ the test suite list,
* Test suite and test case results,
* All errors that are caught and recorded during DTS execution.
@@ -317,44 +315,61 @@ def get_return_code(self) -> int:
class TestRunResult(BaseResult):
"""The test run specific result.
- The internal list stores the results of all DPDK builds in a given test run.
+ The internal list stores the results of all test suites in a given test run.
Attributes:
+ arch: The DPDK build architecture.
+ os: The DPDK build operating system.
+ cpu: The DPDK build CPU.
+ compiler: The DPDK build compiler.
+ compiler_version: The DPDK build compiler version.
+ dpdk_version: The built DPDK version.
sut_os_name: The operating system of the SUT node.
sut_os_version: The operating system version of the SUT node.
sut_kernel_version: The operating system kernel version of the SUT node.
"""
+ arch: Architecture
+ os: OS
+ cpu: CPUType
+ compiler: Compiler
+ compiler_version: str | None
+ dpdk_version: str | None
sut_os_name: str
sut_os_version: str
sut_kernel_version: str
_config: TestRunConfiguration
- _parent_result: DTSResult
_test_suites_with_cases: list[TestSuiteWithCases]
def __init__(self, test_run_config: TestRunConfiguration):
- """Extend the constructor with the test run's config and DTSResult.
+ """Extend the constructor with the test run's config.
Args:
test_run_config: A test run configuration.
"""
super().__init__()
+ self.arch = test_run_config.dpdk_build.arch
+ self.os = test_run_config.dpdk_build.os
+ self.cpu = test_run_config.dpdk_build.cpu
+ self.compiler = test_run_config.dpdk_build.compiler
+ self.compiler_version = None
+ self.dpdk_version = None
self._config = test_run_config
self._test_suites_with_cases = []
- def add_dpdk_build(self, dpdk_build_config: DPDKBuildConfiguration) -> "DPDKBuildResult":
- """Add and return the child result (DPDK build).
+ def add_test_suite(
+ self,
+ test_suite_with_cases: TestSuiteWithCases,
+ ) -> "TestSuiteResult":
+ """Add and return the child result (test suite).
Args:
- dpdk_build: The DPDK build's test run configuration.
+ test_suite_with_cases: The test suite with test cases.
Returns:
- The DPDK build's result.
+ The test suite's result.
"""
- result = DPDKBuildResult(
- self._test_suites_with_cases,
- dpdk_build_config,
- )
+ result = TestSuiteResult(test_suite_with_cases)
self.child_results.append(result)
return result
@@ -390,71 +405,6 @@ def add_sut_info(self, sut_info: NodeInfo) -> None:
self.sut_os_version = sut_info.os_version
self.sut_kernel_version = sut_info.kernel_version
- def _block_result(self) -> None:
- r"""Mark the result as :attr:`~Result.BLOCK`\ed."""
- for dpdk_build in self._config.dpdk_builds:
- child_result = self.add_dpdk_build(dpdk_build)
- child_result.update_setup(Result.BLOCK)
-
-
-class DPDKBuildResult(BaseResult):
- """The DPDK build specific result.
-
- The internal list stores the results of all test suites in a given DPDK build.
-
- Attributes:
- arch: The DPDK DPDK build architecture.
- os: The DPDK DPDK build operating system.
- cpu: The DPDK DPDK build CPU.
- compiler: The DPDK DPDK build compiler.
- compiler_version: The DPDK DPDK build compiler version.
- dpdk_version: The built DPDK version.
- """
-
- arch: Architecture
- os: OS
- cpu: CPUType
- compiler: Compiler
- compiler_version: str | None
- dpdk_version: str | None
- _test_suites_with_cases: list[TestSuiteWithCases]
-
- def __init__(
- self,
- test_suites_with_cases: list[TestSuiteWithCases],
- dpdk_build_config: DPDKBuildConfiguration,
- ):
- """Extend the constructor with the DPDK build's config and test suites with cases.
-
- Args:
- test_suites_with_cases: The test suites with test cases to be run in this DPDK build.
- dpdk_build_config: The DPDK build's test run configuration.
- """
- super().__init__()
- self.arch = dpdk_build_config.arch
- self.os = dpdk_build_config.os
- self.cpu = dpdk_build_config.cpu
- self.compiler = dpdk_build_config.compiler
- self.compiler_version = None
- self.dpdk_version = None
- self._test_suites_with_cases = test_suites_with_cases
-
- def add_test_suite(
- self,
- test_suite_with_cases: TestSuiteWithCases,
- ) -> "TestSuiteResult":
- """Add and return the child result (test suite).
-
- Args:
- test_suite_with_cases: The test suite with test cases.
-
- Returns:
- The test suite's result.
- """
- result = TestSuiteResult(test_suite_with_cases)
- self.child_results.append(result)
- return result
-
def add_dpdk_build_info(self, versions: DPDKBuildInfo) -> None:
"""Add information about the DPDK build gathered at runtime.
@@ -482,11 +432,10 @@ class TestSuiteResult(BaseResult):
test_suite_name: str
_test_suite_with_cases: TestSuiteWithCases
- _parent_result: DPDKBuildResult
_child_configs: list[str]
def __init__(self, test_suite_with_cases: TestSuiteWithCases):
- """Extend the constructor with test suite's config and DPDKBuildResult.
+ """Extend the constructor with test suite's config.
Args:
test_suite_with_cases: The test suite with test cases.
@@ -529,7 +478,7 @@ class TestCaseResult(BaseResult, FixtureResult):
test_case_name: str
def __init__(self, test_case_name: str):
- """Extend the constructor with test case's name and TestSuiteResult.
+ """Extend the constructor with test case's name.
Args:
test_case_name: The test case's name.
@@ -65,7 +65,7 @@ class TestSuite:
sut_node: SutNode
tg_node: TGNode
#: Whether the test suite is blocking. A failure of a blocking test suite
- #: will block the execution of all subsequent test suites in the current DPDK build.
+ #: will block the execution of all subsequent test suites in the current test run.
is_blocking: ClassVar[bool] = False
_logger: DTSLogger
_port_links: list[PortLink]
@@ -181,13 +181,15 @@ def set_up_test_run(self, test_run_config: TestRunConfiguration) -> None:
super().set_up_test_run(test_run_config)
for vdev in test_run_config.vdevs:
self.virtual_devices.append(VirtualDevice(vdev))
+ self._set_up_dpdk(test_run_config.dpdk_build)
def tear_down_test_run(self) -> None:
"""Extend the test run teardown with virtual device teardown."""
super().tear_down_test_run()
self.virtual_devices = []
+ self._tear_down_dpdk()
- def set_up_dpdk(self, dpdk_build_config: DPDKBuildConfiguration) -> None:
+ def _set_up_dpdk(self, dpdk_build_config: DPDKBuildConfiguration) -> None:
"""Set up DPDK the SUT node and bind ports.
DPDK setup includes setting all internals needed for the build, the copying of DPDK tarball
@@ -202,7 +204,7 @@ def set_up_dpdk(self, dpdk_build_config: DPDKBuildConfiguration) -> None:
self._build_dpdk()
self.bind_ports_to_driver()
- def tear_down_dpdk(self) -> None:
+ def _tear_down_dpdk(self) -> None:
"""Reset DPDK variables and bind port driver to the OS driver."""
self._env_vars = {}
self._dpdk_build_config = None
@@ -29,7 +29,7 @@ class TestSmokeTests(TestSuite):
Attributes:
is_blocking: This test suite will block the execution of all other test suites
- in the DPDK build after it.
+ in the test run after it.
nics_in_node: The NICs present on the SUT node.
"""