[v4,7/7] dts: run all test suites by default
Checks
Commit Message
The configuration requires the user to explicitly set the requested test
suites in the files. Sometimes we want to run all the test suites and
don't want to manually specify all of them. It is therefore reasonable
to change the default behaviour to automatically run all the available
test suites if none are specified.
Bugzilla ID: 1360
Signed-off-by: Luca Vizzarro <luca.vizzarro@arm.com>
Reviewed-by: Paul Szczepanek <paul.szczepanek@arm.com>
Reviewed-by: Dean Marx <dmarx@iol.unh.edu>
---
dts/framework/config/test_run.py | 16 +++++++++++++++-
dts/test_runs.example.yaml | 3 ++-
2 files changed, 17 insertions(+), 2 deletions(-)
Comments
Nice touch!
Reviewed-by: Nicholas Pratte <npratte@iol.unh.edu>
On Fri, Jan 24, 2025 at 6:39 AM Luca Vizzarro <luca.vizzarro@arm.com> wrote:
>
> The configuration requires the user to explicitly set the requested test
> suites in the files. Sometimes we want to run all the test suites and
> don't want to manually specify all of them. It is therefore reasonable
> to change the default behaviour to automatically run all the available
> test suites if none are specified.
>
> Bugzilla ID: 1360
>
> Signed-off-by: Luca Vizzarro <luca.vizzarro@arm.com>
> Reviewed-by: Paul Szczepanek <paul.szczepanek@arm.com>
> Reviewed-by: Dean Marx <dmarx@iol.unh.edu>
> ---
> dts/framework/config/test_run.py | 16 +++++++++++++++-
> dts/test_runs.example.yaml | 3 ++-
> 2 files changed, 17 insertions(+), 2 deletions(-)
>
> diff --git a/dts/framework/config/test_run.py b/dts/framework/config/test_run.py
> index dc0e46047d..006410b467 100644
> --- a/dts/framework/config/test_run.py
> +++ b/dts/framework/config/test_run.py
> @@ -259,6 +259,20 @@ def validate_names(self) -> Self:
> return self
>
>
> +def fetch_all_test_suites() -> list[TestSuiteConfig]:
> + """Returns all the available test suites as configuration objects.
> +
> + This function does not include the smoke tests.
> + """
> + from framework.test_suite import AVAILABLE_TEST_SUITES
> +
> + return [
> + TestSuiteConfig(test_suite=test_suite.name)
> + for test_suite in AVAILABLE_TEST_SUITES
> + if test_suite.name != "smoke_tests"
> + ]
> +
> +
> class TestRunConfiguration(FrozenModel):
> """The configuration of a test run.
>
> @@ -275,7 +289,7 @@ class TestRunConfiguration(FrozenModel):
> #: Whether to skip smoke tests.
> skip_smoke_tests: bool = False
> #: The names of test suites and/or test cases to execute.
> - test_suites: list[TestSuiteConfig] = Field(min_length=1)
> + test_suites: list[TestSuiteConfig] = Field(default_factory=fetch_all_test_suites)
> #: The SUT node name to use in this test run.
> system_under_test_node: str
> #: The TG node name to use in this test run.
> diff --git a/dts/test_runs.example.yaml b/dts/test_runs.example.yaml
> index 5b6afb153e..5cc167ebe1 100644
> --- a/dts/test_runs.example.yaml
> +++ b/dts/test_runs.example.yaml
> @@ -23,6 +23,7 @@
> perf: false # disable performance testing
> func: true # enable functional testing
> skip_smoke_tests: false # optional
> + # by removing the `test_suites` field, this test run will run every test suite available
> test_suites: # the following test suites will be run in their entirety
> - hello_world
> vdevs: # optional; if removed, vdevs won't be used in the execution
> @@ -30,4 +31,4 @@
> # The machine running the DPDK test executable
> system_under_test_node: "SUT 1"
> # Traffic generator node to use for this execution environment
> - traffic_generator_node: "TG 1"
> \ No newline at end of file
> + traffic_generator_node: "TG 1"
> --
> 2.43.0
>
@@ -259,6 +259,20 @@ def validate_names(self) -> Self:
return self
+def fetch_all_test_suites() -> list[TestSuiteConfig]:
+ """Returns all the available test suites as configuration objects.
+
+ This function does not include the smoke tests.
+ """
+ from framework.test_suite import AVAILABLE_TEST_SUITES
+
+ return [
+ TestSuiteConfig(test_suite=test_suite.name)
+ for test_suite in AVAILABLE_TEST_SUITES
+ if test_suite.name != "smoke_tests"
+ ]
+
+
class TestRunConfiguration(FrozenModel):
"""The configuration of a test run.
@@ -275,7 +289,7 @@ class TestRunConfiguration(FrozenModel):
#: Whether to skip smoke tests.
skip_smoke_tests: bool = False
#: The names of test suites and/or test cases to execute.
- test_suites: list[TestSuiteConfig] = Field(min_length=1)
+ test_suites: list[TestSuiteConfig] = Field(default_factory=fetch_all_test_suites)
#: The SUT node name to use in this test run.
system_under_test_node: str
#: The TG node name to use in this test run.
@@ -23,6 +23,7 @@
perf: false # disable performance testing
func: true # enable functional testing
skip_smoke_tests: false # optional
+ # by removing the `test_suites` field, this test run will run every test suite available
test_suites: # the following test suites will be run in their entirety
- hello_world
vdevs: # optional; if removed, vdevs won't be used in the execution
@@ -30,4 +31,4 @@
# The machine running the DPDK test executable
system_under_test_node: "SUT 1"
# Traffic generator node to use for this execution environment
- traffic_generator_node: "TG 1"
\ No newline at end of file
+ traffic_generator_node: "TG 1"