[v2,2/5] dts: add random generation seed setting

When introducing pseudo-random generation in the test runs we need to
ensure that these can be reproduced by setting a pre-defined seed.
This commits adds the ability to set one or allow for one to be
generated and reported back to the user.

Signed-off-by: Luca Vizzarro <luca.vizzarro@arm.com>
Reviewed-by: Paul Szczepanek <paul.szczepanek@arm.com>
Reviewed-by: Alex Chapman <alex.chapman@arm.com>
---
 doc/guides/tools/dts.rst                   |  5 +++++
 dts/framework/config/__init__.py           |  4 ++++
 dts/framework/config/conf_yaml_schema.json |  4 ++++
 dts/framework/config/types.py              |  2 ++
 dts/framework/runner.py                    |  8 ++++++++
 dts/framework/settings.py                  | 17 +++++++++++++++++
 6 files changed, 40 insertions(+)
  

Makes a lot of sense to me.

On Tue, Aug 6, 2024 at 8:49 AM Luca Vizzarro <luca.vizzarro@arm.com> wrote:
>
> When introducing pseudo-random generation in the test runs we need to
> ensure that these can be reproduced by setting a pre-defined seed.
> This commits adds the ability to set one or allow for one to be
> generated and reported back to the user.
>
> Signed-off-by: Luca Vizzarro <luca.vizzarro@arm.com>
> Reviewed-by: Paul Szczepanek <paul.szczepanek@arm.com>
> Reviewed-by: Alex Chapman <alex.chapman@arm.com>
> ---

Reviewed-by: Jeremy Spewock <jspewock@iol.unh.edu>
  

This is a great idea. I originally didn't think the config file option 
would be very useful, but thinking a bit more about it, when debugging 
and trying to find a solution the devs need to run with the same seed 
over and over, so it's going to have a place.

Reviewed-by: Juraj Linkeš <juraj.linkes@pantheon.tech>

On 6. 8. 2024 14:46, Luca Vizzarro wrote:
> When introducing pseudo-random generation in the test runs we need to
> ensure that these can be reproduced by setting a pre-defined seed.
> This commits adds the ability to set one or allow for one to be
> generated and reported back to the user.
> 
> Signed-off-by: Luca Vizzarro <luca.vizzarro@arm.com>
> Reviewed-by: Paul Szczepanek <paul.szczepanek@arm.com>
> Reviewed-by: Alex Chapman <alex.chapman@arm.com>
> ---
>   doc/guides/tools/dts.rst                   |  5 +++++
>   dts/framework/config/__init__.py           |  4 ++++
>   dts/framework/config/conf_yaml_schema.json |  4 ++++
>   dts/framework/config/types.py              |  2 ++
>   dts/framework/runner.py                    |  8 ++++++++
>   dts/framework/settings.py                  | 17 +++++++++++++++++
>   6 files changed, 40 insertions(+)
> 
> diff --git a/doc/guides/tools/dts.rst b/doc/guides/tools/dts.rst
> index 515b15e4d8..9b5ea9779c 100644
> --- a/doc/guides/tools/dts.rst
> +++ b/doc/guides/tools/dts.rst
> @@ -251,6 +251,8 @@ DTS is run with ``main.py`` located in the ``dts`` directory after entering Poet
>                              ... | DTS_TEST_SUITES='suite, suite case, ...' (default: [])
>        --re-run N_TIMES, --re_run N_TIMES
>                              [DTS_RERUN] Re-run each test case the specified number of times if a test failure occurs. (default: 0)
> +     --random-seed NUMBER  [DTS_RANDOM_SEED] The seed to use with the pseudo-random generator. If not specified, the configuration value is
> +                           used instead. If that's also not specified, a random seed is generated. (default: None)
>   
>   
>   The brackets contain the names of environment variables that set the same thing.
> @@ -548,6 +550,9 @@ involved in the testing. These can be defined with the following mappings:
>      +----------------------------+---------------+---------------------------------------------------+
>      | ``traffic_generator_node`` | Node name for the traffic generator node.                         |
>      +----------------------------+-------------------------------------------------------------------+
> +   | ``random_seed``            | (*optional*) *int* – Allows you to set a seed for pseudo-random   |
> +   |                            | generation.                                                       |
> +   +----------------------------+-------------------------------------------------------------------+
>   
>   ``nodes``
>      `sequence <https://docs.python.org/3/library/stdtypes.html#sequence-types-list-tuple-range>`_ listing
> diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py
> index df60a5030e..269d9ec318 100644
> --- a/dts/framework/config/__init__.py
> +++ b/dts/framework/config/__init__.py
> @@ -445,6 +445,7 @@ class TestRunConfiguration:
>           system_under_test_node: The SUT node to use in this test run.
>           traffic_generator_node: The TG node to use in this test run.
>           vdevs: The names of virtual devices to test.
> +        random_seed: The seed to use for pseudo-random generation.
>       """
>   
>       build_targets: list[BuildTargetConfiguration]
> @@ -455,6 +456,7 @@ class TestRunConfiguration:
>       system_under_test_node: SutNodeConfiguration
>       traffic_generator_node: TGNodeConfiguration
>       vdevs: list[str]
> +    random_seed: int | None
>   
>       @classmethod
>       def from_dict(
> @@ -497,6 +499,7 @@ def from_dict(
>           vdevs = (
>               d["system_under_test_node"]["vdevs"] if "vdevs" in d["system_under_test_node"] else []
>           )
> +        random_seed = d.get("random_seed", None)
>           return cls(
>               build_targets=build_targets,
>               perf=d["perf"],
> @@ -506,6 +509,7 @@ def from_dict(
>               system_under_test_node=system_under_test_node,
>               traffic_generator_node=traffic_generator_node,
>               vdevs=vdevs,
> +            random_seed=random_seed,
>           )
>   
>       def copy_and_modify(self, **kwargs) -> Self:
> diff --git a/dts/framework/config/conf_yaml_schema.json b/dts/framework/config/conf_yaml_schema.json
> index f02a310bb5..df390e8ae2 100644
> --- a/dts/framework/config/conf_yaml_schema.json
> +++ b/dts/framework/config/conf_yaml_schema.json
> @@ -379,6 +379,10 @@
>             },
>             "traffic_generator_node": {
>               "$ref": "#/definitions/node_name"
> +          },
> +          "random_seed": {
> +            "type": "integer",
> +            "description": "Optional field. Allows you to set a seed for pseudo-random generation."
>             }
>           },
>           "additionalProperties": false,
> diff --git a/dts/framework/config/types.py b/dts/framework/config/types.py
> index cf16556403..ce7b784ac8 100644
> --- a/dts/framework/config/types.py
> +++ b/dts/framework/config/types.py
> @@ -121,6 +121,8 @@ class TestRunConfigDict(TypedDict):
>       system_under_test_node: TestRunSUTConfigDict
>       #:
>       traffic_generator_node: str
> +    #:
> +    random_seed: int
>   
>   
>   class ConfigurationDict(TypedDict):
> diff --git a/dts/framework/runner.py b/dts/framework/runner.py
> index 6b6f6a05f5..34b1dad5c4 100644
> --- a/dts/framework/runner.py
> +++ b/dts/framework/runner.py
> @@ -20,6 +20,7 @@
>   import importlib
>   import inspect
>   import os
> +import random
>   import re
>   import sys
>   from pathlib import Path
> @@ -147,6 +148,7 @@ def run(self) -> None:
>                   self._logger.info(
>                       f"Running test run with SUT '{test_run_config.system_under_test_node.name}'."
>                   )
> +                self._init_random_seed(test_run_config)
>                   test_run_result = self._result.add_test_run(test_run_config)
>                   # we don't want to modify the original config, so create a copy
>                   test_run_test_suites = list(
> @@ -723,3 +725,9 @@ def _exit_dts(self) -> None:
>               self._logger.info("DTS execution has ended.")
>   
>           sys.exit(self._result.get_return_code())
> +
> +    def _init_random_seed(self, conf: TestRunConfiguration) -> None:
> +        """Initialize the random seed to use for the test run."""
> +        seed = SETTINGS.random_seed or conf.random_seed or random.randrange(0xFFFF_FFFF)
> +        self._logger.info(f"Initializing test run with random seed {seed}")
> +        random.seed(seed)
> diff --git a/dts/framework/settings.py b/dts/framework/settings.py
> index f6303066d4..7744e37f54 100644
> --- a/dts/framework/settings.py
> +++ b/dts/framework/settings.py
> @@ -66,6 +66,12 @@
>   
>       Re-run each test case this many times in case of a failure.
>   
> +.. option:: --random-seed
> +.. envvar:: DTS_RANDOM_SEED
> +
> +    The seed to use with the pseudo-random generator. If not specified, the configuration value is
> +    used instead. If that's also not specified, a random seed is generated.
> +
>   The module provides one key module-level variable:
>   
>   Attributes:
> @@ -115,6 +121,8 @@ class Settings:
>       test_suites: list[TestSuiteConfig] = field(default_factory=list)
>       #:
>       re_run: int = 0
> +    #:
> +    random_seed: int | None = None
>   
>   
>   SETTINGS: Settings = Settings()
> @@ -375,6 +383,15 @@ def _get_parser() -> _DTSArgumentParser:
>       )
>       _add_env_var_to_action(action, "RERUN")
>   
> +    action = parser.add_argument(
> +        "--random-seed",
> +        type=int,
> +        help="The seed to use with the pseudo-random generator. If not specified, the configuration"
> +        " value is used instead. If that's also not specified, a random seed is generated.",
> +        metavar="NUMBER",
> +    )
> +    _add_env_var_to_action(action)
> +
>       return parser
>   
>