From patchwork Tue Aug 6 12:46:39 2024 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: Luca Vizzarro X-Patchwork-Id: 142950 X-Patchwork-Delegate: juraj.linkes@pantheon.tech Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 6FDCA4574D; Tue, 6 Aug 2024 14:49:43 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id BEB4E42EF6; Tue, 6 Aug 2024 14:49:37 +0200 (CEST) Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by mails.dpdk.org (Postfix) with ESMTP id 855BA42EEF for ; Tue, 6 Aug 2024 14:49:34 +0200 (CEST) Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id C87F1113E; Tue, 6 Aug 2024 05:49:59 -0700 (PDT) Received: from localhost.localdomain (JR4XG4HTQC.cambridge.arm.com [10.1.36.46]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 2BE963F766; Tue, 6 Aug 2024 05:49:33 -0700 (PDT) From: Luca Vizzarro To: dev@dpdk.org Cc: Jeremy Spewock , =?utf-8?q?Juraj_Linke=C5=A1?= , Honnappa Nagarahalli , Luca Vizzarro , Paul Szczepanek , Alex Chapman Subject: [PATCH v2 2/5] dts: add random generation seed setting Date: Tue, 6 Aug 2024 13:46:39 +0100 Message-Id: <20240806124642.2580828-3-luca.vizzarro@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20240806124642.2580828-1-luca.vizzarro@arm.com> References: <20240806121417.2567708-1-Luca.Vizzarro@arm.com> <20240806124642.2580828-1-luca.vizzarro@arm.com> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org When introducing pseudo-random generation in the test runs we need to ensure that these can be reproduced by setting a pre-defined seed. This commits adds the ability to set one or allow for one to be generated and reported back to the user. Signed-off-by: Luca Vizzarro Reviewed-by: Paul Szczepanek Reviewed-by: Alex Chapman Reviewed-by: Jeremy Spewock Reviewed-by: Juraj Linkeš --- doc/guides/tools/dts.rst | 5 +++++ dts/framework/config/__init__.py | 4 ++++ dts/framework/config/conf_yaml_schema.json | 4 ++++ dts/framework/config/types.py | 2 ++ dts/framework/runner.py | 8 ++++++++ dts/framework/settings.py | 17 +++++++++++++++++ 6 files changed, 40 insertions(+) diff --git a/doc/guides/tools/dts.rst b/doc/guides/tools/dts.rst index 515b15e4d8..9b5ea9779c 100644 --- a/doc/guides/tools/dts.rst +++ b/doc/guides/tools/dts.rst @@ -251,6 +251,8 @@ DTS is run with ``main.py`` located in the ``dts`` directory after entering Poet ... | DTS_TEST_SUITES='suite, suite case, ...' (default: []) --re-run N_TIMES, --re_run N_TIMES [DTS_RERUN] Re-run each test case the specified number of times if a test failure occurs. (default: 0) + --random-seed NUMBER [DTS_RANDOM_SEED] The seed to use with the pseudo-random generator. If not specified, the configuration value is + used instead. If that's also not specified, a random seed is generated. (default: None) The brackets contain the names of environment variables that set the same thing. @@ -548,6 +550,9 @@ involved in the testing. These can be defined with the following mappings: +----------------------------+---------------+---------------------------------------------------+ | ``traffic_generator_node`` | Node name for the traffic generator node. | +----------------------------+-------------------------------------------------------------------+ + | ``random_seed`` | (*optional*) *int* – Allows you to set a seed for pseudo-random | + | | generation. | + +----------------------------+-------------------------------------------------------------------+ ``nodes`` `sequence `_ listing diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py index df60a5030e..269d9ec318 100644 --- a/dts/framework/config/__init__.py +++ b/dts/framework/config/__init__.py @@ -445,6 +445,7 @@ class TestRunConfiguration: system_under_test_node: The SUT node to use in this test run. traffic_generator_node: The TG node to use in this test run. vdevs: The names of virtual devices to test. + random_seed: The seed to use for pseudo-random generation. """ build_targets: list[BuildTargetConfiguration] @@ -455,6 +456,7 @@ class TestRunConfiguration: system_under_test_node: SutNodeConfiguration traffic_generator_node: TGNodeConfiguration vdevs: list[str] + random_seed: int | None @classmethod def from_dict( @@ -497,6 +499,7 @@ def from_dict( vdevs = ( d["system_under_test_node"]["vdevs"] if "vdevs" in d["system_under_test_node"] else [] ) + random_seed = d.get("random_seed", None) return cls( build_targets=build_targets, perf=d["perf"], @@ -506,6 +509,7 @@ def from_dict( system_under_test_node=system_under_test_node, traffic_generator_node=traffic_generator_node, vdevs=vdevs, + random_seed=random_seed, ) def copy_and_modify(self, **kwargs) -> Self: diff --git a/dts/framework/config/conf_yaml_schema.json b/dts/framework/config/conf_yaml_schema.json index f02a310bb5..df390e8ae2 100644 --- a/dts/framework/config/conf_yaml_schema.json +++ b/dts/framework/config/conf_yaml_schema.json @@ -379,6 +379,10 @@ }, "traffic_generator_node": { "$ref": "#/definitions/node_name" + }, + "random_seed": { + "type": "integer", + "description": "Optional field. Allows you to set a seed for pseudo-random generation." } }, "additionalProperties": false, diff --git a/dts/framework/config/types.py b/dts/framework/config/types.py index cf16556403..ce7b784ac8 100644 --- a/dts/framework/config/types.py +++ b/dts/framework/config/types.py @@ -121,6 +121,8 @@ class TestRunConfigDict(TypedDict): system_under_test_node: TestRunSUTConfigDict #: traffic_generator_node: str + #: + random_seed: int class ConfigurationDict(TypedDict): diff --git a/dts/framework/runner.py b/dts/framework/runner.py index 6b6f6a05f5..34b1dad5c4 100644 --- a/dts/framework/runner.py +++ b/dts/framework/runner.py @@ -20,6 +20,7 @@ import importlib import inspect import os +import random import re import sys from pathlib import Path @@ -147,6 +148,7 @@ def run(self) -> None: self._logger.info( f"Running test run with SUT '{test_run_config.system_under_test_node.name}'." ) + self._init_random_seed(test_run_config) test_run_result = self._result.add_test_run(test_run_config) # we don't want to modify the original config, so create a copy test_run_test_suites = list( @@ -723,3 +725,9 @@ def _exit_dts(self) -> None: self._logger.info("DTS execution has ended.") sys.exit(self._result.get_return_code()) + + def _init_random_seed(self, conf: TestRunConfiguration) -> None: + """Initialize the random seed to use for the test run.""" + seed = SETTINGS.random_seed or conf.random_seed or random.randrange(0xFFFF_FFFF) + self._logger.info(f"Initializing test run with random seed {seed}") + random.seed(seed) diff --git a/dts/framework/settings.py b/dts/framework/settings.py index f6303066d4..7744e37f54 100644 --- a/dts/framework/settings.py +++ b/dts/framework/settings.py @@ -66,6 +66,12 @@ Re-run each test case this many times in case of a failure. +.. option:: --random-seed +.. envvar:: DTS_RANDOM_SEED + + The seed to use with the pseudo-random generator. If not specified, the configuration value is + used instead. If that's also not specified, a random seed is generated. + The module provides one key module-level variable: Attributes: @@ -115,6 +121,8 @@ class Settings: test_suites: list[TestSuiteConfig] = field(default_factory=list) #: re_run: int = 0 + #: + random_seed: int | None = None SETTINGS: Settings = Settings() @@ -375,6 +383,15 @@ def _get_parser() -> _DTSArgumentParser: ) _add_env_var_to_action(action, "RERUN") + action = parser.add_argument( + "--random-seed", + type=int, + help="The seed to use with the pseudo-random generator. If not specified, the configuration" + " value is used instead. If that's also not specified, a random seed is generated.", + metavar="NUMBER", + ) + _add_env_var_to_action(action) + return parser