From patchwork Thu Apr 13 17:54:15 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Jeremy Spewock X-Patchwork-Id: 126036 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id CA81E42935; Thu, 13 Apr 2023 19:54:55 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 189AF42BC9; Thu, 13 Apr 2023 19:54:50 +0200 (CEST) Received: from mail-io1-f100.google.com (mail-io1-f100.google.com [209.85.166.100]) by mails.dpdk.org (Postfix) with ESMTP id 9BEA0410F9 for ; Thu, 13 Apr 2023 19:54:48 +0200 (CEST) Received: by mail-io1-f100.google.com with SMTP id g16so15807903iom.11 for ; Thu, 13 Apr 2023 10:54:48 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=iol.unh.edu; s=unh-iol; t=1681408488; x=1684000488; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=SL04xfpHqgg7ELzkmRjjxotTW4wgL59vZA417jvyu+A=; b=Vj8T4n6voC0S274jJlrMi6HUC9oofFTwEBNarMejQ16pvS/Ic3xugVYL+nG4lRWw9s dMSeXk3lSKRncGfWxIlNqjnflg6g3YK8KS0H5zXisUbmN2Ck11pi2aRzMafLsXzeaWtg JwDB0XP70cmEHNRZd0gN/2bZgodj8wIEStG2g= X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1681408488; x=1684000488; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=SL04xfpHqgg7ELzkmRjjxotTW4wgL59vZA417jvyu+A=; b=NDPq1Y+xdrKCWuOqzGpVoq6nb+d1XlraPePYUE6LDkQB68QnP9IHc1EudLOv/H0SD6 pYIv5oqIhWnTGJORywvxu/lT5vdS928TkQd3ATeaG42iUU4CgfuzqjnQNb8XsuMeWD0k tWW8Htfs2jZxgKIBILNlCvd6ow4LdUjOc/g6hN3KH33SMF6L6ziBpGRtsDwGdPpnyw5c +YT6wD4POJgbqSBHuwXuVHH1nfbDNSvPkoeotBZe0lre5QNVDYXKfKLnak0uNKcyVbeP 6havuft4wZYl1E1xQ3jKKrLDs0xVv90szQGwy3PCnVuJaJxQFd2YtCB+wH0lCRL/H6oJ bAyQ== X-Gm-Message-State: AAQBX9c5KxP39PgWUEmdC7/g5f9Y9M4DzzM6nrfj2BMAeODi9rvaOAoh fagMS8zpq5vWGEf7yOzn8oSfzsTAGne1uKKFjmakmX0502SdBR/D30OWHZ2I1GV1T89ywz4K6SI nklfza9xnRBDR0hNdP5r94g3UTYFKn+NM5sal1LYvk8wZw9OUBd8r4AmSdAbxOA5/SjwolwchtF 0DvOAkXoc4eUjUJGqJgDiCfA== X-Google-Smtp-Source: AKy350bol9kLTXOtG6dRVcPfQbe59Lf+SYC8R0NIRqI45IRvKUGvTvD/xMreHOUjIi6bci9CQjtBOcJomVz1 X-Received: by 2002:a6b:5b02:0:b0:760:b712:96d0 with SMTP id v2-20020a6b5b02000000b00760b71296d0mr1950691ioh.8.1681408487915; Thu, 13 Apr 2023 10:54:47 -0700 (PDT) Received: from postal.iol.unh.edu (postal.iol.unh.edu. [2606:4100:3880:1234::84]) by smtp-relay.gmail.com with ESMTPS id a19-20020a056602149300b00760af5dcf0esm422794iow.0.2023.04.13.10.54.47 (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128); Thu, 13 Apr 2023 10:54:47 -0700 (PDT) X-Relaying-Domain: iol.unh.edu Received: from iol.unh.edu (unknown [IPv6:2606:4100:3880:1220:58fe:317e:37bd:a524]) by postal.iol.unh.edu (Postfix) with ESMTP id 64EC2605246B; Thu, 13 Apr 2023 13:54:47 -0400 (EDT) From: jspewock@iol.unh.edu To: dev@dpdk.org Cc: Jeremy Spewock Subject: [RFC PATCH 1/1] dts: add smoke tests Date: Thu, 13 Apr 2023 13:54:15 -0400 Message-Id: <20230413175415.7683-3-jspewock@iol.unh.edu> X-Mailer: git-send-email 2.40.0 In-Reply-To: <20230413175415.7683-2-jspewock@iol.unh.edu> References: <20230413175415.7683-2-jspewock@iol.unh.edu> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org From: Jeremy Spewock Adds a new test suite for running smoke tests that verify general configuration aspects of the system under test. If any of these tests fail, the DTS execution terminates as part of a "fail-fast" model. Signed-off-by: Jeremy Spewock --- dts/conf.yaml | 7 ++- dts/framework/config/__init__.py | 15 ++++++ dts/framework/config/conf_yaml_schema.json | 16 +++++- dts/framework/dts.py | 19 ++++++- dts/framework/exception.py | 11 ++++ dts/framework/test_result.py | 13 +++-- dts/framework/test_suite.py | 24 ++++++++- dts/tests/TestSuite_smoke_tests.py | 63 ++++++++++++++++++++++ 8 files changed, 159 insertions(+), 9 deletions(-) create mode 100644 dts/tests/TestSuite_smoke_tests.py diff --git a/dts/conf.yaml b/dts/conf.yaml index a9bd8a3e..8caef719 100644 --- a/dts/conf.yaml +++ b/dts/conf.yaml @@ -10,13 +10,18 @@ executions: compiler_wrapper: ccache perf: false func: true + nic: #physical devices to be used for testing + address: "0000:00:10.1" + driver: "vfio-pci" test_suites: + - smoke_tests - hello_world system_under_test: "SUT 1" nodes: - name: "SUT 1" - hostname: sut1.change.me.localhost + hostname: host.example.name user: root + password: "" arch: x86_64 os: linux lcores: "" diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py index ebb0823f..78d23422 100644 --- a/dts/framework/config/__init__.py +++ b/dts/framework/config/__init__.py @@ -106,6 +106,18 @@ def from_dict(d: dict) -> "NodeConfiguration": hugepages=hugepage_config, ) +@dataclass(slots=True, frozen=True) +class NICConfiguration: + address: str + driver: str + + @staticmethod + def from_dict(d:dict) -> "NICConfiguration": + return NICConfiguration( + address=d.get("address"), + driver=d.get("driver") + ) + @dataclass(slots=True, frozen=True) class BuildTargetConfiguration: @@ -157,6 +169,7 @@ class ExecutionConfiguration: func: bool test_suites: list[TestSuiteConfig] system_under_test: NodeConfiguration + nic: NICConfiguration @staticmethod def from_dict(d: dict, node_map: dict) -> "ExecutionConfiguration": @@ -166,6 +179,7 @@ def from_dict(d: dict, node_map: dict) -> "ExecutionConfiguration": test_suites: list[TestSuiteConfig] = list( map(TestSuiteConfig.from_dict, d["test_suites"]) ) + nic_conf: NICConfiguration = NICConfiguration.from_dict(d['nic']) sut_name = d["system_under_test"] assert sut_name in node_map, f"Unknown SUT {sut_name} in execution {d}" @@ -174,6 +188,7 @@ def from_dict(d: dict, node_map: dict) -> "ExecutionConfiguration": perf=d["perf"], func=d["func"], test_suites=test_suites, + nic=nic_conf, system_under_test=node_map[sut_name], ) diff --git a/dts/framework/config/conf_yaml_schema.json b/dts/framework/config/conf_yaml_schema.json index ca2d4a1e..c98a9370 100644 --- a/dts/framework/config/conf_yaml_schema.json +++ b/dts/framework/config/conf_yaml_schema.json @@ -97,7 +97,8 @@ "test_suite": { "type": "string", "enum": [ - "hello_world" + "hello_world", + "smoke_tests" ] }, "test_target": { @@ -211,6 +212,19 @@ ] } }, + "nic": { + "type": "object", + "properties": { + "address": { + "type":"string", + "description": "PCI address of a physical device to test" + }, + "driver": { + "type": "string", + "description": "The name of the driver that the physical device should be bound to" + } + } + }, "system_under_test": { "$ref": "#/definitions/node_name" } diff --git a/dts/framework/dts.py b/dts/framework/dts.py index 05022845..0d03e158 100644 --- a/dts/framework/dts.py +++ b/dts/framework/dts.py @@ -5,6 +5,8 @@ import sys +from .exception import BlockingTestSuiteError + from .config import CONFIGURATION, BuildTargetConfiguration, ExecutionConfiguration from .logger import DTSLOG, getLogger from .test_result import BuildTargetResult, DTSResult, ExecutionResult, Result @@ -49,6 +51,7 @@ def run_all() -> None: nodes[sut_node.name] = sut_node if sut_node: + #SMOKE TEST EXECUTION GOES HERE! _run_execution(sut_node, execution, result) except Exception as e: @@ -118,7 +121,7 @@ def _run_build_target( try: sut_node.set_up_build_target(build_target) - result.dpdk_version = sut_node.dpdk_version + # result.dpdk_version = sut_node.dpdk_version build_target_result.update_setup(Result.PASS) except Exception as e: dts_logger.exception("Build target setup failed.") @@ -146,6 +149,7 @@ def _run_suites( with possibly only a subset of test cases. If no subset is specified, run all test cases. """ + end_execution = False for test_suite_config in execution.test_suites: try: full_suite_path = f"tests.TestSuite_{test_suite_config.test_suite}" @@ -160,13 +164,24 @@ def _run_suites( else: for test_suite_class in test_suite_classes: + #HERE NEEDS CHANGING test_suite = test_suite_class( sut_node, test_suite_config.test_cases, execution.func, build_target_result, + sut_node._build_target_config, + result ) - test_suite.run() + try: + test_suite.run() + except BlockingTestSuiteError as e: + dts_logger.exception("An error occurred within a blocking TestSuite, execution will now end.") + result.add_error(e) + end_execution = True + #if a blocking test failed and we need to bail out of suite executions + if end_execution: + break def _exit_dts() -> None: diff --git a/dts/framework/exception.py b/dts/framework/exception.py index ca353d98..4e3f63d1 100644 --- a/dts/framework/exception.py +++ b/dts/framework/exception.py @@ -25,6 +25,7 @@ class ErrorSeverity(IntEnum): SSH_ERR = 4 DPDK_BUILD_ERR = 10 TESTCASE_VERIFY_ERR = 20 + BLOCKING_TESTSUITE_ERR = 25 class DTSError(Exception): @@ -144,3 +145,13 @@ def __init__(self, value: str): def __str__(self) -> str: return repr(self.value) + +class BlockingTestSuiteError(DTSError): + suite_name: str + severity: ClassVar[ErrorSeverity] = ErrorSeverity.BLOCKING_TESTSUITE_ERR + + def __init__(self, suite_name:str) -> None: + self.suite_name = suite_name + + def __str__(self) -> str: + return f"Blocking suite {self.suite_name} failed." diff --git a/dts/framework/test_result.py b/dts/framework/test_result.py index 74391982..77202ae2 100644 --- a/dts/framework/test_result.py +++ b/dts/framework/test_result.py @@ -8,6 +8,7 @@ import os.path from collections.abc import MutableSequence from enum import Enum, auto +from typing import Dict from .config import ( OS, @@ -67,12 +68,13 @@ class Statistics(dict): Using a dict provides a convenient way to format the data. """ - def __init__(self, dpdk_version): + def __init__(self, output_info: Dict[str, str] | None): super(Statistics, self).__init__() for result in Result: self[result.name] = 0 self["PASS RATE"] = 0.0 - self["DPDK VERSION"] = dpdk_version + if output_info: + for info_key, info_val in output_info.items(): self[info_key] = info_val def __iadd__(self, other: Result) -> "Statistics": """ @@ -258,6 +260,7 @@ class DTSResult(BaseResult): """ dpdk_version: str | None + output: dict | None _logger: DTSLOG _errors: list[Exception] _return_code: ErrorSeverity @@ -267,6 +270,7 @@ class DTSResult(BaseResult): def __init__(self, logger: DTSLOG): super(DTSResult, self).__init__() self.dpdk_version = None + self.output = None self._logger = logger self._errors = [] self._return_code = ErrorSeverity.NO_ERR @@ -296,7 +300,10 @@ def process(self) -> None: for error in self._errors: self._logger.debug(repr(error)) - self._stats_result = Statistics(self.dpdk_version) + self._stats_result = Statistics(self.output) + #add information gathered from the smoke tests to the statistics + # for info_key, info_val in smoke_test_info.items(): self._stats_result[info_key] = info_val + # print(self._stats_result) self.add_stats(self._stats_result) with open(self._stats_filename, "w+") as stats_file: stats_file.write(str(self._stats_result)) diff --git a/dts/framework/test_suite.py b/dts/framework/test_suite.py index 0705f38f..1518fb8a 100644 --- a/dts/framework/test_suite.py +++ b/dts/framework/test_suite.py @@ -10,11 +10,14 @@ import inspect import re from types import MethodType +from typing import Dict -from .exception import ConfigurationError, SSHTimeoutError, TestCaseVerifyError +from .config import BuildTargetConfiguration + +from .exception import BlockingTestSuiteError, ConfigurationError, SSHTimeoutError, TestCaseVerifyError from .logger import DTSLOG, getLogger from .settings import SETTINGS -from .test_result import BuildTargetResult, Result, TestCaseResult, TestSuiteResult +from .test_result import BuildTargetResult, DTSResult, Result, TestCaseResult, TestSuiteResult from .testbed_model import SutNode @@ -37,10 +40,12 @@ class TestSuite(object): """ sut_node: SutNode + is_blocking = False _logger: DTSLOG _test_cases_to_run: list[str] _func: bool _result: TestSuiteResult + _dts_result: DTSResult def __init__( self, @@ -48,6 +53,8 @@ def __init__( test_cases: list[str], func: bool, build_target_result: BuildTargetResult, + build_target_conf: BuildTargetConfiguration, + dts_result: DTSResult ): self.sut_node = sut_node self._logger = getLogger(self.__class__.__name__) @@ -55,6 +62,8 @@ def __init__( self._test_cases_to_run.extend(SETTINGS.test_cases) self._func = func self._result = build_target_result.add_test_suite(self.__class__.__name__) + self.build_target_info = build_target_conf + self._dts_result = dts_result def set_up_suite(self) -> None: """ @@ -118,6 +127,9 @@ def run(self) -> None: f"the next test suite may be affected." ) self._result.update_setup(Result.ERROR, e) + if len(self._result.get_errors()) > 0 and self.is_blocking: + raise BlockingTestSuiteError(test_suite_name) + def _execute_test_suite(self) -> None: """ @@ -137,6 +149,7 @@ def _execute_test_suite(self) -> None: f"Attempt number {attempt_nr} out of {all_attempts}." ) self._run_test_case(test_case_method, test_case_result) + def _get_functional_test_cases(self) -> list[MethodType]: """ @@ -232,6 +245,11 @@ def _execute_test_case( test_case_result.update(Result.SKIP) raise KeyboardInterrupt("Stop DTS") + def write_to_statistics_file(self, output: Dict[str, str]): + if self._dts_result.output != None: + self._dts_result.output.update(output) + else: + self._dts_result.output = output def get_test_suites(testsuite_module_path: str) -> list[type[TestSuite]]: def is_test_suite(object) -> bool: @@ -252,3 +270,5 @@ def is_test_suite(object) -> bool: test_suite_class for _, test_suite_class in inspect.getmembers(testcase_module, is_test_suite) ] + + diff --git a/dts/tests/TestSuite_smoke_tests.py b/dts/tests/TestSuite_smoke_tests.py new file mode 100644 index 00000000..b661f850 --- /dev/null +++ b/dts/tests/TestSuite_smoke_tests.py @@ -0,0 +1,63 @@ +from framework.test_suite import TestSuite +from framework.testbed_model.sut_node import SutNode + + +def get_compiler_version(compiler_name: str, sut_node: SutNode) -> str: + match compiler_name: + case "gcc": + return sut_node.main_session.send_command(f"{compiler_name} --version", 60).stdout.split("\n")[0] + case "clang": + return sut_node.main_session.send_command(f"{compiler_name} --version", 60).stdout.split("\n")[0] + case "msvc": + return sut_node.main_session.send_command(f"cl", 60).stdout + case "icc": + return sut_node.main_session.send_command(f"{compiler_name} -V", 60).stdout + +class SmokeTests(TestSuite): + is_blocking = True + + + def set_up_suite(self) -> None: + """ + Setup: + build all DPDK + """ + self.dpdk_build_dir_path = self.sut_node.remote_dpdk_build_dir + + + def test_unit_tests(self) -> None: + """ + Test: + run the fast-test unit-test suite through meson + """ + self.sut_node.main_session.send_command(f"meson test -C {self.dpdk_build_dir_path} --suite fast-tests", 300, True) + + def test_driver_tests(self) -> None: + """ + Test: + run the driver-test unit-test suite through meson + """ + self.sut_node.main_session.send_command(f"meson test -C {self.dpdk_build_dir_path} --suite driver-tests", 300) + + def test_gather_info(self) -> None: + out = {} + + out['OS'] = self.sut_node.main_session.send_command("awk -F= '$1==\"NAME\" {print $2}' /etc/os-release", 60).stdout + out["OS VERSION"] = self.sut_node.main_session.send_command("awk -F= '$1==\"VERSION\" {print $2}' /etc/os-release", 60, True).stdout + out["COMPILER VERSION"] = get_compiler_version(self.build_target_info.compiler.name, self.sut_node) + out["DPDK VERSION"] = self.sut_node.dpdk_version + if self.build_target_info.os.name == "linux": + out['KERNEL VERSION'] = self.sut_node.main_session.send_command("uname -r", 60).stdout + elif self.build_target_info.os.name == "windows": + out['KERNEL VERSION'] = self.sut_node.main_session.send_command("uname -a", 60).stdout + self.write_to_statistics_file(out) + + def test_start_testpmd(self) -> None: + """ + Still heavily in development + """ + self.sut_node.main_session.send_command(f"{self.dpdk_build_dir_path}/app/dpdk-testpmd -- -i", 60) + out = self.sut_node.main_session.send_command("show port summary all") + self.sut_node.main_session.send_command("quit") + print(out.stdout) + \ No newline at end of file