From patchwork Wed Dec 20 10:33:28 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Juraj_Linke=C5=A1?= X-Patchwork-Id: 135389 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 08F4643723; Wed, 20 Dec 2023 11:33:52 +0100 (CET) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id BCDA940E96; Wed, 20 Dec 2023 11:33:41 +0100 (CET) Received: from mail-ed1-f43.google.com (mail-ed1-f43.google.com [209.85.208.43]) by mails.dpdk.org (Postfix) with ESMTP id 090F740A80 for ; Wed, 20 Dec 2023 11:33:40 +0100 (CET) Received: by mail-ed1-f43.google.com with SMTP id 4fb4d7f45d1cf-5532b348d30so4291860a12.1 for ; Wed, 20 Dec 2023 02:33:40 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=pantheon.tech; s=google; t=1703068419; x=1703673219; darn=dpdk.org; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:from:to:cc:subject:date :message-id:reply-to; bh=JsbpHYpWeoH4MSajR9/GoFfMDHdjZc9/Q/tLniUsi84=; b=r8ukcduDPQlY29lHAzHQGHPhxE2lOplKHzBfjNc4xZ/cceUEdDYWdgLdI30qpjyBUS ZQmacpsV0v6+WtS71W31O7ybRm54JhATMwpA/qKpsryHjvkXLLTjWMboH0g6zz9mzdVE xjmVcwqWXSR4HBk1CuAI2YQHruCIiRV+Yr/sM/fTgFHBBEvSRlxML3Dolff9t3XRHGxC DYyNEJ9iqM2PS/Ztdq1+zKomQExz5kIsp9oB6I681bRahlPZvvBnAXq+1EPaLdQLNEE1 3YGBWRwCvTWaEg75Teq3pJGw1JU/B52kNpoqKKgyfptnRrcW3dfUHQ5dXP0E7aJPfkMs QgxQ== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20230601; t=1703068419; x=1703673219; h=content-transfer-encoding:mime-version:references:in-reply-to :message-id:date:subject:cc:to:from:x-gm-message-state:from:to:cc :subject:date:message-id:reply-to; bh=JsbpHYpWeoH4MSajR9/GoFfMDHdjZc9/Q/tLniUsi84=; b=LGGRmqdUPhPpiJ7eo/wFexW09yiN/rj/CaJfIrp70x01B/ftD4lwCieVM8/FIemyLk uyAstS6OHP36FVPcFWsAs3tOMmjoH2hdscLr4On94WhZIUo58cGHS3kA8BS/JN44Srs+ 5lhUl8ZwCQQyykTC0kmmvz4o+Zu7vml/G7MCmUouwXDxwoZNNZQ6R7VS3LmjmZNGJ9KB 9+HyDpodmf8+ovWqjKm3PnKu+JV11QN1P0M35t1rxO+52C589/FO/sLUrVmbiZyVpJMA 0hTqDbrwc9njee3XpdF9haML4mjC6lD5LwgPJP1tiKG9EIlHbsLjuoFgfvENSSxx6LNd y+zw== X-Gm-Message-State: AOJu0YzoRBNXe+zySNhZLcneBgpRy6Fj6y87eVoMTondyqm+jX/A/Jlv akDkQUrkA2FlVBO9b40ftmDcRA== X-Google-Smtp-Source: AGHT+IFh6LfV2/AHAYHwcVmb5knjA2/bR81yPy9NbPP1EcGcKj4+oF3XbkwuwDYo6+WKkRAR2PoSvg== X-Received: by 2002:a50:cd55:0:b0:553:2294:8170 with SMTP id d21-20020a50cd55000000b0055322948170mr3618843edj.11.1703068419559; Wed, 20 Dec 2023 02:33:39 -0800 (PST) Received: from jlinkes-PT-Latitude-5530.pantheon.local (81.89.53.154.host.vnet.sk. [81.89.53.154]) by smtp.gmail.com with ESMTPSA id bd18-20020a056402207200b00542db304680sm12588981edb.63.2023.12.20.02.33.37 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Wed, 20 Dec 2023 02:33:38 -0800 (PST) From: =?utf-8?q?Juraj_Linke=C5=A1?= To: thomas@monjalon.net, Honnappa.Nagarahalli@arm.com, jspewock@iol.unh.edu, probb@iol.unh.edu, paul.szczepanek@arm.com, yoan.picchi@foss.arm.com, Luca.Vizzarro@arm.com Cc: dev@dpdk.org, =?utf-8?q?Juraj_Linke=C5=A1?= Subject: [RFC PATCH v1 2/5] dts: move test suite execution logic to DTSRunner Date: Wed, 20 Dec 2023 11:33:28 +0100 Message-Id: <20231220103331.60888-3-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20231220103331.60888-1-juraj.linkes@pantheon.tech> References: <20231220103331.60888-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Move the code responsible for running the test suite from the TestSuite class to the DTSRunner class. This restructuring decision was made to consolidate and unify the related logic into a single unit. Signed-off-by: Juraj Linkeš --- dts/framework/runner.py | 156 +++++++++++++++++++++++++++++++++--- dts/framework/test_suite.py | 134 +------------------------------ 2 files changed, 147 insertions(+), 143 deletions(-) diff --git a/dts/framework/runner.py b/dts/framework/runner.py index 5b077c5805..5e145a8066 100644 --- a/dts/framework/runner.py +++ b/dts/framework/runner.py @@ -5,6 +5,7 @@ import logging import sys +from types import MethodType from .config import ( BuildTargetConfiguration, @@ -12,10 +13,18 @@ ExecutionConfiguration, TestSuiteConfig, ) -from .exception import BlockingTestSuiteError +from .exception import BlockingTestSuiteError, SSHTimeoutError, TestCaseVerifyError from .logger import DTSLOG, getLogger -from .test_result import BuildTargetResult, DTSResult, ExecutionResult, Result -from .test_suite import get_test_suites +from .settings import SETTINGS +from .test_result import ( + BuildTargetResult, + DTSResult, + ExecutionResult, + Result, + TestCaseResult, + TestSuiteResult, +) +from .test_suite import TestSuite, get_test_suites from .testbed_model import SutNode, TGNode from .utils import check_dts_python_version @@ -148,7 +157,7 @@ def _run_build_target( build_target_result.update_setup(Result.FAIL, e) else: - self._run_all_suites(sut_node, tg_node, execution, build_target_result) + self._run_test_suites(sut_node, tg_node, execution, build_target_result) finally: try: @@ -158,7 +167,7 @@ def _run_build_target( self._logger.exception("Build target teardown failed.") build_target_result.update_teardown(Result.FAIL, e) - def _run_all_suites( + def _run_test_suites( self, sut_node: SutNode, tg_node: TGNode, @@ -175,7 +184,7 @@ def _run_all_suites( execution.test_suites[:0] = [TestSuiteConfig.from_dict("smoke_tests")] for test_suite_config in execution.test_suites: try: - self._run_single_suite( + self._run_test_suite( sut_node, tg_node, execution, build_target_result, test_suite_config ) except BlockingTestSuiteError as e: @@ -189,7 +198,7 @@ def _run_all_suites( if end_build_target: break - def _run_single_suite( + def _run_test_suite( self, sut_node: SutNode, tg_node: TGNode, @@ -198,6 +207,9 @@ def _run_single_suite( test_suite_config: TestSuiteConfig, ) -> None: """Runs a single test suite. + Setup, execute and teardown the whole suite. + Suite execution consists of running all test cases scheduled to be executed. + A test cast run consists of setup, execution and teardown of said test case. Args: sut_node: Node to run tests on. @@ -222,13 +234,131 @@ def _run_single_suite( else: for test_suite_class in test_suite_classes: test_suite = test_suite_class( - sut_node, - tg_node, - test_suite_config.test_cases, - execution.func, - build_target_result, + sut_node, tg_node, test_suite_config.test_cases + ) + + test_suite_name = test_suite.__class__.__name__ + test_suite_result = build_target_result.add_test_suite(test_suite_name) + try: + self._logger.info(f"Starting test suite setup: {test_suite_name}") + test_suite.set_up_suite() + test_suite_result.update_setup(Result.PASS) + self._logger.info(f"Test suite setup successful: {test_suite_name}") + except Exception as e: + self._logger.exception(f"Test suite setup ERROR: {test_suite_name}") + test_suite_result.update_setup(Result.ERROR, e) + + else: + self._execute_test_suite( + execution.func, test_suite, test_suite_result + ) + + finally: + try: + test_suite.tear_down_suite() + sut_node.kill_cleanup_dpdk_apps() + test_suite_result.update_teardown(Result.PASS) + except Exception as e: + self._logger.exception( + f"Test suite teardown ERROR: {test_suite_name}" + ) + self._logger.warning( + f"Test suite '{test_suite_name}' teardown failed, " + f"the next test suite may be affected." + ) + test_suite_result.update_setup(Result.ERROR, e) + if ( + len(test_suite_result.get_errors()) > 0 + and test_suite.is_blocking + ): + raise BlockingTestSuiteError(test_suite_name) + + def _execute_test_suite( + self, func: bool, test_suite: TestSuite, test_suite_result: TestSuiteResult + ) -> None: + """ + Execute all test cases scheduled to be executed in this suite. + """ + if func: + for test_case_method in test_suite._get_functional_test_cases(): + test_case_name = test_case_method.__name__ + test_case_result = test_suite_result.add_test_case(test_case_name) + all_attempts = SETTINGS.re_run + 1 + attempt_nr = 1 + self._run_test_case(test_suite, test_case_method, test_case_result) + while not test_case_result and attempt_nr < all_attempts: + attempt_nr += 1 + self._logger.info( + f"Re-running FAILED test case '{test_case_name}'. " + f"Attempt number {attempt_nr} out of {all_attempts}." + ) + self._run_test_case(test_suite, test_case_method, test_case_result) + + def _run_test_case( + self, + test_suite: TestSuite, + test_case_method: MethodType, + test_case_result: TestCaseResult, + ) -> None: + """ + Setup, execute and teardown a test case in this suite. + Exceptions are caught and recorded in logs and results. + """ + test_case_name = test_case_method.__name__ + + try: + # run set_up function for each case + test_suite.set_up_test_case() + test_case_result.update_setup(Result.PASS) + except SSHTimeoutError as e: + self._logger.exception(f"Test case setup FAILED: {test_case_name}") + test_case_result.update_setup(Result.FAIL, e) + except Exception as e: + self._logger.exception(f"Test case setup ERROR: {test_case_name}") + test_case_result.update_setup(Result.ERROR, e) + + else: + # run test case if setup was successful + self._execute_test_case(test_case_method, test_case_result) + + finally: + try: + test_suite.tear_down_test_case() + test_case_result.update_teardown(Result.PASS) + except Exception as e: + self._logger.exception(f"Test case teardown ERROR: {test_case_name}") + self._logger.warning( + f"Test case '{test_case_name}' teardown failed, " + f"the next test case may be affected." ) - test_suite.run() + test_case_result.update_teardown(Result.ERROR, e) + test_case_result.update(Result.ERROR) + + def _execute_test_case( + self, test_case_method: MethodType, test_case_result: TestCaseResult + ) -> None: + """ + Execute one test case and handle failures. + """ + test_case_name = test_case_method.__name__ + try: + self._logger.info(f"Starting test case execution: {test_case_name}") + test_case_method() + test_case_result.update(Result.PASS) + self._logger.info(f"Test case execution PASSED: {test_case_name}") + + except TestCaseVerifyError as e: + self._logger.exception(f"Test case execution FAILED: {test_case_name}") + test_case_result.update(Result.FAIL, e) + except Exception as e: + self._logger.exception(f"Test case execution ERROR: {test_case_name}") + test_case_result.update(Result.ERROR, e) + except KeyboardInterrupt: + self._logger.error( + f"Test case execution INTERRUPTED by user: {test_case_name}" + ) + test_case_result.update(Result.SKIP) + raise KeyboardInterrupt("Stop DTS") def _exit_dts(self) -> None: """ diff --git a/dts/framework/test_suite.py b/dts/framework/test_suite.py index 4a7907ec33..e96305deb0 100644 --- a/dts/framework/test_suite.py +++ b/dts/framework/test_suite.py @@ -17,15 +17,9 @@ from scapy.layers.l2 import Ether # type: ignore[import] from scapy.packet import Packet, Padding # type: ignore[import] -from .exception import ( - BlockingTestSuiteError, - ConfigurationError, - SSHTimeoutError, - TestCaseVerifyError, -) +from .exception import ConfigurationError, TestCaseVerifyError from .logger import DTSLOG, getLogger from .settings import SETTINGS -from .test_result import BuildTargetResult, Result, TestCaseResult, TestSuiteResult from .testbed_model import SutNode, TGNode from .testbed_model.hw.port import Port, PortLink from .utils import get_packet_summaries @@ -50,11 +44,10 @@ class TestSuite(object): """ sut_node: SutNode + tg_node: TGNode is_blocking = False _logger: DTSLOG _test_cases_to_run: list[str] - _func: bool - _result: TestSuiteResult _port_links: list[PortLink] _sut_port_ingress: Port _sut_port_egress: Port @@ -69,17 +62,13 @@ def __init__( self, sut_node: SutNode, tg_node: TGNode, - test_cases: list[str], - func: bool, - build_target_result: BuildTargetResult, + test_cases_to_run: list[str], ): self.sut_node = sut_node self.tg_node = tg_node self._logger = getLogger(self.__class__.__name__) - self._test_cases_to_run = test_cases + self._test_cases_to_run = test_cases_to_run self._test_cases_to_run.extend(SETTINGS.test_cases) - self._func = func - self._result = build_target_result.add_test_suite(self.__class__.__name__) self._port_links = [] self._process_links() self._sut_port_ingress, self._tg_port_egress = ( @@ -280,60 +269,6 @@ def _verify_l3_packet(self, received_packet: IP, expected_packet: IP) -> bool: return False return True - def run(self) -> None: - """ - Setup, execute and teardown the whole suite. - Suite execution consists of running all test cases scheduled to be executed. - A test cast run consists of setup, execution and teardown of said test case. - """ - test_suite_name = self.__class__.__name__ - - try: - self._logger.info(f"Starting test suite setup: {test_suite_name}") - self.set_up_suite() - self._result.update_setup(Result.PASS) - self._logger.info(f"Test suite setup successful: {test_suite_name}") - except Exception as e: - self._logger.exception(f"Test suite setup ERROR: {test_suite_name}") - self._result.update_setup(Result.ERROR, e) - - else: - self._execute_test_suite() - - finally: - try: - self.tear_down_suite() - self.sut_node.kill_cleanup_dpdk_apps() - self._result.update_teardown(Result.PASS) - except Exception as e: - self._logger.exception(f"Test suite teardown ERROR: {test_suite_name}") - self._logger.warning( - f"Test suite '{test_suite_name}' teardown failed, " - f"the next test suite may be affected." - ) - self._result.update_setup(Result.ERROR, e) - if len(self._result.get_errors()) > 0 and self.is_blocking: - raise BlockingTestSuiteError(test_suite_name) - - def _execute_test_suite(self) -> None: - """ - Execute all test cases scheduled to be executed in this suite. - """ - if self._func: - for test_case_method in self._get_functional_test_cases(): - test_case_name = test_case_method.__name__ - test_case_result = self._result.add_test_case(test_case_name) - all_attempts = SETTINGS.re_run + 1 - attempt_nr = 1 - self._run_test_case(test_case_method, test_case_result) - while not test_case_result and attempt_nr < all_attempts: - attempt_nr += 1 - self._logger.info( - f"Re-running FAILED test case '{test_case_name}'. " - f"Attempt number {attempt_nr} out of {all_attempts}." - ) - self._run_test_case(test_case_method, test_case_result) - def _get_functional_test_cases(self) -> list[MethodType]: """ Get all functional test cases. @@ -363,67 +298,6 @@ def _should_be_executed(self, test_case_name: str, test_case_regex: str) -> bool return match - def _run_test_case( - self, test_case_method: MethodType, test_case_result: TestCaseResult - ) -> None: - """ - Setup, execute and teardown a test case in this suite. - Exceptions are caught and recorded in logs and results. - """ - test_case_name = test_case_method.__name__ - - try: - # run set_up function for each case - self.set_up_test_case() - test_case_result.update_setup(Result.PASS) - except SSHTimeoutError as e: - self._logger.exception(f"Test case setup FAILED: {test_case_name}") - test_case_result.update_setup(Result.FAIL, e) - except Exception as e: - self._logger.exception(f"Test case setup ERROR: {test_case_name}") - test_case_result.update_setup(Result.ERROR, e) - - else: - # run test case if setup was successful - self._execute_test_case(test_case_method, test_case_result) - - finally: - try: - self.tear_down_test_case() - test_case_result.update_teardown(Result.PASS) - except Exception as e: - self._logger.exception(f"Test case teardown ERROR: {test_case_name}") - self._logger.warning( - f"Test case '{test_case_name}' teardown failed, " - f"the next test case may be affected." - ) - test_case_result.update_teardown(Result.ERROR, e) - test_case_result.update(Result.ERROR) - - def _execute_test_case( - self, test_case_method: MethodType, test_case_result: TestCaseResult - ) -> None: - """ - Execute one test case and handle failures. - """ - test_case_name = test_case_method.__name__ - try: - self._logger.info(f"Starting test case execution: {test_case_name}") - test_case_method() - test_case_result.update(Result.PASS) - self._logger.info(f"Test case execution PASSED: {test_case_name}") - - except TestCaseVerifyError as e: - self._logger.exception(f"Test case execution FAILED: {test_case_name}") - test_case_result.update(Result.FAIL, e) - except Exception as e: - self._logger.exception(f"Test case execution ERROR: {test_case_name}") - test_case_result.update(Result.ERROR, e) - except KeyboardInterrupt: - self._logger.error(f"Test case execution INTERRUPTED by user: {test_case_name}") - test_case_result.update(Result.SKIP) - raise KeyboardInterrupt("Stop DTS") - def get_test_suites(testsuite_module_path: str) -> list[type[TestSuite]]: def is_test_suite(object) -> bool: