[v2,4/7] dts: reorganize test result

Message ID 20240206145716.71435-5-juraj.linkes@pantheon.tech (mailing list archive)
State New
Delegated to: Thomas Monjalon
Headers
Series test case blocking and logging |

Checks

Context Check Description
ci/checkpatch success coding style OK

Commit Message

Juraj Linkeš Feb. 6, 2024, 2:57 p.m. UTC
  The current order of Result classes in the test_suite.py module is
guided by the needs of type hints, which is not as intuitively readable
as ordering them by the occurrences in code. The order goes from the
topmost level to lowermost:
BaseResult
DTSResult
ExecutionResult
BuildTargetResult
TestSuiteResult
TestCaseResult

This is the same order as they're used in the runner module and they're
also used in the same order between themselves in the test_result
module.

Signed-off-by: Juraj Linkeš <juraj.linkes@pantheon.tech>
---
 dts/framework/test_result.py | 411 ++++++++++++++++++-----------------
 1 file changed, 206 insertions(+), 205 deletions(-)
  

Patch

diff --git a/dts/framework/test_result.py b/dts/framework/test_result.py
index 075195fd5b..abdbafab10 100644
--- a/dts/framework/test_result.py
+++ b/dts/framework/test_result.py
@@ -28,6 +28,7 @@ 
 from dataclasses import dataclass
 from enum import Enum, auto
 from types import MethodType
+from typing import Union
 
 from .config import (
     OS,
@@ -129,58 +130,6 @@  def __bool__(self) -> bool:
         return bool(self.result)
 
 
-class Statistics(dict):
-    """How many test cases ended in which result state along some other basic information.
-
-    Subclassing :class:`dict` provides a convenient way to format the data.
-
-    The data are stored in the following keys:
-
-    * **PASS RATE** (:class:`int`) -- The FAIL/PASS ratio of all test cases.
-    * **DPDK VERSION** (:class:`str`) -- The tested DPDK version.
-    """
-
-    def __init__(self, dpdk_version: str | None):
-        """Extend the constructor with keys in which the data are stored.
-
-        Args:
-            dpdk_version: The version of tested DPDK.
-        """
-        super(Statistics, self).__init__()
-        for result in Result:
-            self[result.name] = 0
-        self["PASS RATE"] = 0.0
-        self["DPDK VERSION"] = dpdk_version
-
-    def __iadd__(self, other: Result) -> "Statistics":
-        """Add a Result to the final count.
-
-        Example:
-            stats: Statistics = Statistics()  # empty Statistics
-            stats += Result.PASS  # add a Result to `stats`
-
-        Args:
-            other: The Result to add to this statistics object.
-
-        Returns:
-            The modified statistics object.
-        """
-        self[other.name] += 1
-        self["PASS RATE"] = (
-            float(self[Result.PASS.name]) * 100 / sum(self[result.name] for result in Result)
-        )
-        return self
-
-    def __str__(self) -> str:
-        """Each line contains the formatted key = value pair."""
-        stats_str = ""
-        for key, value in self.items():
-            stats_str += f"{key:<12} = {value}\n"
-            # according to docs, we should use \n when writing to text files
-            # on all platforms
-        return stats_str
-
-
 class BaseResult(object):
     """Common data and behavior of DTS results.
 
@@ -245,7 +194,7 @@  def get_errors(self) -> list[Exception]:
         """
         return self._get_setup_teardown_errors() + self._get_inner_errors()
 
-    def add_stats(self, statistics: Statistics) -> None:
+    def add_stats(self, statistics: "Statistics") -> None:
         """Collate stats from the whole result hierarchy.
 
         Args:
@@ -255,91 +204,149 @@  def add_stats(self, statistics: Statistics) -> None:
             inner_result.add_stats(statistics)
 
 
-class TestCaseResult(BaseResult, FixtureResult):
-    r"""The test case specific result.
+class DTSResult(BaseResult):
+    """Stores environment information and test results from a DTS run.
 
-    Stores the result of the actual test case. This is done by adding an extra superclass
-    in :class:`FixtureResult`. The setup and teardown results are :class:`FixtureResult`\s and
-    the class is itself a record of the test case.
+        * Execution level information, such as testbed and the test suite list,
+        * Build target level information, such as compiler, target OS and cpu,
+        * Test suite and test case results,
+        * All errors that are caught and recorded during DTS execution.
+
+    The information is stored hierarchically. This is the first level of the hierarchy
+    and as such is where the data form the whole hierarchy is collated or processed.
+
+    The internal list stores the results of all executions.
 
     Attributes:
-        test_case_name: The test case name.
+        dpdk_version: The DPDK version to record.
     """
 
-    test_case_name: str
+    dpdk_version: str | None
+    _logger: DTSLOG
+    _errors: list[Exception]
+    _return_code: ErrorSeverity
+    _stats_result: Union["Statistics", None]
+    _stats_filename: str
 
-    def __init__(self, test_case_name: str):
-        """Extend the constructor with `test_case_name`.
+    def __init__(self, logger: DTSLOG):
+        """Extend the constructor with top-level specifics.
 
         Args:
-            test_case_name: The test case's name.
+            logger: The logger instance the whole result will use.
         """
-        super(TestCaseResult, self).__init__()
-        self.test_case_name = test_case_name
+        super(DTSResult, self).__init__()
+        self.dpdk_version = None
+        self._logger = logger
+        self._errors = []
+        self._return_code = ErrorSeverity.NO_ERR
+        self._stats_result = None
+        self._stats_filename = os.path.join(SETTINGS.output_dir, "statistics.txt")
 
-    def update(self, result: Result, error: Exception | None = None) -> None:
-        """Update the test case result.
+    def add_execution(self, sut_node: NodeConfiguration) -> "ExecutionResult":
+        """Add and return the inner result (execution).
 
-        This updates the result of the test case itself and doesn't affect
-        the results of the setup and teardown steps in any way.
+        Args:
+            sut_node: The SUT node's test run configuration.
+
+        Returns:
+            The execution's result.
+        """
+        execution_result = ExecutionResult(sut_node)
+        self._inner_results.append(execution_result)
+        return execution_result
+
+    def add_error(self, error: Exception) -> None:
+        """Record an error that occurred outside any execution.
 
         Args:
-            result: The result of the test case.
-            error: The error that occurred in case of a failure.
+            error: The exception to record.
         """
-        self.result = result
-        self.error = error
+        self._errors.append(error)
 
-    def _get_inner_errors(self) -> list[Exception]:
-        if self.error:
-            return [self.error]
-        return []
+    def process(self) -> None:
+        """Process the data after a whole DTS run.
 
-    def add_stats(self, statistics: Statistics) -> None:
-        r"""Add the test case result to statistics.
+        The data is added to inner objects during runtime and this object is not updated
+        at that time. This requires us to process the inner data after it's all been gathered.
 
-        The base method goes through the hierarchy recursively and this method is here to stop
-        the recursion, as the :class:`TestCaseResult`\s are the leaves of the hierarchy tree.
+        The processing gathers all errors and the statistics of test case results.
+        """
+        self._errors += self.get_errors()
+        if self._errors and self._logger:
+            self._logger.debug("Summary of errors:")
+            for error in self._errors:
+                self._logger.debug(repr(error))
 
-        Args:
-            statistics: The :class:`Statistics` object where the stats will be added.
+        self._stats_result = Statistics(self.dpdk_version)
+        self.add_stats(self._stats_result)
+        with open(self._stats_filename, "w+") as stats_file:
+            stats_file.write(str(self._stats_result))
+
+    def get_return_code(self) -> int:
+        """Go through all stored Exceptions and return the final DTS error code.
+
+        Returns:
+            The highest error code found.
         """
-        statistics += self.result
+        for error in self._errors:
+            error_return_code = ErrorSeverity.GENERIC_ERR
+            if isinstance(error, DTSError):
+                error_return_code = error.severity
 
-    def __bool__(self) -> bool:
-        """The test case passed only if setup, teardown and the test case itself passed."""
-        return bool(self.setup_result) and bool(self.teardown_result) and bool(self.result)
+            if error_return_code > self._return_code:
+                self._return_code = error_return_code
 
+        return int(self._return_code)
 
-class TestSuiteResult(BaseResult):
-    """The test suite specific result.
 
-    The internal list stores the results of all test cases in a given test suite.
+class ExecutionResult(BaseResult):
+    """The execution specific result.
+
+    The internal list stores the results of all build targets in a given execution.
 
     Attributes:
-        suite_name: The test suite name.
+        sut_node: The SUT node used in the execution.
+        sut_os_name: The operating system of the SUT node.
+        sut_os_version: The operating system version of the SUT node.
+        sut_kernel_version: The operating system kernel version of the SUT node.
     """
 
-    suite_name: str
+    sut_node: NodeConfiguration
+    sut_os_name: str
+    sut_os_version: str
+    sut_kernel_version: str
 
-    def __init__(self, suite_name: str):
-        """Extend the constructor with `suite_name`.
+    def __init__(self, sut_node: NodeConfiguration):
+        """Extend the constructor with the `sut_node`'s config.
 
         Args:
-            suite_name: The test suite's name.
+            sut_node: The SUT node's test run configuration used in the execution.
         """
-        super(TestSuiteResult, self).__init__()
-        self.suite_name = suite_name
+        super(ExecutionResult, self).__init__()
+        self.sut_node = sut_node
 
-    def add_test_case(self, test_case_name: str) -> TestCaseResult:
-        """Add and return the inner result (test case).
+    def add_build_target(self, build_target: BuildTargetConfiguration) -> "BuildTargetResult":
+        """Add and return the inner result (build target).
+
+        Args:
+            build_target: The build target's test run configuration.
 
         Returns:
-            The test case's result.
+            The build target's result.
         """
-        test_case_result = TestCaseResult(test_case_name)
-        self._inner_results.append(test_case_result)
-        return test_case_result
+        build_target_result = BuildTargetResult(build_target)
+        self._inner_results.append(build_target_result)
+        return build_target_result
+
+    def add_sut_info(self, sut_info: NodeInfo) -> None:
+        """Add SUT information gathered at runtime.
+
+        Args:
+            sut_info: The additional SUT node information.
+        """
+        self.sut_os_name = sut_info.os_name
+        self.sut_os_version = sut_info.os_version
+        self.sut_kernel_version = sut_info.kernel_version
 
 
 class BuildTargetResult(BaseResult):
@@ -386,7 +393,7 @@  def add_build_target_info(self, versions: BuildTargetInfo) -> None:
         self.compiler_version = versions.compiler_version
         self.dpdk_version = versions.dpdk_version
 
-    def add_test_suite(self, test_suite_name: str) -> TestSuiteResult:
+    def add_test_suite(self, test_suite_name: str) -> "TestSuiteResult":
         """Add and return the inner result (test suite).
 
         Returns:
@@ -397,146 +404,140 @@  def add_test_suite(self, test_suite_name: str) -> TestSuiteResult:
         return test_suite_result
 
 
-class ExecutionResult(BaseResult):
-    """The execution specific result.
+class TestSuiteResult(BaseResult):
+    """The test suite specific result.
 
-    The internal list stores the results of all build targets in a given execution.
+    The internal list stores the results of all test cases in a given test suite.
 
     Attributes:
-        sut_node: The SUT node used in the execution.
-        sut_os_name: The operating system of the SUT node.
-        sut_os_version: The operating system version of the SUT node.
-        sut_kernel_version: The operating system kernel version of the SUT node.
+        suite_name: The test suite name.
     """
 
-    sut_node: NodeConfiguration
-    sut_os_name: str
-    sut_os_version: str
-    sut_kernel_version: str
+    suite_name: str
 
-    def __init__(self, sut_node: NodeConfiguration):
-        """Extend the constructor with the `sut_node`'s config.
+    def __init__(self, suite_name: str):
+        """Extend the constructor with `suite_name`.
 
         Args:
-            sut_node: The SUT node's test run configuration used in the execution.
+            suite_name: The test suite's name.
         """
-        super(ExecutionResult, self).__init__()
-        self.sut_node = sut_node
-
-    def add_build_target(self, build_target: BuildTargetConfiguration) -> BuildTargetResult:
-        """Add and return the inner result (build target).
+        super(TestSuiteResult, self).__init__()
+        self.suite_name = suite_name
 
-        Args:
-            build_target: The build target's test run configuration.
+    def add_test_case(self, test_case_name: str) -> "TestCaseResult":
+        """Add and return the inner result (test case).
 
         Returns:
-            The build target's result.
-        """
-        build_target_result = BuildTargetResult(build_target)
-        self._inner_results.append(build_target_result)
-        return build_target_result
-
-    def add_sut_info(self, sut_info: NodeInfo) -> None:
-        """Add SUT information gathered at runtime.
-
-        Args:
-            sut_info: The additional SUT node information.
+            The test case's result.
         """
-        self.sut_os_name = sut_info.os_name
-        self.sut_os_version = sut_info.os_version
-        self.sut_kernel_version = sut_info.kernel_version
+        test_case_result = TestCaseResult(test_case_name)
+        self._inner_results.append(test_case_result)
+        return test_case_result
 
 
-class DTSResult(BaseResult):
-    """Stores environment information and test results from a DTS run.
-
-        * Execution level information, such as testbed and the test suite list,
-        * Build target level information, such as compiler, target OS and cpu,
-        * Test suite and test case results,
-        * All errors that are caught and recorded during DTS execution.
-
-    The information is stored hierarchically. This is the first level of the hierarchy
-    and as such is where the data form the whole hierarchy is collated or processed.
+class TestCaseResult(BaseResult, FixtureResult):
+    r"""The test case specific result.
 
-    The internal list stores the results of all executions.
+    Stores the result of the actual test case. This is done by adding an extra superclass
+    in :class:`FixtureResult`. The setup and teardown results are :class:`FixtureResult`\s and
+    the class is itself a record of the test case.
 
     Attributes:
-        dpdk_version: The DPDK version to record.
+        test_case_name: The test case name.
     """
 
-    dpdk_version: str | None
-    _logger: DTSLOG
-    _errors: list[Exception]
-    _return_code: ErrorSeverity
-    _stats_result: Statistics | None
-    _stats_filename: str
+    test_case_name: str
 
-    def __init__(self, logger: DTSLOG):
-        """Extend the constructor with top-level specifics.
+    def __init__(self, test_case_name: str):
+        """Extend the constructor with `test_case_name`.
 
         Args:
-            logger: The logger instance the whole result will use.
+            test_case_name: The test case's name.
         """
-        super(DTSResult, self).__init__()
-        self.dpdk_version = None
-        self._logger = logger
-        self._errors = []
-        self._return_code = ErrorSeverity.NO_ERR
-        self._stats_result = None
-        self._stats_filename = os.path.join(SETTINGS.output_dir, "statistics.txt")
+        super(TestCaseResult, self).__init__()
+        self.test_case_name = test_case_name
 
-    def add_execution(self, sut_node: NodeConfiguration) -> ExecutionResult:
-        """Add and return the inner result (execution).
+    def update(self, result: Result, error: Exception | None = None) -> None:
+        """Update the test case result.
 
-        Args:
-            sut_node: The SUT node's test run configuration.
+        This updates the result of the test case itself and doesn't affect
+        the results of the setup and teardown steps in any way.
 
-        Returns:
-            The execution's result.
+        Args:
+            result: The result of the test case.
+            error: The error that occurred in case of a failure.
         """
-        execution_result = ExecutionResult(sut_node)
-        self._inner_results.append(execution_result)
-        return execution_result
+        self.result = result
+        self.error = error
 
-    def add_error(self, error: Exception) -> None:
-        """Record an error that occurred outside any execution.
+    def _get_inner_errors(self) -> list[Exception]:
+        if self.error:
+            return [self.error]
+        return []
+
+    def add_stats(self, statistics: "Statistics") -> None:
+        r"""Add the test case result to statistics.
+
+        The base method goes through the hierarchy recursively and this method is here to stop
+        the recursion, as the :class:`TestCaseResult`\s are the leaves of the hierarchy tree.
 
         Args:
-            error: The exception to record.
+            statistics: The :class:`Statistics` object where the stats will be added.
         """
-        self._errors.append(error)
+        statistics += self.result
 
-    def process(self) -> None:
-        """Process the data after a whole DTS run.
+    def __bool__(self) -> bool:
+        """The test case passed only if setup, teardown and the test case itself passed."""
+        return bool(self.setup_result) and bool(self.teardown_result) and bool(self.result)
 
-        The data is added to inner objects during runtime and this object is not updated
-        at that time. This requires us to process the inner data after it's all been gathered.
 
-        The processing gathers all errors and the statistics of test case results.
+class Statistics(dict):
+    """How many test cases ended in which result state along some other basic information.
+
+    Subclassing :class:`dict` provides a convenient way to format the data.
+
+    The data are stored in the following keys:
+
+    * **PASS RATE** (:class:`int`) -- The FAIL/PASS ratio of all test cases.
+    * **DPDK VERSION** (:class:`str`) -- The tested DPDK version.
+    """
+
+    def __init__(self, dpdk_version: str | None):
+        """Extend the constructor with keys in which the data are stored.
+
+        Args:
+            dpdk_version: The version of tested DPDK.
         """
-        self._errors += self.get_errors()
-        if self._errors and self._logger:
-            self._logger.debug("Summary of errors:")
-            for error in self._errors:
-                self._logger.debug(repr(error))
+        super(Statistics, self).__init__()
+        for result in Result:
+            self[result.name] = 0
+        self["PASS RATE"] = 0.0
+        self["DPDK VERSION"] = dpdk_version
 
-        self._stats_result = Statistics(self.dpdk_version)
-        self.add_stats(self._stats_result)
-        with open(self._stats_filename, "w+") as stats_file:
-            stats_file.write(str(self._stats_result))
+    def __iadd__(self, other: Result) -> "Statistics":
+        """Add a Result to the final count.
 
-    def get_return_code(self) -> int:
-        """Go through all stored Exceptions and return the final DTS error code.
+        Example:
+            stats: Statistics = Statistics()  # empty Statistics
+            stats += Result.PASS  # add a Result to `stats`
+
+        Args:
+            other: The Result to add to this statistics object.
 
         Returns:
-            The highest error code found.
+            The modified statistics object.
         """
-        for error in self._errors:
-            error_return_code = ErrorSeverity.GENERIC_ERR
-            if isinstance(error, DTSError):
-                error_return_code = error.severity
-
-            if error_return_code > self._return_code:
-                self._return_code = error_return_code
+        self[other.name] += 1
+        self["PASS RATE"] = (
+            float(self[Result.PASS.name]) * 100 / sum(self[result.name] for result in Result)
+        )
+        return self
 
-        return int(self._return_code)
+    def __str__(self) -> str:
+        """Each line contains the formatted key = value pair."""
+        stats_str = ""
+        for key, value in self.items():
+            stats_str += f"{key:<12} = {value}\n"
+            # according to docs, we should use \n when writing to text files
+            # on all platforms
+        return stats_str