@@ -10,7 +10,15 @@ executions:
compiler_wrapper: ccache
perf: false
func: true
+ nics: #physical devices to be used for testing
+ - addresses:
+ - "0000:11:00.0"
+ - "0000:11:00.1"
+ driver: "i40e"
+ vdevs: #names of virtual devices to be used for testing
+ - "crypto_openssl"
test_suites:
+ - smoke_tests
- hello_world
system_under_test: "SUT 1"
nodes:
@@ -107,6 +107,46 @@ def from_dict(d: dict) -> "NodeConfiguration":
)
+@dataclass(slots=True)
+class NodeVersionInfo:
+ """Class to hold important versions within the node.
+
+ This class, unlike the NodeConfiguration class, cannot be generated at the start.
+ This is because we need to initialize a connection with the node before we can
+ collet the information needed in this class. Therefore, it cannot be a part of
+ the configuration class above.
+ """
+
+ os_name: str
+ os_version: str
+ kernel_version: str
+
+ @staticmethod
+ def from_dict(d: dict):
+ return NodeVersionInfo(
+ os_name=d["os_name"],
+ os_version=d["os_version"],
+ kernel_version=d["kernel_version"],
+ )
+
+
+@dataclass(slots=True, frozen=True)
+class NICConfiguration:
+ addresses: list[str]
+ driver: str
+
+ @staticmethod
+ def from_dict(d: dict) -> "NICConfiguration":
+ return NICConfiguration(
+ addresses=[addr for addr in d.get("addresses", [])],
+ driver=d.get("driver", ""),
+ )
+
+ @staticmethod
+ def from_list(nics: list[dict]) -> list["NICConfiguration"]:
+ return [NICConfiguration.from_dict(x) for x in nics]
+
+
@dataclass(slots=True, frozen=True)
class BuildTargetConfiguration:
arch: Architecture
@@ -128,6 +168,24 @@ def from_dict(d: dict) -> "BuildTargetConfiguration":
)
+@dataclass(slots=True)
+class BuildTargetVersionInfo:
+ """Class to hold important versions within the build target.
+
+ This is very similar to the NodeVersionInfo class, it just instead holds information
+ for the build target.
+ """
+
+ dpdk_version: str
+ compiler_version: str
+
+ @staticmethod
+ def from_dict(d: dict):
+ return BuildTargetVersionInfo(
+ dpdk_version=d["dpdk_version"], compiler_version=d["compiler_version"]
+ )
+
+
class TestSuiteConfigDict(TypedDict):
suite: str
cases: list[str]
@@ -157,6 +215,8 @@ class ExecutionConfiguration:
func: bool
test_suites: list[TestSuiteConfig]
system_under_test: NodeConfiguration
+ nics: list[NICConfiguration]
+ vdevs: list[str]
@staticmethod
def from_dict(d: dict, node_map: dict) -> "ExecutionConfiguration":
@@ -166,7 +226,9 @@ def from_dict(d: dict, node_map: dict) -> "ExecutionConfiguration":
test_suites: list[TestSuiteConfig] = list(
map(TestSuiteConfig.from_dict, d["test_suites"])
)
+ nic_conf = NICConfiguration.from_list(d["nics"])
sut_name = d["system_under_test"]
+ list_of_vdevs = d["vdevs"]
assert sut_name in node_map, f"Unknown SUT {sut_name} in execution {d}"
return ExecutionConfiguration(
@@ -174,7 +236,9 @@ def from_dict(d: dict, node_map: dict) -> "ExecutionConfiguration":
perf=d["perf"],
func=d["func"],
test_suites=test_suites,
+ nics=nic_conf,
system_under_test=node_map[sut_name],
+ vdevs=list_of_vdevs,
)
@@ -221,3 +285,27 @@ def load_config() -> Configuration:
CONFIGURATION = load_config()
+
+
+@unique
+class InteractiveApp(Enum):
+ """An enum that represents different supported interactive applications
+
+ The values in this enum must all be set to objects that have a key called
+ "default_path" where "default_path" represents an array for the path to the
+ application. This default path will be passed into the handler class for the
+ application so that it can start the application. For every key other than
+ the default shell option, the path will be appended to the path to the DPDK
+ build directory for the current SUT node.
+ """
+
+ shell = {"default_path": [""]}
+ testpmd = {"default_path": ["app", "dpdk-testpmd"]}
+
+ def get_path(self):
+ """A method for getting the default paths of an application
+
+ Returns:
+ String array that represents an OS agnostic path to the application.
+ """
+ return self.value["default_path"]
@@ -40,6 +40,18 @@
"mscv"
]
},
+ "single_nic" : {
+ "type":"object",
+ "description": "an object that holds nic information",
+ "properties": {
+ "addresses": {
+ "type":"array",
+ "items": {
+ "type":"string"
+ }
+ }
+ }
+ },
"build_target": {
"type": "object",
"description": "Targets supported by DTS",
@@ -97,7 +109,8 @@
"test_suite": {
"type": "string",
"enum": [
- "hello_world"
+ "hello_world",
+ "smoke_tests"
]
},
"test_target": {
@@ -211,6 +224,23 @@
]
}
},
+ "nics": {
+ "type": "array",
+ "items": {
+ "oneOf": [
+ {
+ "$ref": "#/definitions/single_nic"
+ }
+ ]
+ }
+ },
+ "vdevs": {
+ "description": "Names of vdevs to be used in execution",
+ "type": "array",
+ "items": {
+ "type":"string"
+ }
+ },
"system_under_test": {
"$ref": "#/definitions/node_name"
}
@@ -6,6 +6,7 @@
import sys
from .config import CONFIGURATION, BuildTargetConfiguration, ExecutionConfiguration
+from .exception import BlockingTestSuiteError
from .logger import DTSLOG, getLogger
from .test_result import BuildTargetResult, DTSResult, ExecutionResult, Result
from .test_suite import get_test_suites
@@ -82,7 +83,9 @@ def _run_execution(
running all build targets in the given execution.
"""
dts_logger.info(f"Running execution with SUT '{execution.system_under_test.name}'.")
- execution_result = result.add_execution(sut_node.config)
+ execution_result = result.add_execution(
+ sut_node.config, sut_node.get_node_versions()
+ )
try:
sut_node.set_up_execution(execution)
@@ -118,7 +121,10 @@ def _run_build_target(
try:
sut_node.set_up_build_target(build_target)
- result.dpdk_version = sut_node.dpdk_version
+ # result.dpdk_version = sut_node.dpdk_version
+ build_target_result.add_build_target_versions(
+ sut_node.get_build_target_versions()
+ )
build_target_result.update_setup(Result.PASS)
except Exception as e:
dts_logger.exception("Build target setup failed.")
@@ -146,6 +152,7 @@ def _run_suites(
with possibly only a subset of test cases.
If no subset is specified, run all test cases.
"""
+ end_build_target = False
for test_suite_config in execution.test_suites:
try:
full_suite_path = f"tests.TestSuite_{test_suite_config.test_suite}"
@@ -165,8 +172,21 @@ def _run_suites(
test_suite_config.test_cases,
execution.func,
build_target_result,
+ sut_node._build_target_config,
+ result,
)
- test_suite.run()
+ try:
+ test_suite.run()
+ except BlockingTestSuiteError as e:
+ dts_logger.exception(
+ "An error occurred within a blocking TestSuite, "
+ "execution will now end."
+ )
+ result.add_error(e)
+ end_build_target = True
+ # if a blocking test failed and we need to bail out of suite executions
+ if end_build_target:
+ break
def _exit_dts() -> None:
@@ -25,6 +25,7 @@ class ErrorSeverity(IntEnum):
SSH_ERR = 4
DPDK_BUILD_ERR = 10
TESTCASE_VERIFY_ERR = 20
+ BLOCKING_TESTSUITE_ERR = 25
class DTSError(Exception):
@@ -144,3 +145,14 @@ def __init__(self, value: str):
def __str__(self) -> str:
return repr(self.value)
+
+
+class BlockingTestSuiteError(DTSError):
+ suite_name: str
+ severity: ClassVar[ErrorSeverity] = ErrorSeverity.BLOCKING_TESTSUITE_ERR
+
+ def __init__(self, suite_name: str) -> None:
+ self.suite_name = suite_name
+
+ def __str__(self) -> str:
+ return f"Blocking suite {self.suite_name} failed."
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright(c) 2023 PANTHEON.tech s.r.o.
+# Copyright(c) 2022-2023 University of New Hampshire
"""
The package provides modules for managing remote connections to a remote host (node),
@@ -17,7 +18,14 @@
from .linux_session import LinuxSession
from .os_session import OSSession
-from .remote import CommandResult, RemoteSession, SSHSession
+from .remote import (
+ CommandResult,
+ InteractiveRemoteSession,
+ InteractiveShell,
+ RemoteSession,
+ SSHSession,
+ TestPmdShell,
+)
def create_session(
@@ -12,7 +12,13 @@
from framework.testbed_model import LogicalCore
from framework.utils import EnvVarsDict, MesonArgs
-from .remote import CommandResult, RemoteSession, create_remote_session
+from .remote import (
+ CommandResult,
+ InteractiveRemoteSession,
+ RemoteSession,
+ create_interactive_session,
+ create_remote_session,
+)
class OSSession(ABC):
@@ -26,6 +32,7 @@ class OSSession(ABC):
name: str
_logger: DTSLOG
remote_session: RemoteSession
+ interactive_session: InteractiveRemoteSession
def __init__(
self,
@@ -37,6 +44,7 @@ def __init__(
self.name = name
self._logger = logger
self.remote_session = create_remote_session(node_config, name, logger)
+ self.interactive_session = create_interactive_session(node_config, name, logger)
def close(self, force: bool = False) -> None:
"""
@@ -173,3 +181,27 @@ def setup_hugepages(self, hugepage_amount: int, force_first_numa: bool) -> None:
if needed and mount the hugepages if needed.
If force_first_numa is True, configure hugepages just on the first socket.
"""
+
+ @abstractmethod
+ def get_compiler_version(self, compiler_name: str) -> str:
+ """
+ Get installed version of compiler used for DPDK
+ """
+
+ @abstractmethod
+ def get_os_name(self) -> str:
+ """
+ Get name of OS for the node
+ """
+
+ @abstractmethod
+ def get_os_version(self) -> str:
+ """
+ Get version of OS for the node
+ """
+
+ @abstractmethod
+ def get_kernel_version(self) -> str:
+ """
+ Get kernel version for the node
+ """
@@ -219,3 +219,33 @@ def _remove_dpdk_runtime_dirs(
def get_dpdk_file_prefix(self, dpdk_prefix) -> str:
return ""
+
+ def get_compiler_version(self, compiler_name: str) -> str:
+ match compiler_name:
+ case "gcc":
+ return self.send_command(f"{compiler_name} --version", 60).stdout.split(
+ "\n"
+ )[0]
+ case "clang":
+ return self.send_command(f"{compiler_name} --version", 60).stdout.split(
+ "\n"
+ )[0]
+ case "msvc":
+ return self.send_command("cl", 60).stdout
+ case "icc":
+ return self.send_command(f"{compiler_name} -V", 60).stdout
+ case _:
+ raise ValueError(f"Unknown compiler {compiler_name}")
+
+ def get_os_name(self) -> str:
+ return self.send_command(
+ "awk -F= '$1==\"NAME\" {print $2}' /etc/os-release", 60
+ ).stdout
+
+ def get_os_version(self) -> str:
+ return self.send_command(
+ "awk -F= '$1==\"VERSION\" {print $2}' /etc/os-release", 60, True
+ ).stdout
+
+ def get_kernel_version(self) -> str:
+ return self.send_command("uname -r", 60).stdout
@@ -1,16 +1,28 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright(c) 2023 PANTHEON.tech s.r.o.
+# Copyright(c) 2022-2023 University of New Hampshire
# pylama:ignore=W0611
+from paramiko import AutoAddPolicy, SSHClient
+
from framework.config import NodeConfiguration
from framework.logger import DTSLOG
+from .interactive_remote_session import InteractiveRemoteSession
+from .interactive_shell import InteractiveShell
from .remote_session import CommandResult, RemoteSession
from .ssh_session import SSHSession
+from .testpmd_shell import TestPmdShell
def create_remote_session(
node_config: NodeConfiguration, name: str, logger: DTSLOG
) -> RemoteSession:
return SSHSession(node_config, name, logger)
+
+
+def create_interactive_session(
+ node_config: NodeConfiguration, name: str, logger: DTSLOG
+) -> InteractiveRemoteSession:
+ return InteractiveRemoteSession(node_config, logger)
new file mode 100644
@@ -0,0 +1,113 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2023 University of New Hampshire
+
+import socket
+import traceback
+from pathlib import PurePath
+
+from paramiko import AutoAddPolicy, SSHClient, Transport
+from paramiko.ssh_exception import (
+ AuthenticationException,
+ BadHostKeyException,
+ NoValidConnectionsError,
+ SSHException,
+)
+
+from framework.config import InteractiveApp, NodeConfiguration
+from framework.exception import SSHConnectionError
+from framework.logger import DTSLOG
+from framework.settings import SETTINGS
+
+from .interactive_shell import InteractiveShell
+from .testpmd_shell import TestPmdShell
+
+
+class InteractiveRemoteSession:
+ hostname: str
+ ip: str
+ port: int
+ username: str
+ password: str
+ _logger: DTSLOG
+ _node_config: NodeConfiguration
+ session: SSHClient
+ _transport: Transport | None
+
+ def __init__(self, node_config: NodeConfiguration, _logger: DTSLOG) -> None:
+ self._node_config = node_config
+ self._logger = _logger
+ self.hostname = node_config.hostname
+ self.username = node_config.user
+ self.password = node_config.password if node_config.password else ""
+ port = 22
+ self.ip = node_config.hostname
+ if ":" in node_config.hostname:
+ self.ip, port = node_config.hostname.split(":")
+ self.port = int(port)
+ self._logger.info(
+ f"Initializing interactive connection for {self.username}@{self.hostname}"
+ )
+ self._connect()
+ self._logger.info(
+ f"Interactive connection successful for {self.username}@{self.hostname}"
+ )
+
+ def _connect(self) -> None:
+ client = SSHClient()
+ client.set_missing_host_key_policy(AutoAddPolicy)
+ self.session = client
+ retry_attempts = 10
+ for retry_attempt in range(retry_attempts):
+ try:
+ client.connect(
+ self.ip,
+ username=self.username,
+ port=self.port,
+ password=self.password,
+ timeout=20 if self.port else 10,
+ )
+ except (TypeError, BadHostKeyException, AuthenticationException) as e:
+ self._logger.exception(e)
+ raise SSHConnectionError(self.hostname) from e
+ except (NoValidConnectionsError, socket.error, SSHException) as e:
+ self._logger.debug(traceback.format_exc())
+ self._logger.warning(e)
+ self._logger.info(
+ "Retrying interactive session connection: "
+ f"retry number {retry_attempt +1}"
+ )
+ else:
+ break
+ else:
+ raise SSHConnectionError(self.hostname)
+ # Interactive sessions are used on an "as needed" basis so we have
+ # to set a keepalive
+ self._transport = self.session.get_transport()
+ if self._transport is not None:
+ self._transport.set_keepalive(30)
+
+ def create_interactive_shell(
+ self,
+ shell_type: InteractiveApp,
+ path_to_app: PurePath,
+ timeout: float = SETTINGS.timeout,
+ eal_parameters: str = "",
+ app_parameters="",
+ ) -> InteractiveShell:
+ """
+ See "create_interactive_shell" in SutNode
+ """
+ match (shell_type.name):
+ case "shell":
+ return InteractiveShell(
+ self.session, self._logger, path_to_app, timeout
+ )
+ case "testpmd":
+ return TestPmdShell(
+ self.session,
+ self._logger,
+ path_to_app,
+ timeout=timeout,
+ eal_flags=eal_parameters,
+ cmd_line_options=app_parameters,
+ )
new file mode 100644
@@ -0,0 +1,98 @@
+from pathlib import PurePath
+
+from paramiko import Channel, SSHClient, channel
+
+from framework.logger import DTSLOG
+from framework.settings import SETTINGS
+
+
+class InteractiveShell:
+
+ _interactive_session: SSHClient
+ _stdin: channel.ChannelStdinFile
+ _stdout: channel.ChannelFile
+ _ssh_channel: Channel
+ _logger: DTSLOG
+ _timeout: float
+
+ def __init__(
+ self,
+ interactive_session: SSHClient,
+ logger: DTSLOG,
+ path_to_app: PurePath | None = None,
+ timeout: float = SETTINGS.timeout,
+ ) -> None:
+ self._interactive_session = interactive_session
+ self._ssh_channel = self._interactive_session.invoke_shell()
+ self._stdin = self._ssh_channel.makefile_stdin("w")
+ self._stdout = self._ssh_channel.makefile("r")
+ self._ssh_channel.settimeout(timeout)
+ self._ssh_channel.set_combine_stderr(True) # combines stdout and stderr streams
+ self._logger = logger
+ self._timeout = timeout
+ if path_to_app:
+ self.send_command_no_output(str(path_to_app)) # starts the application
+
+ def empty_stdout_buffer(self) -> None:
+ """Removes all data from the stdout buffer.
+
+ Because of the way paramiko handles read buffers, there is no way to effectively
+ remove data from, or "flush", read buffers. This method essentially moves our
+ offset on the buffer to the end and thus "removes" the data from the buffer.
+ Timeouts are thrown on read operations of paramiko pipes based on whether data
+ had been recieved before timeout so we assume that if we reach the timeout then
+ we are at the end of the buffer.
+ """
+ self._ssh_channel.settimeout(1)
+ try:
+ for line in self._stdout:
+ pass
+ except TimeoutError:
+ pass
+ self._ssh_channel.settimeout(self._timeout) # reset timeout
+
+ def send_command_no_output(self, command: str) -> None:
+ """Send command to channel without recording output.
+
+ This method will not verify any input or output, it will simply assume the
+ command succeeded. This method will also consume all output in the buffer
+ after executing the command.
+ """
+ self._logger.info(
+ f"Sending command {command.strip()} and not collecting output"
+ )
+ self._stdin.write(f"{command}\n")
+ self._stdin.flush()
+ self.empty_stdout_buffer()
+
+ def send_command_get_output(self, command: str, prompt: str) -> str:
+ """Send a command and get all output before the expected ending string.
+
+ Lines that expect input are not included in the stdout buffer so they cannot be
+ used for expect. For example, if you were prompted to log into something
+ with a username and password, you cannot expect "username:" because it wont
+ yet be in the stdout buffer. A work around for this could be consuming an
+ extra newline character to force the current prompt into the stdout buffer.
+
+ Returns:
+ All output in the buffer before expected string
+ """
+ self._logger.info(f"Sending command {command.strip()}...")
+ self._stdin.write(f"{command}\n")
+ self._stdin.flush()
+ out: str = ""
+ for line in self._stdout:
+ out += line
+ if prompt in line and not line.rstrip().endswith(
+ command.rstrip()
+ ): # ignore line that sent command
+ break
+ self._logger.debug(f"Got output: {out}")
+ return out
+
+ def close(self) -> None:
+ self._stdin.close()
+ self._ssh_channel.close()
+
+ def __del__(self) -> None:
+ self.close()
new file mode 100644
@@ -0,0 +1,58 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2023 University of New Hampshire
+
+
+from pathlib import PurePath
+
+from paramiko import SSHClient
+
+from framework.logger import DTSLOG
+from framework.settings import SETTINGS
+
+from .interactive_shell import InteractiveShell
+
+
+class TestPmdShell(InteractiveShell):
+ expected_prompt: str = "testpmd>"
+
+ def __init__(
+ self,
+ interactive_session: SSHClient,
+ logger: DTSLOG,
+ path_to_testpmd: PurePath,
+ eal_flags: str = "",
+ cmd_line_options: str = "",
+ timeout: float = SETTINGS.timeout,
+ ) -> None:
+ """Initalizes an interactive testpmd session using specified parameters."""
+ super(TestPmdShell, self).__init__(
+ interactive_session, logger=logger, timeout=timeout
+ )
+ self.send_command_get_output(
+ f"{path_to_testpmd} {eal_flags} -- -i {cmd_line_options}\n",
+ self.expected_prompt,
+ )
+
+ def send_command(self, command: str, prompt: str = expected_prompt) -> str:
+ """Specific way of handling the command for testpmd
+
+ An extra newline character is consumed in order to force the current line into
+ the stdout buffer.
+ """
+ return self.send_command_get_output(f"{command}\n", prompt)
+
+ def get_devices(self) -> list[str]:
+ """Get a list of device names that are known to testpmd
+
+ Uses the device info listed in testpmd and then parses the output to
+ return only the names of the devices.
+
+ Returns:
+ A list of strings representing device names (e.g. 0000:14:00.1)
+ """
+ dev_info: str = self.send_command("show device info all")
+ dev_list: list[str] = []
+ for line in dev_info.split("\n"):
+ if "device name:" in line.lower():
+ dev_list.append(line.strip().split(": ")[1].strip())
+ return dev_list
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright(c) 2023 PANTHEON.tech s.r.o.
+# Copyright(c) 2022-2023 University of New Hampshire
"""
Generic result container and reporters
@@ -8,14 +9,17 @@
import os.path
from collections.abc import MutableSequence
from enum import Enum, auto
+from typing import Dict
from .config import (
OS,
Architecture,
BuildTargetConfiguration,
+ BuildTargetVersionInfo,
Compiler,
CPUType,
NodeConfiguration,
+ NodeVersionInfo,
)
from .exception import DTSError, ErrorSeverity
from .logger import DTSLOG
@@ -67,12 +71,14 @@ class Statistics(dict):
Using a dict provides a convenient way to format the data.
"""
- def __init__(self, dpdk_version):
+ def __init__(self, output_info: Dict[str, str] | None):
super(Statistics, self).__init__()
for result in Result:
self[result.name] = 0
self["PASS RATE"] = 0.0
- self["DPDK VERSION"] = dpdk_version
+ if output_info:
+ for info_key, info_val in output_info.items():
+ self[info_key] = info_val
def __iadd__(self, other: Result) -> "Statistics":
"""
@@ -206,6 +212,8 @@ class BuildTargetResult(BaseResult):
os: OS
cpu: CPUType
compiler: Compiler
+ compiler_version: str | None
+ dpdk_version: str | None
def __init__(self, build_target: BuildTargetConfiguration):
super(BuildTargetResult, self).__init__()
@@ -213,6 +221,12 @@ def __init__(self, build_target: BuildTargetConfiguration):
self.os = build_target.os
self.cpu = build_target.cpu
self.compiler = build_target.compiler
+ self.compiler_version = None
+ self.dpdk_version = None
+
+ def add_build_target_versions(self, versions: BuildTargetVersionInfo) -> None:
+ self.compiler_version = versions.compiler_version
+ self.dpdk_version = versions.dpdk_version
def add_test_suite(self, test_suite_name: str) -> TestSuiteResult:
test_suite_result = TestSuiteResult(test_suite_name)
@@ -228,10 +242,17 @@ class ExecutionResult(BaseResult):
"""
sut_node: NodeConfiguration
+ sut_os_name: str
+ sut_os_version: str
+ sut_kernel_version: str
- def __init__(self, sut_node: NodeConfiguration):
+ def __init__(self, sut_node: NodeConfiguration, sut_version_info: NodeVersionInfo):
super(ExecutionResult, self).__init__()
self.sut_node = sut_node
+ self.sut_version_info = sut_version_info
+ self.sut_os_name = sut_version_info.os_name
+ self.sut_os_version = sut_version_info.os_version
+ self.sut_kernel_version = sut_version_info.kernel_version
def add_build_target(
self, build_target: BuildTargetConfiguration
@@ -258,6 +279,7 @@ class DTSResult(BaseResult):
"""
dpdk_version: str | None
+ output: dict | None
_logger: DTSLOG
_errors: list[Exception]
_return_code: ErrorSeverity
@@ -267,14 +289,17 @@ class DTSResult(BaseResult):
def __init__(self, logger: DTSLOG):
super(DTSResult, self).__init__()
self.dpdk_version = None
+ self.output = None
self._logger = logger
self._errors = []
self._return_code = ErrorSeverity.NO_ERR
self._stats_result = None
self._stats_filename = os.path.join(SETTINGS.output_dir, "statistics.txt")
- def add_execution(self, sut_node: NodeConfiguration) -> ExecutionResult:
- execution_result = ExecutionResult(sut_node)
+ def add_execution(
+ self, sut_node: NodeConfiguration, sut_version_info: NodeVersionInfo
+ ) -> ExecutionResult:
+ execution_result = ExecutionResult(sut_node, sut_version_info)
self._inner_results.append(execution_result)
return execution_result
@@ -296,7 +321,8 @@ def process(self) -> None:
for error in self._errors:
self._logger.debug(repr(error))
- self._stats_result = Statistics(self.dpdk_version)
+ self._stats_result = Statistics(self.output)
+ # add information gathered from the smoke tests to the statistics
self.add_stats(self._stats_result)
with open(self._stats_filename, "w+") as stats_file:
stats_file.write(str(self._stats_result))
@@ -10,11 +10,24 @@
import inspect
import re
from types import MethodType
+from typing import Dict
-from .exception import ConfigurationError, SSHTimeoutError, TestCaseVerifyError
+from .config import BuildTargetConfiguration
+from .exception import (
+ BlockingTestSuiteError,
+ ConfigurationError,
+ SSHTimeoutError,
+ TestCaseVerifyError,
+)
from .logger import DTSLOG, getLogger
from .settings import SETTINGS
-from .test_result import BuildTargetResult, Result, TestCaseResult, TestSuiteResult
+from .test_result import (
+ BuildTargetResult,
+ DTSResult,
+ Result,
+ TestCaseResult,
+ TestSuiteResult,
+)
from .testbed_model import SutNode
@@ -37,10 +50,12 @@ class TestSuite(object):
"""
sut_node: SutNode
+ is_blocking = False
_logger: DTSLOG
_test_cases_to_run: list[str]
_func: bool
_result: TestSuiteResult
+ _dts_result: DTSResult
def __init__(
self,
@@ -48,6 +63,8 @@ def __init__(
test_cases: list[str],
func: bool,
build_target_result: BuildTargetResult,
+ build_target_conf: BuildTargetConfiguration,
+ dts_result: DTSResult,
):
self.sut_node = sut_node
self._logger = getLogger(self.__class__.__name__)
@@ -55,6 +72,8 @@ def __init__(
self._test_cases_to_run.extend(SETTINGS.test_cases)
self._func = func
self._result = build_target_result.add_test_suite(self.__class__.__name__)
+ self.build_target_info = build_target_conf
+ self._dts_result = dts_result
def set_up_suite(self) -> None:
"""
@@ -118,6 +137,8 @@ def run(self) -> None:
f"the next test suite may be affected."
)
self._result.update_setup(Result.ERROR, e)
+ if len(self._result.get_errors()) > 0 and self.is_blocking:
+ raise BlockingTestSuiteError(test_suite_name)
def _execute_test_suite(self) -> None:
"""
@@ -232,6 +253,12 @@ def _execute_test_case(
test_case_result.update(Result.SKIP)
raise KeyboardInterrupt("Stop DTS")
+ def write_to_statistics_file(self, output: Dict[str, str]):
+ if self._dts_result.output is not None:
+ self._dts_result.output.update(output)
+ else:
+ self._dts_result.output = output
+
def get_test_suites(testsuite_module_path: str) -> list[type[TestSuite]]:
def is_test_suite(object) -> bool:
@@ -40,6 +40,7 @@ class Node(object):
lcores: list[LogicalCore]
_logger: DTSLOG
_other_sessions: list[OSSession]
+ _execution_config: ExecutionConfiguration
def __init__(self, node_config: NodeConfiguration):
self.config = node_config
@@ -64,6 +65,7 @@ def set_up_execution(self, execution_config: ExecutionConfiguration) -> None:
"""
self._setup_hugepages()
self._set_up_execution(execution_config)
+ self._execution_config = execution_config
def _set_up_execution(self, execution_config: ExecutionConfiguration) -> None:
"""
@@ -1,14 +1,21 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright(c) 2010-2014 Intel Corporation
# Copyright(c) 2023 PANTHEON.tech s.r.o.
+# Copyright(c) 2022-2023 University of New Hampshire
import os
import tarfile
import time
from pathlib import PurePath
-from framework.config import BuildTargetConfiguration, NodeConfiguration
-from framework.remote_session import CommandResult, OSSession
+from framework.config import (
+ BuildTargetConfiguration,
+ BuildTargetVersionInfo,
+ InteractiveApp,
+ NodeConfiguration,
+ NodeVersionInfo,
+)
+from framework.remote_session import CommandResult, InteractiveShell, OSSession
from framework.settings import SETTINGS
from framework.utils import EnvVarsDict, MesonArgs
@@ -26,13 +33,17 @@ class SutNode(Node):
_dpdk_prefix_list: list[str]
_dpdk_timestamp: str
- _build_target_config: BuildTargetConfiguration | None
+ _build_target_config: BuildTargetConfiguration
_env_vars: EnvVarsDict
_remote_tmp_dir: PurePath
__remote_dpdk_dir: PurePath | None
- _dpdk_version: str | None
_app_compile_timeout: float
_dpdk_kill_session: OSSession | None
+ _dpdk_version: str | None
+ _os_name: str | None
+ _os_version: str | None
+ _kernel_version: str | None
+ _compiler_version: str | None
def __init__(self, node_config: NodeConfiguration):
super(SutNode, self).__init__(node_config)
@@ -41,12 +52,16 @@ def __init__(self, node_config: NodeConfiguration):
self._env_vars = EnvVarsDict()
self._remote_tmp_dir = self.main_session.get_remote_tmp_dir()
self.__remote_dpdk_dir = None
- self._dpdk_version = None
self._app_compile_timeout = 90
self._dpdk_kill_session = None
self._dpdk_timestamp = (
f"{str(os.getpid())}_{time.strftime('%Y%m%d%H%M%S', time.localtime())}"
)
+ self._dpdk_version = None
+ self._os_name = None
+ self._os_version = None
+ self._kernel_version = None
+ self._compiler_version = None
@property
def _remote_dpdk_dir(self) -> PurePath:
@@ -75,6 +90,44 @@ def dpdk_version(self) -> str:
)
return self._dpdk_version
+ @property
+ def os_name(self) -> str:
+ if self._os_name is None:
+ self._os_name = self.main_session.get_os_name()
+ return self._os_name
+
+ @property
+ def os_version(self) -> str:
+ if self._os_version is None:
+ self._os_version = self.main_session.get_os_version()
+ return self._os_version
+
+ @property
+ def kernel_version(self) -> str:
+ if self._kernel_version is None:
+ self._kernel_version = self.main_session.get_kernel_version()
+ return self._kernel_version
+
+ @property
+ def compiler_version(self) -> str:
+ if self._compiler_version is None:
+ self._compiler_version = self.main_session.get_compiler_version(
+ self._build_target_config.compiler.name
+ )
+ return self._compiler_version
+
+ def get_node_versions(self) -> NodeVersionInfo:
+ return NodeVersionInfo(
+ os_name=self.os_name,
+ os_version=self.os_version,
+ kernel_version=self.kernel_version,
+ )
+
+ def get_build_target_versions(self) -> BuildTargetVersionInfo:
+ return BuildTargetVersionInfo(
+ dpdk_version=self.dpdk_version, compiler_version=self.compiler_version
+ )
+
def _guess_dpdk_remote_dir(self) -> PurePath:
return self.main_session.guess_dpdk_remote_dir(self._remote_tmp_dir)
@@ -84,6 +137,10 @@ def _set_up_build_target(
"""
Setup DPDK on the SUT node.
"""
+ # we want to ensure that dpdk_version and compiler_version is reset for new
+ # build targets
+ self._dpdk_version = None
+ self._compiler_version = None
self._configure_build_target(build_target_config)
self._copy_dpdk_tarball()
self._build_dpdk()
@@ -262,6 +319,49 @@ def run_dpdk_app(
f"{app_path} {eal_args}", timeout, verify=True
)
+ def create_interactive_shell(
+ self,
+ shell_type: InteractiveApp,
+ path_to_app: PurePath | None = None,
+ timeout: float = SETTINGS.timeout,
+ eal_parameters: str = "",
+ app_parameters="",
+ ) -> InteractiveShell:
+ """Create a handler for an interactive session.
+
+ This method is a factory that calls a method in OSSession to create shells for
+ different DPDK applications.
+
+ Args:
+ shell_type: Enum value representing the desired application.
+ path_to_app: Represents a path to the application you are attempting to
+ launch. This path will be executed at the start of the app
+ initialization. If one isn't provided, the default specified in the
+ enumeration will be used.
+ timeout: Timeout for the ssh channel
+ eal_parameters: List of EAL parameters to use to launch the app. This is
+ ignored for base "shell" types.
+ app_parameters: Command-line flags to pass into the application on launch.
+ Returns:
+ Instance of the desired interactive application.
+ """
+ default_path: PurePath | None
+ # if we just want a default shell, there is no need to append the DPDK build
+ # directory to the path
+ if shell_type.name == "shell":
+ default_path = None
+ else:
+ default_path = self.main_session.join_remote_path(
+ self.remote_dpdk_build_dir, *shell_type.get_path()
+ )
+ return self.main_session.interactive_session.create_interactive_shell(
+ shell_type,
+ path_to_app or default_path,
+ timeout,
+ eal_parameters,
+ app_parameters,
+ )
+
class EalParameters(object):
def __init__(
new file mode 100644
@@ -0,0 +1,109 @@
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright(c) 2023 University of New Hampshire
+
+from framework.config import InteractiveApp
+from framework.remote_session import TestPmdShell
+from framework.test_suite import TestSuite
+
+
+class SmokeTests(TestSuite):
+ is_blocking = True
+ # in this list, the first index is the address of the nic and the second is
+ # the driver for that nic.
+ list_of_nics: list[tuple[str, str]] = []
+
+ def set_up_suite(self) -> None:
+ """
+ Setup:
+ build all DPDK
+ """
+ self.dpdk_build_dir_path = self.sut_node.remote_dpdk_build_dir
+ for nic in self.sut_node._execution_config.nics:
+ for address in nic.addresses:
+ new_tuple = (address, nic.driver.strip())
+ self.list_of_nics.append(new_tuple)
+
+ def test_unit_tests(self) -> None:
+ """
+ Test:
+ run the fast-test unit-test suite through meson
+ """
+ self.sut_node.main_session.send_command(
+ f"meson test -C {self.dpdk_build_dir_path} --suite fast-tests", 300
+ )
+
+ def test_driver_tests(self) -> None:
+ """
+ Test:
+ run the driver-test unit-test suite through meson
+ """
+ list_of_vdevs = ""
+ for dev in self.sut_node._execution_config.vdevs:
+ list_of_vdevs += f"{dev},"
+ if list_of_vdevs:
+ self._logger.info(
+ "Running driver tests with the following virtual "
+ f"devices: {list_of_vdevs}"
+ )
+ self.sut_node.main_session.send_command(
+ "meson test -C {self.dpdk_build_dir_path} --suite driver-tests "
+ f'--test-args "--vdev {list_of_vdevs}"',
+ 300,
+ )
+ else:
+ self.sut_node.main_session.send_command(
+ f"meson test -C {self.dpdk_build_dir_path} --suite driver-tests", 300
+ )
+
+ def test_gather_info(self) -> None:
+ """
+ Test:
+ gather information about the system and send output to statistics.txt
+ """
+ out = {}
+
+ out["OS"] = self.sut_node.os_name
+ out["OS VERSION"] = self.sut_node.os_version
+ out["COMPILER VERSION"] = self.sut_node.compiler_version
+ out["DPDK VERSION"] = self.sut_node.dpdk_version
+ out["KERNEL VERSION"] = self.sut_node.kernel_version
+ self.write_to_statistics_file(out)
+
+ def test_devices_listed_in_testpmd(self) -> None:
+ """
+ Uses testpmd driver to verify that devices have been found by testpmd
+ """
+ testpmd_driver: TestPmdShell = self.sut_node.create_interactive_shell(
+ InteractiveApp.testpmd
+ )
+ dev_list: list[str] = testpmd_driver.get_devices()
+ for nic in self.list_of_nics:
+ self.verify(
+ nic[0] in dev_list,
+ f"Device {nic[0]} was not listed in testpmd's availible devices, "
+ "please check your configuration",
+ )
+
+ def test_device_bound_to_driver(self) -> None:
+ """
+ Test:
+ ensure that all drivers listed in the config are bound to the correct driver
+ """
+ path_to_dev = self.sut_node.main_session.join_remote_path(
+ self.sut_node._remote_dpdk_dir, "usertools", "dpdk-devbind.py"
+ )
+ for nic in self.list_of_nics:
+ out = self.sut_node.main_session.send_command(
+ f"{path_to_dev} --status | grep {nic[0]}", 60
+ )
+ self.verify(
+ len(out.stdout) != 0,
+ f"Failed to find configured device ({nic[0]}) using dpdk-devbind.py",
+ )
+ for string in out.stdout.split(" "):
+ if "drv=" in string:
+ self.verify(
+ string.split("=")[1] == nic[1],
+ f"Driver for device {nic[0]} does not match driver listed in "
+ f'configuration (bound to {string.split("=")[1]})',
+ )