From patchwork Mon Jul 11 14:51:19 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Juraj_Linke=C5=A1?= X-Patchwork-Id: 113903 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id C79B2A0032; Mon, 11 Jul 2022 16:51:32 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id BA13C41614; Mon, 11 Jul 2022 16:51:32 +0200 (CEST) Received: from lb.pantheon.sk (lb.pantheon.sk [46.229.239.20]) by mails.dpdk.org (Postfix) with ESMTP id 3F1EF41611 for ; Mon, 11 Jul 2022 16:51:31 +0200 (CEST) Received: from localhost (localhost [127.0.0.1]) by lb.pantheon.sk (Postfix) with ESMTP id 82592243CE0; Mon, 11 Jul 2022 16:51:29 +0200 (CEST) X-Virus-Scanned: amavisd-new at siecit.sk Received: from lb.pantheon.sk ([127.0.0.1]) by localhost (lb.pantheon.sk [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id MTcpkkjTNvkP; Mon, 11 Jul 2022 16:51:28 +0200 (CEST) Received: from entguard.lab.pantheon.local (unknown [46.229.239.141]) by lb.pantheon.sk (Postfix) with ESMTP id BAB98243CC6; Mon, 11 Jul 2022 16:51:27 +0200 (CEST) From: =?utf-8?q?Juraj_Linke=C5=A1?= To: thomas@monjalon.net, david.marchand@redhat.com, jerinjacobk@gmail.com, ronan.randles@intel.com, Honnappa.Nagarahalli@arm.com, ohilyard@iol.unh.edu, lijuan.tu@intel.com Cc: dev@dpdk.org, =?utf-8?q?Juraj_Linke=C5=A1?= Subject: [PATCH v2 1/8] dts: add basic logging facility Date: Mon, 11 Jul 2022 14:51:19 +0000 Message-Id: <20220711145126.295427-2-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20220711145126.295427-1-juraj.linkes@pantheon.tech> References: <20220622121448.3304251-1-juraj.linkes@pantheon.tech> <20220711145126.295427-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org The logging module provides loggers distinguished by two attributes, a custom format and a verbosity switch. The loggers log to both console and more verbosely to files. Signed-off-by: Juraj Linkeš --- dts/framework/logger.py | 118 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 dts/framework/logger.py diff --git a/dts/framework/logger.py b/dts/framework/logger.py new file mode 100644 index 0000000000..2c026ee5eb --- /dev/null +++ b/dts/framework/logger.py @@ -0,0 +1,118 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2022 PANTHEON.tech s.r.o. +# Copyright(c) 2022 University of New Hampshire +# + +import logging +import os.path +from typing import TypedDict + +""" +DTS logger module with several log level. DTS framework and TestSuite log +will saved into different log files. +""" +verbose = False +date_fmt = "%d/%m/%Y %H:%M:%S" +stream_fmt = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + + +class LoggerDictType(TypedDict): + logger: "DTSLOG" + name: str + node: str + + +# List for saving all using loggers +global Loggers +Loggers: list[LoggerDictType] = [] + + +def set_verbose() -> None: + global verbose + verbose = True + + +class DTSLOG(logging.LoggerAdapter): + """ + DTS log class for framework and testsuite. + """ + node: str + logger: logging.Logger + sh: logging.StreamHandler + fh: logging.FileHandler + verbose_handler: logging.FileHandler + + def __init__(self, logger: logging.Logger, node: str = "suite"): + global log_dir + + self.logger = logger + self.logger.setLevel(1) # 1 means log everything + + self.node = node + + # add handler to emit to stdout + sh = logging.StreamHandler() + sh.setFormatter(logging.Formatter()) + sh.setFormatter(logging.Formatter(stream_fmt, date_fmt)) + + sh.setLevel(logging.DEBUG) # file handler default level + global verbose + if verbose is True: + sh.setLevel(logging.DEBUG) + else: + sh.setLevel(logging.INFO) # console handler defaultlevel + + self.logger.addHandler(sh) + self.sh = sh + + if not os.path.exists("output"): + os.mkdir("output") + + fh = logging.FileHandler(f"output/{node}.log") + fh.setFormatter(logging.Formatter( + fmt='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + datefmt=date_fmt)) + + fh.setLevel(1) # We want all the logs we can get in the file + self.logger.addHandler(fh) + self.fh = fh + + # This outputs EVERYTHING, intended for post-mortem debugging + # Also optimized for processing via AWK (awk -F '|' ...) + verbose_handler = logging.FileHandler(f"output/{node}.verbose.log") + verbose_handler.setFormatter(logging.Formatter( + fmt='%(asctime)s|%(name)s|%(levelname)s|%(pathname)s|%(lineno)d|%(funcName)s|' + '%(process)d|%(thread)d|%(threadName)s|%(message)s', + datefmt=date_fmt)) + + verbose_handler.setLevel(1) # We want all the logs we can get in the file + self.logger.addHandler(verbose_handler) + self.verbose_handler = verbose_handler + + super(DTSLOG, self).__init__(self.logger, dict(node=self.node)) + + def logger_exit(self) -> None: + """ + Remove stream handler and logfile handler. + """ + for handler in (self.sh, self.fh, self.verbose_handler): + handler.flush() + self.logger.removeHandler(handler) + + +def getLogger(name: str, node: str = "suite") -> DTSLOG: + """ + Get logger handler and if there's no handler for specified Node will create one. + """ + global Loggers + # return saved logger + logger: LoggerDictType + for logger in Loggers: + if logger["name"] == name and logger["node"] == node: + return logger["logger"] + + # return new logger + dts_logger: DTSLOG = DTSLOG(logging.getLogger(name), node) + Loggers.append({"logger": dts_logger, "name": name, "node": node}) + return dts_logger From patchwork Mon Jul 11 14:51:20 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Juraj_Linke=C5=A1?= X-Patchwork-Id: 113905 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id BD8A1A0032; Mon, 11 Jul 2022 16:51:44 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 801104282B; Mon, 11 Jul 2022 16:51:34 +0200 (CEST) Received: from lb.pantheon.sk (lb.pantheon.sk [46.229.239.20]) by mails.dpdk.org (Postfix) with ESMTP id D7ED242825 for ; Mon, 11 Jul 2022 16:51:33 +0200 (CEST) Received: from localhost (localhost [127.0.0.1]) by lb.pantheon.sk (Postfix) with ESMTP id 2A4C9243CE7; Mon, 11 Jul 2022 16:51:32 +0200 (CEST) X-Virus-Scanned: amavisd-new at siecit.sk Received: from lb.pantheon.sk ([127.0.0.1]) by localhost (lb.pantheon.sk [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id 7YPpxQAQ4zsF; Mon, 11 Jul 2022 16:51:31 +0200 (CEST) Received: from entguard.lab.pantheon.local (unknown [46.229.239.141]) by lb.pantheon.sk (Postfix) with ESMTP id 410B5243CD1; Mon, 11 Jul 2022 16:51:28 +0200 (CEST) From: =?utf-8?q?Juraj_Linke=C5=A1?= To: thomas@monjalon.net, david.marchand@redhat.com, jerinjacobk@gmail.com, ronan.randles@intel.com, Honnappa.Nagarahalli@arm.com, ohilyard@iol.unh.edu, lijuan.tu@intel.com Cc: dev@dpdk.org, =?utf-8?q?Juraj_Linke=C5=A1?= Subject: [PATCH v2 2/8] dts: add ssh pexpect library Date: Mon, 11 Jul 2022 14:51:20 +0000 Message-Id: <20220711145126.295427-3-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20220711145126.295427-1-juraj.linkes@pantheon.tech> References: <20220622121448.3304251-1-juraj.linkes@pantheon.tech> <20220711145126.295427-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org The library uses the pexpect python library and implements connection to a node and two ways to interact with the node: 1. Send a string with specified prompt which will be matched after the string has been sent to the node. 2. Send a command to be executed. No prompt is specified here. Signed-off-by: Juraj Linkeš --- dts/framework/exception.py | 60 ++++++++++ dts/framework/ssh_pexpect.py | 207 +++++++++++++++++++++++++++++++++++ dts/framework/utils.py | 11 ++ 3 files changed, 278 insertions(+) create mode 100644 dts/framework/exception.py create mode 100644 dts/framework/ssh_pexpect.py create mode 100644 dts/framework/utils.py diff --git a/dts/framework/exception.py b/dts/framework/exception.py new file mode 100644 index 0000000000..54f9f189a4 --- /dev/null +++ b/dts/framework/exception.py @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2022 PANTHEON.tech s.r.o. +# Copyright(c) 2022 University of New Hampshire +# + +""" +User-defined exceptions used across the framework. +""" + + +class TimeoutException(Exception): + + """ + Command execution timeout. + """ + + command: str + output: str + + def __init__(self, command: str, output: str): + self.command = command + self.output = output + + def __str__(self) -> str: + return f"TIMEOUT on {self.command}" + + def get_output(self) -> str: + return self.output + + +class SSHConnectionException(Exception): + + """ + SSH connection error. + """ + + host: str + + def __init__(self, host: str): + self.host = host + + def __str__(self) -> str: + return f"Error trying to connect with {self.host}" + + +class SSHSessionDeadException(Exception): + + """ + SSH session is not alive. + It can no longer be used. + """ + + host: str + + def __init__(self, host: str): + self.host = host + + def __str__(self) -> str: + return f"SSH session with {self.host} has died" diff --git a/dts/framework/ssh_pexpect.py b/dts/framework/ssh_pexpect.py new file mode 100644 index 0000000000..c73c1048a4 --- /dev/null +++ b/dts/framework/ssh_pexpect.py @@ -0,0 +1,207 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2022 PANTHEON.tech s.r.o. +# Copyright(c) 2022 University of New Hampshire +# + +import time +from typing import Optional + +from pexpect import pxssh + +from .exception import (SSHConnectionException, SSHSessionDeadException, + TimeoutException) +from .logger import DTSLOG +from .utils import GREEN, RED + +""" +Module handles ssh sessions to TG and SUT. +Implements send_expect function to send commands and get output data. +""" + + +class SSHPexpect: + username: str + password: str + node: str + logger: DTSLOG + magic_prompt: str + + def __init__( + self, + node: str, + username: str, + password: Optional[str], + logger: DTSLOG, + ): + self.magic_prompt = "MAGIC PROMPT" + self.logger = logger + + self.node = node + self.username = username + self.password = password or "" + self.logger.info(f"ssh {self.username}@{self.node}") + + self._connect_host() + + def _connect_host(self) -> None: + """ + Create connection to assigned node. + """ + retry_times = 10 + try: + if ":" in self.node: + while retry_times: + self.ip = self.node.split(":")[0] + self.port = int(self.node.split(":")[1]) + self.session = pxssh.pxssh(encoding="utf-8") + try: + self.session.login( + self.ip, + self.username, + self.password, + original_prompt="[$#>]", + port=self.port, + login_timeout=20, + password_regex=r"(?i)(?:password:)|(?:passphrase for key)|(?i)(password for .+:)", + ) + except Exception as e: + print(e) + time.sleep(2) + retry_times -= 1 + print("retry %d times connecting..." % (10 - retry_times)) + else: + break + else: + raise Exception("connect to %s:%s failed" % (self.ip, self.port)) + else: + self.session = pxssh.pxssh(encoding="utf-8") + self.session.login( + self.node, + self.username, + self.password, + original_prompt="[$#>]", + password_regex=r"(?i)(?:password:)|(?:passphrase for key)|(?i)(password for .+:)", + ) + self.logger.info(f"Connection to {self.node} succeeded") + self.send_expect("stty -echo", "#") + self.send_expect("stty columns 1000", "#") + except Exception as e: + print(RED(str(e))) + if getattr(self, "port", None): + suggestion = ( + "\nSuggession: Check if the firewall on [ %s ] " % self.ip + + "is stopped\n" + ) + print(GREEN(suggestion)) + + raise SSHConnectionException(self.node) + + def send_expect_base(self, command: str, expected: str, timeout: float) -> str: + self.clean_session() + self.session.PROMPT = expected + self.__sendline(command) + self.__prompt(command, timeout) + + before = self.get_output_before() + return before + + def send_expect( + self, command: str, expected: str, timeout: float = 15, + verify: bool = False + ) -> str | int: + + try: + ret = self.send_expect_base(command, expected, timeout) + if verify: + ret_status = self.send_expect_base("echo $?", expected, timeout) + if not int(ret_status): + return ret + else: + self.logger.error("Command: %s failure!" % command) + self.logger.error(ret) + return int(ret_status) + else: + return ret + except Exception as e: + print( + RED( + "Exception happened in [%s] and output is [%s]" + % (command, self.get_output_before()) + ) + ) + raise e + + def send_command(self, command: str, timeout: float = 1) -> str: + try: + self.clean_session() + self.__sendline(command) + except Exception as e: + raise e + + output = self.get_session_before(timeout=timeout) + self.session.PROMPT = self.session.UNIQUE_PROMPT + self.session.prompt(0.1) + + return output + + def clean_session(self) -> None: + self.get_session_before(timeout=0.01) + + def get_session_before(self, timeout: float = 15) -> str: + """ + Get all output before timeout + """ + self.session.PROMPT = self.magic_prompt + try: + self.session.prompt(timeout) + except Exception as e: + pass + + before = self.get_output_all() + self.__flush() + + return before + + def __flush(self) -> None: + """ + Clear all session buffer + """ + self.session.buffer = "" + self.session.before = "" + + def __prompt(self, command: str, timeout: float) -> None: + if not self.session.prompt(timeout): + raise TimeoutException(command, self.get_output_all()) from None + + def __sendline(self, command: str) -> None: + if not self.isalive(): + raise SSHSessionDeadException(self.node) + if len(command) == 2 and command.startswith("^"): + self.session.sendcontrol(command[1]) + else: + self.session.sendline(command) + + def get_output_before(self) -> str: + if not self.isalive(): + raise SSHSessionDeadException(self.node) + before: list[str] = self.session.before.rsplit("\r\n", 1) + if before[0] == "[PEXPECT]": + before[0] = "" + + return before[0] + + def get_output_all(self) -> str: + output: str = self.session.before + output.replace("[PEXPECT]", "") + return output + + def close(self, force: bool = False) -> None: + if force is True: + self.session.close() + else: + if self.isalive(): + self.session.logout() + + def isalive(self) -> bool: + return self.session.isalive() diff --git a/dts/framework/utils.py b/dts/framework/utils.py new file mode 100644 index 0000000000..7036843dd7 --- /dev/null +++ b/dts/framework/utils.py @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2014 Intel Corporation +# + + +def RED(text: str) -> str: + return f"\u001B[31;1m{str(text)}\u001B[0m" + + +def GREEN(text: str) -> str: + return f"\u001B[32;1m{str(text)}\u001B[0m" From patchwork Mon Jul 11 14:51:21 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Juraj_Linke=C5=A1?= X-Patchwork-Id: 113906 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 8E836A0032; Mon, 11 Jul 2022 16:51:51 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 6EA8F42905; Mon, 11 Jul 2022 16:51:35 +0200 (CEST) Received: from lb.pantheon.sk (lb.pantheon.sk [46.229.239.20]) by mails.dpdk.org (Postfix) with ESMTP id E8A3742826 for ; Mon, 11 Jul 2022 16:51:33 +0200 (CEST) Received: from localhost (localhost [127.0.0.1]) by lb.pantheon.sk (Postfix) with ESMTP id EFB87243CE6; Mon, 11 Jul 2022 16:51:31 +0200 (CEST) X-Virus-Scanned: amavisd-new at siecit.sk Received: from lb.pantheon.sk ([127.0.0.1]) by localhost (lb.pantheon.sk [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id sGpTv19ZAyb0; Mon, 11 Jul 2022 16:51:31 +0200 (CEST) Received: from entguard.lab.pantheon.local (unknown [46.229.239.141]) by lb.pantheon.sk (Postfix) with ESMTP id B822C243CDC; Mon, 11 Jul 2022 16:51:28 +0200 (CEST) From: =?utf-8?q?Juraj_Linke=C5=A1?= To: thomas@monjalon.net, david.marchand@redhat.com, jerinjacobk@gmail.com, ronan.randles@intel.com, Honnappa.Nagarahalli@arm.com, ohilyard@iol.unh.edu, lijuan.tu@intel.com Cc: dev@dpdk.org, =?utf-8?q?Juraj_Linke=C5=A1?= Subject: [PATCH v2 3/8] dts: add locks for parallel node connections Date: Mon, 11 Jul 2022 14:51:21 +0000 Message-Id: <20220711145126.295427-4-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20220711145126.295427-1-juraj.linkes@pantheon.tech> References: <20220622121448.3304251-1-juraj.linkes@pantheon.tech> <20220711145126.295427-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Each lock is held per node. The lock assures that multiple connections to the same node don't execute anything at the same time, removing the possibility of race conditions. Signed-off-by: Juraj Linkeš --- dts/framework/ssh_pexpect.py | 14 ++++-- dts/framework/utils.py | 88 ++++++++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 3 deletions(-) diff --git a/dts/framework/ssh_pexpect.py b/dts/framework/ssh_pexpect.py index c73c1048a4..01ebd1c010 100644 --- a/dts/framework/ssh_pexpect.py +++ b/dts/framework/ssh_pexpect.py @@ -12,7 +12,7 @@ from .exception import (SSHConnectionException, SSHSessionDeadException, TimeoutException) from .logger import DTSLOG -from .utils import GREEN, RED +from .utils import GREEN, RED, parallel_lock """ Module handles ssh sessions to TG and SUT. @@ -33,6 +33,7 @@ def __init__( username: str, password: Optional[str], logger: DTSLOG, + sut_id: int, ): self.magic_prompt = "MAGIC PROMPT" self.logger = logger @@ -42,11 +43,18 @@ def __init__( self.password = password or "" self.logger.info(f"ssh {self.username}@{self.node}") - self._connect_host() + self._connect_host(sut_id=sut_id) - def _connect_host(self) -> None: + @parallel_lock(num=8) + def _connect_host(self, sut_id: int = 0) -> None: """ Create connection to assigned node. + Parameter sut_id will be used in parallel_lock thus can assure + isolated locks for each node. + Parallel ssh connections are limited to MaxStartups option in SSHD + configuration file. By default concurrent number is 10, so default + threads number is limited to 8 which less than 10. Lock number can + be modified along with MaxStartups value. """ retry_times = 10 try: diff --git a/dts/framework/utils.py b/dts/framework/utils.py index 7036843dd7..a637c4641e 100644 --- a/dts/framework/utils.py +++ b/dts/framework/utils.py @@ -1,7 +1,95 @@ # SPDX-License-Identifier: BSD-3-Clause # Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2022 PANTHEON.tech s.r.o. +# Copyright(c) 2022 University of New Hampshire # +import threading +from functools import wraps +from typing import Any, Callable, TypeVar + +locks_info: list[dict[str, Any]] = list() + +T = TypeVar("T") + + +def parallel_lock(num: int = 1) -> Callable[[Callable[..., T]], Callable[..., T]]: + """ + Wrapper function for protect parallel threads, allow multiple threads + share one lock. Locks are created based on function name. Thread locks are + separated between SUTs according to argument 'sut_id'. + Parameter: + num: Number of parallel threads for the lock + """ + global locks_info + + def decorate(func: Callable[..., T]) -> Callable[..., T]: + # mypy does not know how to handle the types of this function, so Any is required + @wraps(func) + def wrapper(*args: Any, **kwargs: Any) -> T: + if "sut_id" in kwargs: + sut_id = kwargs["sut_id"] + else: + sut_id = 0 + + # in case function arguments is not correct + if sut_id >= len(locks_info): + sut_id = 0 + + lock_info = locks_info[sut_id] + uplock = lock_info["update_lock"] + + name = func.__name__ + uplock.acquire() + + if name not in lock_info: + lock_info[name] = dict() + lock_info[name]["lock"] = threading.RLock() + lock_info[name]["current_thread"] = 1 + else: + lock_info[name]["current_thread"] += 1 + + lock = lock_info[name]["lock"] + + # make sure when owned global lock, should also own update lock + if lock_info[name]["current_thread"] >= num: + if lock._is_owned(): + print( + RED( + f"SUT{sut_id:d} {threading.current_thread().name} waiting for func lock {func.__name__}" + ) + ) + lock.acquire() + else: + uplock.release() + + try: + ret = func(*args, **kwargs) + except Exception as e: + if not uplock._is_owned(): + uplock.acquire() + + if lock._is_owned(): + lock.release() + lock_info[name]["current_thread"] = 0 + uplock.release() + raise e + + if not uplock._is_owned(): + uplock.acquire() + + if lock._is_owned(): + lock.release() + lock_info[name]["current_thread"] = 0 + + uplock.release() + + return ret + + return wrapper + + return decorate + def RED(text: str) -> str: return f"\u001B[31;1m{str(text)}\u001B[0m" From patchwork Mon Jul 11 14:51:22 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Juraj_Linke=C5=A1?= X-Patchwork-Id: 113907 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 07153A0032; Mon, 11 Jul 2022 16:51:58 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 84AF942B70; Mon, 11 Jul 2022 16:51:37 +0200 (CEST) Received: from lb.pantheon.sk (lb.pantheon.sk [46.229.239.20]) by mails.dpdk.org (Postfix) with ESMTP id 54FC342836 for ; Mon, 11 Jul 2022 16:51:35 +0200 (CEST) Received: from localhost (localhost [127.0.0.1]) by lb.pantheon.sk (Postfix) with ESMTP id 8F35F243CD1; Mon, 11 Jul 2022 16:51:34 +0200 (CEST) X-Virus-Scanned: amavisd-new at siecit.sk Received: from lb.pantheon.sk ([127.0.0.1]) by localhost (lb.pantheon.sk [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id NsbEenskm3fU; Mon, 11 Jul 2022 16:51:33 +0200 (CEST) Received: from entguard.lab.pantheon.local (unknown [46.229.239.141]) by lb.pantheon.sk (Postfix) with ESMTP id 9F769243CE2; Mon, 11 Jul 2022 16:51:29 +0200 (CEST) From: =?utf-8?q?Juraj_Linke=C5=A1?= To: thomas@monjalon.net, david.marchand@redhat.com, jerinjacobk@gmail.com, ronan.randles@intel.com, Honnappa.Nagarahalli@arm.com, ohilyard@iol.unh.edu, lijuan.tu@intel.com Cc: dev@dpdk.org, =?utf-8?q?Juraj_Linke=C5=A1?= Subject: [PATCH v2 4/8] dts: add ssh connection extension Date: Mon, 11 Jul 2022 14:51:22 +0000 Message-Id: <20220711145126.295427-5-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20220711145126.295427-1-juraj.linkes@pantheon.tech> References: <20220622121448.3304251-1-juraj.linkes@pantheon.tech> <20220711145126.295427-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org The class adds logging and history records to existing pexpect methods. Signed-off-by: Juraj Linkeš --- dts/framework/ssh_connection.py | 70 +++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 dts/framework/ssh_connection.py diff --git a/dts/framework/ssh_connection.py b/dts/framework/ssh_connection.py new file mode 100644 index 0000000000..0ffde0bc58 --- /dev/null +++ b/dts/framework/ssh_connection.py @@ -0,0 +1,70 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2022 PANTHEON.tech s.r.o. +# Copyright(c) 2022 University of New Hampshire +# +import dataclasses +from typing import Any, Optional + +from .logger import DTSLOG +from .ssh_pexpect import SSHPexpect + + +@dataclasses.dataclass(slots=True, frozen=True) +class HistoryRecord: + command: str + name: str + output: str | int + + +class SSHConnection(object): + """ + Module for create session to node. + """ + + name: str + history: list[HistoryRecord] + logger: DTSLOG + session: SSHPexpect | Any + + def __init__( + self, + node: str, + session_name: str, + logger: DTSLOG, + username: str, + password: Optional[str] = "", + sut_id: int = 0, + ): + self.session = SSHPexpect(node, username, password, logger, sut_id=sut_id) + self.name = session_name + self.history = [] + self.logger = logger + + def send_expect( + self, cmds: str, expected: str, timeout: int = 15, verify: bool = False + ) -> str | int: + self.logger.info(cmds) + out = self.session.send_expect(cmds, expected, timeout, verify) + if isinstance(out, str): + self.logger.debug(out.replace(cmds, "")) + self.history.append(HistoryRecord(command=cmds, name=self.name, output=out)) + return out + + def send_command(self, cmds: str, timeout: float = 1) -> str: + self.logger.info(cmds) + out = self.session.send_command(cmds, timeout) + self.logger.debug(out.replace(cmds, "")) + self.history.append(HistoryRecord(command=cmds, name=self.name, output=out)) + return out + + def get_session_before(self, timeout: float = 15) -> str: + out = self.session.get_session_before(timeout) + self.logger.debug(out) + return out + + def close(self, force: bool = False) -> None: + if getattr(self, "logger", None): + self.logger.logger_exit() + + self.session.close(force) From patchwork Mon Jul 11 14:51:23 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Juraj_Linke=C5=A1?= X-Patchwork-Id: 113908 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 77D4EA0032; Mon, 11 Jul 2022 16:52:04 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 84BED42B74; Mon, 11 Jul 2022 16:51:38 +0200 (CEST) Received: from lb.pantheon.sk (lb.pantheon.sk [46.229.239.20]) by mails.dpdk.org (Postfix) with ESMTP id ED35042B6D for ; Mon, 11 Jul 2022 16:51:35 +0200 (CEST) Received: from localhost (localhost [127.0.0.1]) by lb.pantheon.sk (Postfix) with ESMTP id 2DDC6243CDC; Mon, 11 Jul 2022 16:51:35 +0200 (CEST) X-Virus-Scanned: amavisd-new at siecit.sk Received: from lb.pantheon.sk ([127.0.0.1]) by localhost (lb.pantheon.sk [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id lDJKJ0Day2TC; Mon, 11 Jul 2022 16:51:33 +0200 (CEST) Received: from entguard.lab.pantheon.local (unknown [46.229.239.141]) by lb.pantheon.sk (Postfix) with ESMTP id B96851DE465; Mon, 11 Jul 2022 16:51:30 +0200 (CEST) From: =?utf-8?q?Juraj_Linke=C5=A1?= To: thomas@monjalon.net, david.marchand@redhat.com, jerinjacobk@gmail.com, ronan.randles@intel.com, Honnappa.Nagarahalli@arm.com, ohilyard@iol.unh.edu, lijuan.tu@intel.com Cc: dev@dpdk.org, =?utf-8?q?Juraj_Linke=C5=A1?= Subject: [PATCH v2 5/8] dts: add config parser module Date: Mon, 11 Jul 2022 14:51:23 +0000 Message-Id: <20220711145126.295427-6-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20220711145126.295427-1-juraj.linkes@pantheon.tech> References: <20220622121448.3304251-1-juraj.linkes@pantheon.tech> <20220711145126.295427-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org From: Owen Hilyard The configuration is split into two parts, one defining the parameters of the test run and the other defining the topology to be used. The format of the configuration is YAML. It is validated according to a json schema which also servers as detailed documentation of the various configuration fields. This means that the complete set of allowed values are tied to the schema as a source of truth. This enables making changes to parts of DTS that interface with config files without a high risk of breaking someone's configuration. This configuration system uses immutable objects to represent the configuration, making IDE/LSP autocomplete work properly. Signed-off-by: Owen Hilyard Signed-off-by: Juraj Linkeš --- dts/conf.yaml | 7 ++ dts/framework/config/__init__.py | 116 +++++++++++++++++++++ dts/framework/config/conf_yaml_schema.json | 73 +++++++++++++ dts/framework/exception.py | 15 +++ dts/framework/settings.py | 40 +++++++ 5 files changed, 251 insertions(+) create mode 100644 dts/conf.yaml create mode 100644 dts/framework/config/__init__.py create mode 100644 dts/framework/config/conf_yaml_schema.json create mode 100644 dts/framework/settings.py diff --git a/dts/conf.yaml b/dts/conf.yaml new file mode 100644 index 0000000000..11b5f53a66 --- /dev/null +++ b/dts/conf.yaml @@ -0,0 +1,7 @@ +executions: + - system_under_test: "SUT 1" +nodes: + - name: "SUT 1" + hostname: "SUT IP Address or hostname" + user: root + password: "leave blank to use SSH keys" diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py new file mode 100644 index 0000000000..511e70c9a5 --- /dev/null +++ b/dts/framework/config/__init__.py @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2021 Intel Corporation +# Copyright(c) 2022 University of New Hampshire +# + +""" +Generic port and topology nodes configuration file load function +""" +import json +import os.path +import pathlib +from dataclasses import dataclass +from enum import Enum, auto, unique +from typing import Any, Optional + +import warlock +import yaml + +from framework.settings import get_config_file_path + + +class StrEnum(Enum): + @staticmethod + def _generate_next_value_( + name: str, start: int, count: int, last_values: object + ) -> str: + return name + + +@unique +class NodeType(StrEnum): + physical = auto() + virtual = auto() + + +# Slots enables some optimizations, by pre-allocating space for the defined +# attributes in the underlying data structure. +# +# Frozen makes the object immutable. This enables further optimizations, +# and makes it thread safe should we every want to move in that direction. +@dataclass(slots=True, frozen=True) +class NodeConfiguration: + name: str + hostname: str + user: str + password: Optional[str] + + @staticmethod + def from_dict(d: dict) -> "NodeConfiguration": + return NodeConfiguration( + name=d["name"], + hostname=d["hostname"], + user=d["user"], + password=d.get("password"), + ) + + +@dataclass(slots=True, frozen=True) +class ExecutionConfiguration: + system_under_test: str + + @staticmethod + def from_dict(d: dict) -> "ExecutionConfiguration": + return ExecutionConfiguration( + system_under_test=d["system_under_test"], + ) + + +@dataclass(slots=True, frozen=True) +class Configuration: + executions: list[ExecutionConfiguration] + nodes: list[NodeConfiguration] + + @staticmethod + def from_dict(d: dict) -> "Configuration": + executions: list[ExecutionConfiguration] = list( + map(ExecutionConfiguration.from_dict, d["executions"]) + ) + nodes: list[NodeConfiguration] = list( + map(NodeConfiguration.from_dict, d["nodes"]) + ) + assert len(nodes) > 0, "There must be a node to test" + + for i, n1 in enumerate(nodes): + for j, n2 in enumerate(nodes): + if i != j: + assert n1.name == n2.name, "Duplicate node names are not allowed" + + node_names = {node.name for node in nodes} + for execution in executions: + assert ( + execution.system_under_test in node_names + ), f"Unknown SUT {execution.system_under_test} in execution" + + return Configuration(executions=executions, nodes=nodes) + + +def load_config(conf_file_path: str) -> Configuration: + """ + Loads the configuration file and the configuration file schema, + validates the configuration file, and creates a configuration object. + """ + conf_file_path: str = get_config_file_path(conf_file_path) + with open(conf_file_path, "r") as f: + config_data = yaml.safe_load(f) + schema_path = os.path.join( + pathlib.Path(__file__).parent.resolve(), "conf_yaml_schema.json" + ) + + with open(schema_path, "r") as f: + schema = json.load(f) + config: dict[str, Any] = warlock.model_factory(schema, name="_Config")( + config_data + ) + config_obj: Configuration = Configuration.from_dict(dict(config)) + return config_obj diff --git a/dts/framework/config/conf_yaml_schema.json b/dts/framework/config/conf_yaml_schema.json new file mode 100644 index 0000000000..8dea50e285 --- /dev/null +++ b/dts/framework/config/conf_yaml_schema.json @@ -0,0 +1,73 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "title": "DPDK DTS Config Schema", + "definitions": { + "node_name": { + "type": "string", + "description": "A unique identifier for a node" + }, + "node_role": { + "type": "string", + "description": "The role a node plays in DTS", + "enum": [ + "system_under_test", + "traffic_generator" + ] + } + }, + "type": "object", + "properties": { + "nodes": { + "type": "array", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "A unique identifier for this node" + }, + "hostname": { + "type": "string", + "description": "A hostname from which the node running DTS can access this node. This can also be an IP address." + }, + "user": { + "type": "string", + "description": "The user to access this node with." + }, + "password": { + "type": "string", + "description": "The password to use on this node. SSH keys are preferred." + } + }, + "additionalProperties": false, + "required": [ + "name", + "hostname", + "user" + ] + }, + "minimum": 1 + }, + "executions": { + "type": "array", + "items": { + "type": "object", + "properties": { + "system_under_test": { + "$ref": "#/definitions/node_name" + } + }, + "additionalProperties": false, + "required": [ + "system_under_test" + ] + }, + "minimum": 1 + } + }, + "required": [ + "executions", + "nodes" + ], + "additionalProperties": false +} \ No newline at end of file diff --git a/dts/framework/exception.py b/dts/framework/exception.py index 54f9f189a4..252fa47fc4 100644 --- a/dts/framework/exception.py +++ b/dts/framework/exception.py @@ -58,3 +58,18 @@ def __init__(self, host: str): def __str__(self) -> str: return f"SSH session with {self.host} has died" + + +class ConfigParseException(Exception): + + """ + Configuration file parse failure exception. + """ + + config: str + + def __init__(self, conf_file: str): + self.config = conf_file + + def __str__(self) -> str: + return f"Failed to parse config file [{self.config}]" diff --git a/dts/framework/settings.py b/dts/framework/settings.py new file mode 100644 index 0000000000..ca3cbd71b1 --- /dev/null +++ b/dts/framework/settings.py @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2021 Intel Corporation +# Copyright(c) 2022 PANTHEON.tech s.r.o. +# Copyright(c) 2022 University of New Hampshire +# + +import os +import re + +DEFAULT_CONFIG_FILE_PATH: str = "./conf.yaml" + +# DTS global environment variables +DTS_ENV_PAT: str = r"DTS_*" +DTS_CFG_FILE: str = "DTS_CFG_FILE" + + +def load_global_setting(key: str) -> str: + """ + Load DTS global setting + """ + if re.match(DTS_ENV_PAT, key): + env_key = key + else: + env_key = "DTS_" + key + + return os.environ.get(env_key, "") + + +def get_config_file_path(conf_file_path: str) -> str: + """ + The root path of framework configs. + """ + if conf_file_path == DEFAULT_CONFIG_FILE_PATH: + # if the user didn't specify a path on cmdline, they could've specified + # it in the env variable + conf_file_path = load_global_setting(DTS_CFG_FILE) + if conf_file_path == "": + conf_file_path = DEFAULT_CONFIG_FILE_PATH + + return conf_file_path From patchwork Mon Jul 11 14:51:24 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Juraj_Linke=C5=A1?= X-Patchwork-Id: 113909 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 44639A0032; Mon, 11 Jul 2022 16:52:13 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id CBFCF42B7D; Mon, 11 Jul 2022 16:51:39 +0200 (CEST) Received: from lb.pantheon.sk (lb.pantheon.sk [46.229.239.20]) by mails.dpdk.org (Postfix) with ESMTP id DA9E541611 for ; Mon, 11 Jul 2022 16:51:36 +0200 (CEST) Received: from localhost (localhost [127.0.0.1]) by lb.pantheon.sk (Postfix) with ESMTP id 1D79D243CC6; Mon, 11 Jul 2022 16:51:36 +0200 (CEST) X-Virus-Scanned: amavisd-new at siecit.sk Received: from lb.pantheon.sk ([127.0.0.1]) by localhost (lb.pantheon.sk [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id NfdyPyYL8XmX; Mon, 11 Jul 2022 16:51:35 +0200 (CEST) Received: from entguard.lab.pantheon.local (unknown [46.229.239.141]) by lb.pantheon.sk (Postfix) with ESMTP id 51EDE243CCB; Mon, 11 Jul 2022 16:51:31 +0200 (CEST) From: =?utf-8?q?Juraj_Linke=C5=A1?= To: thomas@monjalon.net, david.marchand@redhat.com, jerinjacobk@gmail.com, ronan.randles@intel.com, Honnappa.Nagarahalli@arm.com, ohilyard@iol.unh.edu, lijuan.tu@intel.com Cc: dev@dpdk.org, =?utf-8?q?Juraj_Linke=C5=A1?= Subject: [PATCH v2 6/8] dts: add Node base class Date: Mon, 11 Jul 2022 14:51:24 +0000 Message-Id: <20220711145126.295427-7-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20220711145126.295427-1-juraj.linkes@pantheon.tech> References: <20220622121448.3304251-1-juraj.linkes@pantheon.tech> <20220711145126.295427-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org The base class implements basic node management methods - connect and execute commands. Signed-off-by: Juraj Linkeš --- dts/framework/node.py | 108 ++++++++++++++++++++++++++++++++++++++ dts/framework/settings.py | 3 ++ 2 files changed, 111 insertions(+) create mode 100644 dts/framework/node.py diff --git a/dts/framework/node.py b/dts/framework/node.py new file mode 100644 index 0000000000..09f567f32f --- /dev/null +++ b/dts/framework/node.py @@ -0,0 +1,108 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2022 PANTHEON.tech s.r.o. +# Copyright(c) 2022 University of New Hampshire +# + +from typing import Optional + +from .config import NodeConfiguration +from .logger import DTSLOG, getLogger +from .settings import TIMEOUT +from .ssh_connection import SSHConnection + +""" +A node is a generic host that DTS connects to and manages. +""" + + +class Node(object): + """ + Basic module for node management. This module implements methods that + manage a node, such as information gathering (of CPU/PCI/NIC) and + environment setup. + """ + + _config: NodeConfiguration + name: str + sut_id: int + logger: DTSLOG + session: SSHConnection + + def __init__(self, node_config: NodeConfiguration, sut_id: int = 0): + self._config = node_config + self.name = node_config.name + self.sut_id = sut_id + + self.logger = getLogger(self.name) + self.logger.info(f"Created node: {self.name}") + self.session = SSHConnection( + self.get_ip_address(), + self.name, + self.logger, + self.get_username(), + self.get_password(), + self.sut_id, + ) + + def get_ip_address(self) -> str: + """ + Get SUT's ip address. + """ + return self._config.hostname + + def get_password(self) -> Optional[str]: + """ + Get SUT's login password. + """ + return self._config.password + + def get_username(self) -> str: + """ + Get SUT's login username. + """ + return self._config.user + + def send_expect( + self, + command: str, + expected: str, + timeout: int = TIMEOUT, + verify: bool = False, + trim_whitespace: bool = True, + ) -> str | int: + """ + Send commands to node and return string before expected string. If + there's no expected string found before timeout, TimeoutException will + be raised. + + By default, it will trim the whitespace from the expected string. This + behavior can be turned off via the trim_whitespace argument. + """ + + if trim_whitespace: + expected = expected.strip() + + return self.session.send_expect(command, expected, timeout, verify) + + def send_command(self, cmds: str, timeout: float = TIMEOUT) -> str: + """ + Send commands to node and return string before timeout. + """ + + return self.session.send_command(cmds, timeout) + + def close(self) -> None: + """ + Close ssh session of SUT. + """ + if self.session: + self.session.close() + self.session = None + + def node_exit(self) -> None: + """ + Recover all resource before node exit + """ + self.close() + self.logger.logger_exit() diff --git a/dts/framework/settings.py b/dts/framework/settings.py index ca3cbd71b1..f5028e88c9 100644 --- a/dts/framework/settings.py +++ b/dts/framework/settings.py @@ -7,6 +7,9 @@ import os import re +# Default session timeout. +TIMEOUT: int = 15 + DEFAULT_CONFIG_FILE_PATH: str = "./conf.yaml" # DTS global environment variables From patchwork Mon Jul 11 14:51:25 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Juraj_Linke=C5=A1?= X-Patchwork-Id: 113910 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 62F92A0032; Mon, 11 Jul 2022 16:52:20 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 3DA6542B82; Mon, 11 Jul 2022 16:51:41 +0200 (CEST) Received: from lb.pantheon.sk (lb.pantheon.sk [46.229.239.20]) by mails.dpdk.org (Postfix) with ESMTP id 7CCE942826 for ; Mon, 11 Jul 2022 16:51:37 +0200 (CEST) Received: from localhost (localhost [127.0.0.1]) by lb.pantheon.sk (Postfix) with ESMTP id B53AE243CE2; Mon, 11 Jul 2022 16:51:36 +0200 (CEST) X-Virus-Scanned: amavisd-new at siecit.sk Received: from lb.pantheon.sk ([127.0.0.1]) by localhost (lb.pantheon.sk [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id Jqvw7E3Li-Gl; Mon, 11 Jul 2022 16:51:35 +0200 (CEST) Received: from entguard.lab.pantheon.local (unknown [46.229.239.141]) by lb.pantheon.sk (Postfix) with ESMTP id CB2E3243CE5; Mon, 11 Jul 2022 16:51:31 +0200 (CEST) From: =?utf-8?q?Juraj_Linke=C5=A1?= To: thomas@monjalon.net, david.marchand@redhat.com, jerinjacobk@gmail.com, ronan.randles@intel.com, Honnappa.Nagarahalli@arm.com, ohilyard@iol.unh.edu, lijuan.tu@intel.com Cc: dev@dpdk.org, =?utf-8?q?Juraj_Linke=C5=A1?= Subject: [PATCH v2 7/8] dts: add dts workflow module Date: Mon, 11 Jul 2022 14:51:25 +0000 Message-Id: <20220711145126.295427-8-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20220711145126.295427-1-juraj.linkes@pantheon.tech> References: <20220622121448.3304251-1-juraj.linkes@pantheon.tech> <20220711145126.295427-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org The module implements methods needed to run DTS. It handles the creation of objects and eventually the whole DTS workflow, such as running node setups, test gathering, setup and execution and various cleanups. Signed-off-by: Juraj Linkeš --- dts/framework/dts.py | 76 ++++++++++++++++++++++++++++++++++++++++++ dts/framework/utils.py | 35 +++++++++++++++++-- 2 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 dts/framework/dts.py diff --git a/dts/framework/dts.py b/dts/framework/dts.py new file mode 100644 index 0000000000..52ec0638df --- /dev/null +++ b/dts/framework/dts.py @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2019 Intel Corporation +# Copyright(c) 2022 PANTHEON.tech s.r.o. +# Copyright(c) 2022 University of New Hampshire +# + +import sys +from typing import Iterable, Optional + +import framework.logger as logger + +from .config import Configuration, load_config +from .logger import getLogger +from .node import Node +from .utils import check_dts_python_version, create_parallel_locks + +log_handler: Optional[logger.DTSLOG] = None + + +def dts_nodes_exit(nodes: Iterable[Node]) -> None: + """ + Call SUT and TG exit function after execution finished + """ + for node in nodes: + node.node_exit() + + +def run_all( + config_file, + verbose, +) -> None: + """ + Main process of DTS, it will run all test suites in the config file. + """ + + global log_handler + + # check the python version of the server that run dts + check_dts_python_version() + + # init log_handler handler + if verbose is True: + logger.set_verbose() + + log_handler = getLogger("dts") + + # parse input config file + config: Configuration = load_config(config_file) + + # init global lock + create_parallel_locks(len(config.nodes)) + + nodes = [] + try: + nodes = [ + Node(node_config, sut_id=i) + for i, node_config in enumerate(config.nodes) + ] + dts_nodes_exit(nodes) + finally: + quit_execution(nodes) + + +def quit_execution(nodes: Iterable[Node]) -> None: + """ + Close session to SUT and TG before quit. + Return exit status when failure occurred. + """ + for node in nodes: + # close all session + node.node_exit() + + if log_handler is not None: + log_handler.info("DTS ended") + + sys.exit(0) diff --git a/dts/framework/utils.py b/dts/framework/utils.py index a637c4641e..1f7f28d0c5 100644 --- a/dts/framework/utils.py +++ b/dts/framework/utils.py @@ -4,15 +4,29 @@ # Copyright(c) 2022 University of New Hampshire # +import sys import threading from functools import wraps from typing import Any, Callable, TypeVar locks_info: list[dict[str, Any]] = list() -T = TypeVar("T") + +def create_parallel_locks(num_suts: int) -> None: + """ + Create thread lock dictionary based on SUTs number + """ + global locks_info + + locks_info = list() + for _ in range(num_suts): + lock_info = dict() + lock_info["update_lock"] = threading.RLock() + locks_info.append(lock_info) +T = TypeVar("T") + def parallel_lock(num: int = 1) -> Callable[[Callable[..., T]], Callable[..., T]]: """ Wrapper function for protect parallel threads, allow multiple threads @@ -21,7 +35,6 @@ def parallel_lock(num: int = 1) -> Callable[[Callable[..., T]], Callable[..., T] Parameter: num: Number of parallel threads for the lock """ - global locks_info def decorate(func: Callable[..., T]) -> Callable[..., T]: # mypy does not know how to handle the types of this function, so Any is required @@ -97,3 +110,21 @@ def RED(text: str) -> str: def GREEN(text: str) -> str: return f"\u001B[32;1m{str(text)}\u001B[0m" + + +def check_dts_python_version() -> None: + if ( + sys.version_info.major < 3 + or (sys.version_info.major == 3 and sys.version_info.minor < 10) + ): + print( + RED( + ( + "WARNING: Dts running node python version is lower than python 3.10, " + "it is deprecated for use in DTS, " + "and will not work in future releases." + ) + ), + file=sys.stderr, + ) + print(RED("Please use Python >= 3.10 instead"), file=sys.stderr) \ No newline at end of file From patchwork Mon Jul 11 14:51:26 2022 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Juraj_Linke=C5=A1?= X-Patchwork-Id: 113911 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 64317A0032; Mon, 11 Jul 2022 16:52:26 +0200 (CEST) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 3D06642B85; Mon, 11 Jul 2022 16:51:42 +0200 (CEST) Received: from lb.pantheon.sk (lb.pantheon.sk [46.229.239.20]) by mails.dpdk.org (Postfix) with ESMTP id 4257442B72 for ; Mon, 11 Jul 2022 16:51:38 +0200 (CEST) Received: from localhost (localhost [127.0.0.1]) by lb.pantheon.sk (Postfix) with ESMTP id 79C15243CDE; Mon, 11 Jul 2022 16:51:37 +0200 (CEST) X-Virus-Scanned: amavisd-new at siecit.sk Received: from lb.pantheon.sk ([127.0.0.1]) by localhost (lb.pantheon.sk [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id idWtxf4cGgEI; Mon, 11 Jul 2022 16:51:36 +0200 (CEST) Received: from entguard.lab.pantheon.local (unknown [46.229.239.141]) by lb.pantheon.sk (Postfix) with ESMTP id D0D33243CEA; Mon, 11 Jul 2022 16:51:32 +0200 (CEST) From: =?utf-8?q?Juraj_Linke=C5=A1?= To: thomas@monjalon.net, david.marchand@redhat.com, jerinjacobk@gmail.com, ronan.randles@intel.com, Honnappa.Nagarahalli@arm.com, ohilyard@iol.unh.edu, lijuan.tu@intel.com Cc: dev@dpdk.org, =?utf-8?q?Juraj_Linke=C5=A1?= Subject: [PATCH v2 8/8] dts: add dts executable script Date: Mon, 11 Jul 2022 14:51:26 +0000 Message-Id: <20220711145126.295427-9-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20220711145126.295427-1-juraj.linkes@pantheon.tech> References: <20220622121448.3304251-1-juraj.linkes@pantheon.tech> <20220711145126.295427-1-juraj.linkes@pantheon.tech> MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org The script is an interface to run DTS with standard argument parser. Signed-off-by: Juraj Linkeš --- dts/main.py | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100755 dts/main.py diff --git a/dts/main.py b/dts/main.py new file mode 100755 index 0000000000..e228604b52 --- /dev/null +++ b/dts/main.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: BSD-3-Clause +# Copyright(c) 2010-2014 Intel Corporation +# Copyright(c) 2022 PANTHEON.tech s.r.o. +# Copyright(c) 2022 University of New Hampshire +# + +""" +A test framework for testing DPDK. +""" + +import argparse +import logging + +from framework import dts +from framework.settings import DEFAULT_CONFIG_FILE_PATH + + +def main() -> None: + # Read cmd-line args + parser = argparse.ArgumentParser(description="DPDK test framework.") + + parser.add_argument( + "--config-file", + default=DEFAULT_CONFIG_FILE_PATH, + help="configuration file that describes the test cases, SUTs and targets", + ) + + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="enable verbose output, all message output on screen", +) + + args = parser.parse_args() + + dts.run_all( + args.config_file, + args.verbose, + ) + + +# Main program begins here +if __name__ == "__main__": + logging.raiseExceptions = True + main()