From patchwork Thu Apr 20 14:02:44 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-Patchwork-Submitter: =?utf-8?q?Juraj_Linke=C5=A1?= X-Patchwork-Id: 126333 X-Patchwork-Delegate: thomas@monjalon.net Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 5FD0042998; Thu, 20 Apr 2023 16:16:29 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id EB71240697; Thu, 20 Apr 2023 16:16:28 +0200 (CEST) Received: from mail-ej1-f54.google.com (mail-ej1-f54.google.com [209.85.218.54]) by mails.dpdk.org (Postfix) with ESMTP id 8534F40687 for ; Thu, 20 Apr 2023 16:16:28 +0200 (CEST) Received: by mail-ej1-f54.google.com with SMTP id u3so6607053ejj.12 for ; Thu, 20 Apr 2023 07:16:28 -0700 (PDT) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=pantheon-tech.20221208.gappssmtp.com; s=20221208; t=1682000188; x=1684592188; h=content-transfer-encoding:mime-version:message-id:date:subject:cc :to:from:from:to:cc:subject:date:message-id:reply-to; bh=n9geadbNa+92ZXByjsvh98Y/nF0j1ocmOtRsz24Dtcs=; b=p7JgmpUFqUuDkCoOoTUtGdwea+fzWl8rp3kNcKYH3oO84+47JSeVIq21zfVxCuWgkI yRRDC4Jy9h4+7q6Yc8VSSiy2LWGgnDzMoHyLUboPajZWfyung8WldQROM5gZosZccXK5 i1dQG1gbsGR0QtxzlxcLWHTWqw7VyhuBfe2owtaXYaBEzf8li1rekrouQmnjafFxul9p PFOf1W1uQ9ehHB30UfBvTxX27aGK7ySLLydNALv/cHpzhT3W+lKH/lO2VfuNjBSzfjR1 NiFkGcn3lWO6gwv/r3jtQsxfyRurOiwjMi8lJRyN8OjfkdHBoMvS/l6HV+y1MFzKJFhm XDmw== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20221208; t=1682000188; x=1684592188; h=content-transfer-encoding:mime-version:message-id:date:subject:cc :to:from:x-gm-message-state:from:to:cc:subject:date:message-id :reply-to; bh=n9geadbNa+92ZXByjsvh98Y/nF0j1ocmOtRsz24Dtcs=; b=aBYUhjDVnIJBWnDFY7gPzOj9qj78ENRxx0DsoNWq5NKglXBoYR20rCtWYlGDms4ESm VDp3MQ1MmAYSaB+W2C2X2RPaDt/tk4gB0JNf2OJEt39HhC+U6zEEXj7D7335w2LJPn5C iU1CBzO8Q4RxSlRpRl5AhuSD9ECgp7BmBemXVz/6LlKgtr7Ier45ECe+oU/PIV4E0Dbz sfYkZEig9o00WhMK4fdzhJx/IYjcDAmZjEARN2EGBD2I9V2xXl1sarNMj+5bUPIBelyz 5UctvLqGb9QWf3uOwEIPo7p06mOuVLsZS83vlLl/icX2STo/I2gTPh0oS5i5xkjAhoKQ OdYQ== X-Gm-Message-State: AAQBX9eAwQoOs/6gtbCJBidVl75pgDRijQXRDPCTg8X/uwc3Z9ujxKwT pzLF2NmPaoRqobmGb6CDFJstP+cVZbl+PTu1IW+gfpzEzpV/0bBFTqzbaaBUPq/2dIImidHsUQ= = X-Google-Smtp-Source: AKy350bmUS+SkFsMIm3Q2hB7TodkfgEPF7/PWU7nva0w9j+EI7A+TIcPuXj9dAE0vvf/HegzGd1DKQ== X-Received: by 2002:a17:906:bc81:b0:94e:70bb:5f8a with SMTP id lv1-20020a170906bc8100b0094e70bb5f8amr1524234ejb.66.1682000188005; Thu, 20 Apr 2023 07:16:28 -0700 (PDT) Received: from localhost.localdomain (ip-46.34.246.150.o2inet.sk. [46.34.246.150]) by smtp.gmail.com with ESMTPSA id z8-20020a1709063ac800b0094f75d6e974sm773840ejd.42.2023.04.20.07.16.26 (version=TLS1_3 cipher=TLS_AES_256_GCM_SHA384 bits=256/256); Thu, 20 Apr 2023 07:16:27 -0700 (PDT) From: =?utf-8?q?Juraj_Linke=C5=A1?= To: thomas@monjalon.net, Honnappa.Nagarahalli@arm.com, lijuan.tu@intel.com, wathsala.vithanage@arm.com, jspewock@iol.unh.edu, probb@iol.unh.edu Cc: dev@dpdk.org, =?utf-8?q?Juraj_Linke=C5=A1?= Subject: [PATCH v1] dts: create tarball from git ref Date: Thu, 20 Apr 2023 16:02:44 +0200 Message-Id: <20230420140244.701467-1-juraj.linkes@pantheon.tech> X-Mailer: git-send-email 2.30.2 MIME-Version: 1.0 X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Add additional convenience options for specifying what DPDK version to test. Signed-off-by: Juraj Linkeš Acked-by: Jeremy Spweock --- dts/framework/config/__init__.py | 11 +-- dts/framework/settings.py | 20 ++--- dts/framework/utils.py | 140 +++++++++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 19 deletions(-) diff --git a/dts/framework/config/__init__.py b/dts/framework/config/__init__.py index ebb0823ff5..a4b73483e6 100644 --- a/dts/framework/config/__init__.py +++ b/dts/framework/config/__init__.py @@ -11,21 +11,14 @@ import os.path import pathlib from dataclasses import dataclass -from enum import Enum, auto, unique +from enum import auto, unique from typing import Any, TypedDict import warlock # type: ignore import yaml from framework.settings import SETTINGS - - -class StrEnum(Enum): - @staticmethod - def _generate_next_value_( - name: str, start: int, count: int, last_values: object - ) -> str: - return name +from framework.utils import StrEnum @unique diff --git a/dts/framework/settings.py b/dts/framework/settings.py index 71955f4581..cfa39d011b 100644 --- a/dts/framework/settings.py +++ b/dts/framework/settings.py @@ -10,7 +10,7 @@ from pathlib import Path from typing import Any, TypeVar -from .exception import ConfigurationError +from .utils import DPDKGitTarball _T = TypeVar("_T") @@ -124,11 +124,13 @@ def _get_parser() -> argparse.ArgumentParser: parser.add_argument( "--tarball", "--snapshot", + "--git-ref", action=_env_arg("DTS_DPDK_TARBALL"), default="dpdk.tar.xz", type=Path, - help="[DTS_DPDK_TARBALL] Path to DPDK source code tarball " - "which will be used in testing.", + help="[DTS_DPDK_TARBALL] Path to DPDK source code tarball or a git commit ID, " + "tag ID or tree ID to test. To test local changes, first commit them, " + "then use the commit ID with this option.", ) parser.add_argument( @@ -160,21 +162,19 @@ def _get_parser() -> argparse.ArgumentParser: return parser -def _check_tarball_path(parsed_args: argparse.Namespace) -> None: - if not os.path.exists(parsed_args.tarball): - raise ConfigurationError(f"DPDK tarball '{parsed_args.tarball}' doesn't exist.") - - def _get_settings() -> _Settings: parsed_args = _get_parser().parse_args() - _check_tarball_path(parsed_args) return _Settings( config_file_path=parsed_args.config_file, output_dir=parsed_args.output_dir, timeout=parsed_args.timeout, verbose=(parsed_args.verbose == "Y"), skip_setup=(parsed_args.skip_setup == "Y"), - dpdk_tarball_path=parsed_args.tarball, + dpdk_tarball_path=Path( + DPDKGitTarball(parsed_args.tarball, parsed_args.output_dir) + ) + if not os.path.exists(parsed_args.tarball) + else Path(parsed_args.tarball), compile_timeout=parsed_args.compile_timeout, test_cases=parsed_args.test_cases.split(",") if parsed_args.test_cases else [], re_run=parsed_args.re_run, diff --git a/dts/framework/utils.py b/dts/framework/utils.py index 55e0b0ef0e..0623106b78 100644 --- a/dts/framework/utils.py +++ b/dts/framework/utils.py @@ -3,7 +3,26 @@ # Copyright(c) 2022-2023 PANTHEON.tech s.r.o. # Copyright(c) 2022-2023 University of New Hampshire +import atexit +import os +import subprocess import sys +from enum import Enum +from pathlib import Path +from subprocess import SubprocessError + +from .exception import ConfigurationError + + +class StrEnum(Enum): + @staticmethod + def _generate_next_value_( + name: str, start: int, count: int, last_values: object + ) -> str: + return name + + def __str__(self) -> str: + return self.name def check_dts_python_version() -> None: @@ -80,3 +99,124 @@ def __init__(self, default_library: str | None = None, **dpdk_args: str | bool): def __str__(self) -> str: return " ".join(f"{self._default_library} {self._dpdk_args}".split()) + + +class _TarCompressionFormat(StrEnum): + """Compression formats that tar can use. + + Enum names are the shell compression commands + and Enum values are the associated file extensions. + """ + + gzip = "gz" + compress = "Z" + bzip2 = "bz2" + lzip = "lz" + lzma = "lzma" + lzop = "lzo" + xz = "xz" + zstd = "zst" + + +class DPDKGitTarball(object): + """Create a compressed tarball of DPDK from the repository. + + The DPDK version is specified with git object git_ref. + The tarball will be compressed with _TarCompressionFormat, + which must be supported by the DTS execution environment. + The resulting tarball will be put into output_dir. + + The class supports the os.PathLike protocol, + which is used to get the Path of the tarball:: + + from pathlib import Path + tarball = DPDKGitTarball("HEAD", "output") + tarball_path = Path(tarball) + + Arguments: + git_ref: A git commit ID, tag ID or tree ID. + output_dir: The directory where to put the resulting tarball. + tar_compression_format: The compression format to use. + """ + + _git_ref: str + _tar_compression_format: _TarCompressionFormat + _tarball_dir: Path + _tarball_name: str + _tarball_path: Path | None + + def __init__( + self, + git_ref: str, + output_dir: str, + tar_compression_format: _TarCompressionFormat = _TarCompressionFormat.xz, + ): + self._git_ref = git_ref + self._tar_compression_format = tar_compression_format + + self._tarball_dir = Path(output_dir, "tarball") + + self._get_commit_id() + self._create_tarball_dir() + + self._tarball_name = ( + f"dpdk-tarball-{self._git_ref}.tar.{self._tar_compression_format.value}" + ) + self._tarball_path = self._check_tarball_path() + if not self._tarball_path: + self._create_tarball() + + def _get_commit_id(self) -> None: + result = subprocess.run( + ["git", "rev-parse", "--verify", self._git_ref], + text=True, + capture_output=True, + ) + if result.returncode != 0: + raise ConfigurationError( + f"{self._git_ref} is neither a path to an existing DPDK " + "archive nor a valid git reference.\n" + f"Command: {result.args}\n" + f"Stdout: {result.stdout}\n" + f"Stderr: {result.stderr}" + ) + self._git_ref = result.stdout.strip() + + def _create_tarball_dir(self) -> None: + os.makedirs(self._tarball_dir, exist_ok=True) + + def _check_tarball_path(self) -> Path | None: + if self._tarball_name in os.listdir(self._tarball_dir): + return Path(self._tarball_dir, self._tarball_name) + return None + + def _create_tarball(self) -> None: + self._tarball_path = Path(self._tarball_dir, self._tarball_name) + + atexit.register(self._delete_tarball) + + result = subprocess.run( + 'git -C "$(git rev-parse --show-toplevel)" archive ' + f'{self._git_ref} --prefix="dpdk-tarball-{self._git_ref + os.sep}" | ' + f"{self._tar_compression_format} > {Path(self._tarball_path.absolute())}", + shell=True, + text=True, + capture_output=True, + ) + + if result.returncode != 0: + raise SubprocessError( + f"Git archive creation failed with exit code {result.returncode}.\n" + f"Command: {result.args}\n" + f"Stdout: {result.stdout}\n" + f"Stderr: {result.stderr}" + ) + + atexit.unregister(self._delete_tarball) + + def _delete_tarball(self) -> None: + if self._tarball_path and os.path.exists(self._tarball_path): + os.remove(self._tarball_path) + + def __fspath__(self): + return str(self._tarball_path)