From patchwork Tue Jun 27 15:31:28 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Yuan, DukaiX" X-Patchwork-Id: 129014 Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 5B7C342D6E; Tue, 27 Jun 2023 09:54:44 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 35AB14113F; Tue, 27 Jun 2023 09:54:44 +0200 (CEST) Received: from mga01.intel.com (mga01.intel.com [192.55.52.88]) by mails.dpdk.org (Postfix) with ESMTP id BCF9240F18 for ; Tue, 27 Jun 2023 09:54:42 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1687852482; x=1719388482; h=from:to:cc:subject:date:message-id:mime-version: content-transfer-encoding; bh=gUQl5t2TuHrGCoqvLLcR1+cq/u/fbjCsneCBnsHh2Nw=; b=UP/OGdlAni0b9yr5zMMHMD5OoGGM6T7ovQeVqb0tEeocj518qSD0X2CC pO12Rru45Zkzzmg/kut+lVblGf9MNFOOx7Sye7u/NateJHhdSfhvYRhdi iU/wuNtMvcBtelaj9xrWOWMCJm4VZaJqsdbJJzeg5S3698OVhuuUtfT4W TqAVYzRUJ5Q5hfIlJOMi573fE7oPDgKkCzoiy3UW3s55+KLkLzliAcXGT JpFPcOKZ046MvZ/7YCGwY1sRsIPEXR0XK3P29YMcct7aq9TBf3lZb5cz1 PRNLybBIt1rgGsavB6oIt3nk2lEfebQ2FUDF99U/7rgNi9AAwzNgglt9p w==; X-IronPort-AV: E=McAfee;i="6600,9927,10753"; a="391994763" X-IronPort-AV: E=Sophos;i="6.01,161,1684825200"; d="scan'208";a="391994763" Received: from orsmga008.jf.intel.com ([10.7.209.65]) by fmsmga101.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 27 Jun 2023 00:54:41 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6600,9927,10753"; a="746106748" X-IronPort-AV: E=Sophos;i="6.01,161,1684825200"; d="scan'208";a="746106748" Received: from unknown (HELO localhost.localdomain) ([10.239.252.44]) by orsmga008-auth.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 27 Jun 2023 00:54:39 -0700 From: Dukai Yuan To: dts@dpdk.org Cc: Dukai Yuan Subject: [dts][PATCH V1] tests/virtio_pvp_regression: Add the functionality of reading expected results from configuration files Date: Tue, 27 Jun 2023 15:31:28 +0000 Message-Id: <20230627153128.627541-1-dukaix.yuan@intel.com> X-Mailer: git-send-email 2.31.1 MIME-Version: 1.0 X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dts-bounces@dpdk.org Previously, expected values in the code were fixed for verification. Now, we have made changes to read expected values from a configuration file. Additionally, test results can also be saved to the configuration file as new expected values. This optimization can improve the accuracy and maintainability of testing. Signed-off-by: Dukai Yuan --- tests/TestSuite_virtio_pvp_regression.py | 177 +++++++++++++++++------ 1 file changed, 131 insertions(+), 46 deletions(-) diff --git a/tests/TestSuite_virtio_pvp_regression.py b/tests/TestSuite_virtio_pvp_regression.py index a2db5f87..a28ebc67 100644 --- a/tests/TestSuite_virtio_pvp_regression.py +++ b/tests/TestSuite_virtio_pvp_regression.py @@ -19,7 +19,7 @@ import time import framework.utils as utils from framework.pktgen import PacketGeneratorHelper -from framework.settings import HEADER_SIZE +from framework.settings import HEADER_SIZE, UPDATE_EXPECTED, load_global_setting from framework.test_case import TestCase from framework.virt_common import VM @@ -73,6 +73,8 @@ class TestVirtioPVPRegression(TestCase): self.base_dir = self.dut.base_dir.replace("~", "/root") self.app_testpmd_path = self.dut.apps_name["test-pmd"] self.testpmd_name = self.app_testpmd_path.split("/")[-1] + self.qemu_num = [item["version"] for item in self.qemu_list] + self.gap = self.get_suite_cfg()["accepted_tolerance"] def set_up(self): """ @@ -94,8 +96,10 @@ class TestVirtioPVPRegression(TestCase): "QemuVersion", "FrameSize(B)", "Throughput(Mpps)", + "Expectput(Mpps)", "LineRate(%)", "Queue Number", + "Status", "Cycle", ] @@ -340,24 +344,17 @@ class TestVirtioPVPRegression(TestCase): self.vhost.send_expect("start", "testpmd> ", 60) - @property - def check_value(self): - check_dict = dict.fromkeys(self.frame_sizes) - linerate = { - 64: 0.08, - 128: 0.10, - 256: 0.17, - 512: 0.18, - 1024: 0.40, - 1280: 0.45, - 1518: 0.50, - } - for size in self.frame_sizes: - speed = self.wirespeed(self.nic, size, self.number_of_ports) - check_dict[size] = round(speed * linerate[size], 2) - return check_dict - - def send_verify(self, case_info, qemu_version, tag): + def handle_expected(self): + """ + Update expected numbers to configurate file: $DTS_CFG_FOLDER/$suite_name.cfg + """ + if load_global_setting(UPDATE_EXPECTED) == "yes": + for self.qemu_version in self.qemu_num: + self.expected_throughput[self.qemu_version] = self.throughput[ + self.case_info + ][self.qemu_version] + + def send_verify(self, case_info, qemu_version, tag, i): for frame_size in self.frame_sizes: info = "Running test %s, and %d frame size." % ( self.running_case, @@ -397,11 +394,19 @@ class TestVirtioPVPRegression(TestCase): ) Mpps = pps / 1000000.0 pct = Mpps * 100 / float(self.wirespeed(self.nic, frame_size, 1)) - self.verify( - Mpps > self.check_value[frame_size], - "%s of frame size %d speed verify failed, expect %s, result %s" - % (self.running_case, frame_size, self.check_value[frame_size], Mpps), - ) + try: + exepect = self.expected_throughput[self.qemu_num[i]][tag][frame_size] + + except: + exepect = 0 + self.logger.info( + 'The first test of %s failed to match the expected result.\ + Please use "--update-expect" to update the expected result.' + % self.qemu_num[i] + ) + + self.status = "PASS" if Mpps > exepect * (1 - self.gap * 0.01) else "FAILED" + # check each queue has data self.check_packets_of_each_queue(frame_size) # update print table info @@ -410,12 +415,36 @@ class TestVirtioPVPRegression(TestCase): qemu_version, frame_size, str(Mpps), + str(exepect), str(pct), self.queues_number, + self.status, tag, ] self.result_table_add(data_row) + def handle_throughput(self): + self.throughput = {} + + for outer in self.get_test_result_table: + for item in outer[1:]: + key1, key2, key3, key4, value = ( + item[0], + item[1], + item[-1], + int(item[2]), + float(item[3]), + ) + if key1 not in self.throughput: + self.throughput[key1] = {} + if key2 not in self.throughput[key1]: + self.throughput[key1][key2] = {} + if key3 not in self.throughput[key1][key2]: + self.throughput[key1][key2][key3] = {} + self.throughput[key1][key2][key3][key4] = value + + print(self.throughput) + def close_testpmd_and_qemu(self): """ stop testpmd in vhost and qemu @@ -435,6 +464,7 @@ class TestVirtioPVPRegression(TestCase): modem = 1, start vm as virtio 1.0 virtio_path = mergeable/normal/vector_rx """ + self.get_test_result_table = [] for i in range(len(self.qemu_list)): self.result_table_create(self.header_row) path = self.qemu_list[i]["path"] @@ -451,93 +481,148 @@ class TestVirtioPVPRegression(TestCase): self.start_testpmd_in_vm(virtio_path) self.logger.info("now testing the qemu path of %s" % path) time.sleep(5) - self.send_verify(case_info, version, "before reconnect") + self.send_verify(case_info, version, "before reconnect", i) - self.logger.info("now reconnect from vhost") - self.dut.send_expect("killall -s INT %s" % self.testpmd_name, "# ") - self.start_testpmd_as_vhost() - self.send_verify(case_info, version, "reconnect from vhost") + if "virtio-1.1" not in self.case_info: + self.logger.info("now reconnect from vhost") + self.dut.send_expect("killall -s INT %s" % self.testpmd_name, "# ") + self.start_testpmd_as_vhost() + self.send_verify(case_info, version, "reconnect from vhost", i) self.logger.info("now reconnect from vm") self.dut.send_expect("killall -s INT qemu-system-x86_64", "# ") self.start_vm(path, version, modem, virtio_path, packed=packed) self.start_testpmd_in_vm(virtio_path) - self.send_verify(case_info, version, "reconnect from vm") + self.send_verify(case_info, version, "reconnect from vm", i) + self.get_test_result_table.append(self.result_table_getrows()) self.result_table_print() + self.verify( + "FAIL" not in self.status, + "Excessive gap between test results and expectations", + ) self.close_testpmd_and_qemu() def test_perf_pvp_regression_with_mergeable_path(self): """ Test Case 1: pvp test with virtio 0.95 mergeable path """ - case_info = "virtio-0.95 mergeable" + self.case_info = "virtio-0.95 mergeable" modem = 0 virtio_path = "mergeable" - self.pvp_regression_run(case_info, modem, virtio_path) + self.test_target = self.running_case + self.expected_throughput = self.get_suite_cfg()[ + "expected_throughput_virtio-0.95" + ][self.test_target][self.case_info] + self.pvp_regression_run(self.case_info, modem, virtio_path) + self.handle_throughput() + self.handle_expected() def test_perf_pvp_regression_modern_mergeable_path(self): """ Test Case 4: pvp test with virtio 1.0 mergeable path """ - case_info = "virtio-1.0 mergeable" + self.case_info = "virtio-1.0 mergeable" modem = 1 virtio_path = "mergeable" - self.pvp_regression_run(case_info, modem, virtio_path) + self.test_target = self.running_case + self.expected_throughput = self.get_suite_cfg()[ + "expected_throughput_virtio-1.0" + ][self.test_target][self.case_info] + self.pvp_regression_run(self.case_info, modem, virtio_path) + self.handle_throughput() + self.handle_expected() def test_perf_pvp_regression_non_mergeable_path(self): """ Test Case 2: pvp test with virtio 0.95 non-mergeable path """ - case_info = "virtio-0.95 normal" + self.case_info = "virtio-0.95 normal" modem = 0 virtio_path = "normal" - self.pvp_regression_run(case_info, modem, virtio_path) + self.test_target = self.running_case + self.expected_throughput = self.get_suite_cfg()[ + "expected_throughput_virtio-0.95" + ][self.test_target][self.case_info] + self.pvp_regression_run(self.case_info, modem, virtio_path) + self.handle_throughput() + self.handle_expected() def test_perf_pvp_regression_modern_non_mergeable_path(self): """ Test Case 5: pvp test with virtio 1.0 non-mergeable path """ - case_info = "virtio-1.0 normal" + + self.case_info = "virtio-1.0 normal" modem = 1 virtio_path = "normal" - self.pvp_regression_run(case_info, modem, virtio_path) + self.test_target = self.running_case + self.expected_throughput = self.get_suite_cfg()[ + "expected_throughput_virtio-1.0" + ][self.test_target][self.case_info] + self.pvp_regression_run(self.case_info, modem, virtio_path) + self.handle_throughput() + self.handle_expected() def test_perf_pvp_regression_vector_rx_path(self): """ Test Case 3: pvp test with virtio 0.95 vrctor_rx path """ - case_info = "virtio-0.95 vector_rx" + self.case_info = "virtio-0.95 vector_rx" modem = 0 virtio_path = "vector_rx" - self.pvp_regression_run(case_info, modem, virtio_path) + self.test_target = self.running_case + self.expected_throughput = self.get_suite_cfg()[ + "expected_throughput_virtio-0.95" + ][self.test_target][self.case_info] + self.pvp_regression_run(self.case_info, modem, virtio_path) + self.handle_throughput() + self.handle_expected() def test_perf_pvp_regression_modern_vector_rx_path(self): """ Test Case 6: pvp test with virtio 1.0 vrctor_rx path """ - case_info = "virtio-1.0 normal" + self.case_info = "virtio-1.0 vector_rx" modem = 1 virtio_path = "vector_rx" - self.pvp_regression_run(case_info, modem, virtio_path) + self.test_target = self.running_case + self.expected_throughput = self.get_suite_cfg()[ + "expected_throughput_virtio-1.0" + ][self.test_target][self.case_info] + self.pvp_regression_run(self.case_info, modem, virtio_path) + self.handle_throughput() + self.handle_expected() def test_perf_pvp_with_virtio11_mergeable_path(self): """ Test Case 7: pvp test with virtio 1.1 mergeable path """ - case_info = "virtio-1.1 mergeable" + self.case_info = "virtio-1.1 mergeable" modem = 1 virtio_path = "mergeable" - self.pvp_regression_run(case_info, modem, virtio_path, packed=True) + self.test_target = self.running_case + self.expected_throughput = self.get_suite_cfg()[ + "expected_throughput_virtio-1.1" + ][self.test_target][self.case_info] + self.pvp_regression_run(self.case_info, modem, virtio_path, packed=True) + self.handle_throughput() + self.handle_expected() def test_perf_pvp_with_virtio11_non_mergeable_path(self): """ Test Case 8: pvp test with virtio 1.1 non-mergeable path """ - case_info = "virtio-1.1 normal" + self.case_info = "virtio-1.1 normal" modem = 1 virtio_path = "normal" - self.pvp_regression_run(case_info, modem, virtio_path, packed=True) + self.test_target = self.running_case + self.expected_throughput = self.get_suite_cfg()[ + "expected_throughput_virtio-1.1" + ][self.test_target][self.case_info] + self.pvp_regression_run(self.case_info, modem, virtio_path, packed=True) + self.handle_throughput() + self.handle_expected() def tear_down(self): """