From patchwork Tue Apr 25 05:46:54 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Ling, WeiX" X-Patchwork-Id: 126499 Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 17654429E5; Tue, 25 Apr 2023 07:48:52 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id E721340A7E; Tue, 25 Apr 2023 07:48:51 +0200 (CEST) Received: from mga05.intel.com (mga05.intel.com [192.55.52.43]) by mails.dpdk.org (Postfix) with ESMTP id F4117400D7 for ; Tue, 25 Apr 2023 07:48:49 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1682401730; x=1713937730; h=from:to:cc:subject:date:message-id:mime-version: content-transfer-encoding; bh=zjFFimeXlc6KfqTl3/1sh1VoU4suplTsS6QmQLu7x0g=; b=NaTic+Rpsoc81ahkYrJyg2rKKRUlszX8MhHPKS5mr5l9QB0Ng6RxIOYt gjyfrxpaBsuN8x56MSbyQs2gfX3/Z/lbxRKauGfZidxuhILMSWNDmD3dQ 94tSIZ3Bqub3IITy1584WCxWxjd6zRcfcQgY/AIealykZHguzx7MWf1OL DTlnUZAHNMpTKrTlqEI8wdulkRk4L7P5/7dQXmBlsbWyLvDU0L1k+40Ue 53zDgl6pXkaGqPYrQS8R2AtSQCAU1+N4UEOcbMYU3+IqXNdV5qsJU2JtM zJzvw0B76WkA64omRe0vqt9P9SJ2tKisDI7gLSpVB5V1Qm/QVp6zm+V8J g==; X-IronPort-AV: E=McAfee;i="6600,9927,10690"; a="432926683" X-IronPort-AV: E=Sophos;i="5.99,224,1677571200"; d="scan'208";a="432926683" Received: from orsmga007.jf.intel.com ([10.7.209.58]) by fmsmga105.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Apr 2023 22:48:48 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6600,9927,10690"; a="687244809" X-IronPort-AV: E=Sophos;i="5.99,224,1677571200"; d="scan'208";a="687244809" Received: from unknown (HELO localhost.localdomain) ([10.239.252.222]) by orsmga007-auth.jf.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Apr 2023 22:48:45 -0700 From: Wei Ling To: dts@dpdk.org Cc: Wei Ling Subject: [dts][PATCH V1] tests/pvp_virtio_user_4k_hugepages: add expected value in suite config to verify Date: Tue, 25 Apr 2023 13:46:54 +0800 Message-Id: <20230425054654.2959821-1-weix.ling@intel.com> X-Mailer: git-send-email 2.25.1 MIME-Version: 1.0 X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dts-bounces@dpdk.org 1.Rename from pvp_virtio_user_4k_pages to pvp_virtio_user_4k_hugepages, so need to modify in test_plans/index.rst, and rename testplan and testsuite. 2.Optimize the verify the expected value from the hard code to read from conf/pvp_virtio_user_4k_hugepages.cfg to adapt different platform. 3.Add new conf/pvp_virtio_user_4k_hugepages.cfg to save the test parameters and expected vaule to verify running vaule. Signed-off-by: Wei Ling Acked-by: Xingguang He Acked-by: Lijuan Tu --- conf/pvp_virtio_user_4k_hugepages.cfg | 11 ++ test_plans/index.rst | 2 +- ...vp_virtio_user_4k_hugepages_test_plan.rst} | 0 ...TestSuite_pvp_virtio_user_4k_hugepages.py} | 154 ++++++++++++------ 4 files changed, 113 insertions(+), 54 deletions(-) create mode 100644 conf/pvp_virtio_user_4k_hugepages.cfg rename test_plans/{pvp_virtio_user_4k_pages_test_plan.rst => pvp_virtio_user_4k_hugepages_test_plan.rst} (100%) rename tests/{TestSuite_pvp_virtio_user_4k_pages.py => TestSuite_pvp_virtio_user_4k_hugepages.py} (61%) diff --git a/conf/pvp_virtio_user_4k_hugepages.cfg b/conf/pvp_virtio_user_4k_hugepages.cfg new file mode 100644 index 00000000..514230e9 --- /dev/null +++ b/conf/pvp_virtio_user_4k_hugepages.cfg @@ -0,0 +1,11 @@ +[suite] +update_expected = True +packet_sizes = [64, 128, 256, 512, 1024, 1518] +test_duration = 30 +accepted_tolerance = 5 +expected_throughput = { + 'test_perf_pvp_virtio_user_split_ring_with_4K_hugepages': { + 64: 0.00, 128: 0.00, 256: 0.00, 512: 0.00, 1024: 0.00, 1518: 0.00}, + 'test_perf_pvp_virtio_user_packed_ring_with_4K_hugepages': { + 64: 0.00, 128: 0.00, 256: 0.00, 512: 0.00, 1024: 0.00, 1518: 0.00}} + diff --git a/test_plans/index.rst b/test_plans/index.rst index 2008f50b..a0c056cd 100644 --- a/test_plans/index.rst +++ b/test_plans/index.rst @@ -260,7 +260,7 @@ The following are the test plans for the DPDK DTS automated test system. pvp_qemu_multi_paths_port_restart_test_plan pvp_vhost_user_reconnect_test_plan pvp_virtio_bonding_test_plan - pvp_virtio_user_4k_pages_test_plan + pvp_virtio_user_4k_hugepages_test_plan vdev_primary_secondary_test_plan vhost_1024_ethports_test_plan virtio_pvp_regression_test_plan diff --git a/test_plans/pvp_virtio_user_4k_pages_test_plan.rst b/test_plans/pvp_virtio_user_4k_hugepages_test_plan.rst similarity index 100% rename from test_plans/pvp_virtio_user_4k_pages_test_plan.rst rename to test_plans/pvp_virtio_user_4k_hugepages_test_plan.rst diff --git a/tests/TestSuite_pvp_virtio_user_4k_pages.py b/tests/TestSuite_pvp_virtio_user_4k_hugepages.py similarity index 61% rename from tests/TestSuite_pvp_virtio_user_4k_pages.py rename to tests/TestSuite_pvp_virtio_user_4k_hugepages.py index 3e3e09b4..f7d2fe57 100644 --- a/tests/TestSuite_pvp_virtio_user_4k_pages.py +++ b/tests/TestSuite_pvp_virtio_user_4k_hugepages.py @@ -2,20 +2,14 @@ # Copyright(c) 2019 Intel Corporation # -""" -DPDK Test suite. -vhost/virtio-user pvp with 4K pages. -""" +from copy import deepcopy -import time - -import framework.utils as utils -from framework.packet import Packet from framework.pktgen import PacketGeneratorHelper +from framework.settings import HEADER_SIZE, UPDATE_EXPECTED, load_global_setting from framework.test_case import TestCase -class TestPvpVirtioUser4kPages(TestCase): +class TestPvpVirtioUser4kHugepages(TestCase): def set_up_all(self): """ Run at the start of each test suite. @@ -41,7 +35,10 @@ class TestPvpVirtioUser4kPages(TestCase): self.core_list_virtio_user = self.core_list[0:2] self.core_list_vhost_user = self.core_list[2:4] self.pci_info = self.dut.ports_info[0]["pci"] + self.nb_ports = 1 + self.gap = self.get_suite_cfg()["accepted_tolerance"] self.dst_mac = self.dut.get_mac_address(self.dut_ports[0]) + self.header_size = HEADER_SIZE["eth"] + HEADER_SIZE["ip"] + HEADER_SIZE["tcp"] self.frame_sizes = [64, 128, 256, 512, 1024, 1518] self.logger.info( "You can config packet_size in file %s.cfg," % self.suite_name @@ -68,63 +65,104 @@ class TestPvpVirtioUser4kPages(TestCase): self.dut.send_expect("killall -s INT %s" % self.testpmd_name, "# ") self.vhost_user = self.dut.new_session(suite="vhost-user") self.virtio_user = self.dut.new_session(suite="virtio-user") + self.throughput = dict() + self.test_result = dict() # Prepare the result table self.table_header = ["Frame"] - self.table_header.append("Mode") self.table_header.append("Mpps") - self.table_header.append("Queue Num") self.table_header.append("% linerate") - self.result_table_create(self.table_header) - @property - def check_value(self): - check_dict = dict.fromkeys(self.frame_sizes) - linerate = { - 64: 0.08, - 128: 0.10, - 256: 0.18, - 512: 0.20, - 1024: 0.40, - 1280: 0.45, - 1518: 0.50, - } - for size in self.frame_sizes: - speed = self.wirespeed(self.nic, size, self.number_of_ports) - check_dict[size] = round(speed * linerate[size], 2) - return check_dict - - def send_and_verify(self): - """ - Send packet with packet generator and verify + def perf_test(self): """ + Send packet with packet generator + """ + self.result_table_create(self.table_header) + self.throughput = {} for frame_size in self.frame_sizes: + payload_size = frame_size - self.header_size + info = "Running test %s, and %d frame size." % ( + self.running_case, + frame_size, + ) + self.logger.info(info) tgen_input = [] rx_port = self.tester.get_local_port(self.dut_ports[0]) tx_port = self.tester.get_local_port(self.dut_ports[0]) - pkt = Packet(pkt_type="UDP", pkt_len=frame_size) - pkt.config_layer("ether", {"dst": "%s" % self.dst_mac}) - pkt.save_pcapfile(self.tester, "%s/vhost.pcap" % self.out_path) + self.tester.scapy_append( + 'wrpcap("%s/vhost.pcap", [Ether(dst="%s")/IP()/TCP()/("X"*%d)])' + % (self.out_path, self.dst_mac, payload_size) + ) tgen_input.append((tx_port, rx_port, "%s/vhost.pcap" % self.out_path)) - + self.tester.scapy_execute() self.tester.pktgen.clear_streams() streams = self.pktgen_helper.prepare_stream_from_tginput( tgen_input, 100, None, self.tester.pktgen ) - _, pps = self.tester.pktgen.measure_throughput(stream_ids=streams) - Mpps = pps / 1000000.0 - self.verify( - Mpps > self.check_value[frame_size], - "%s of frame size %d speed verify failed, expect %s, result %s" - % (self.running_case, frame_size, self.check_value[frame_size], Mpps), + # set traffic option + traffic_opt = { + "delay": 5, + "duration": self.get_suite_cfg()["test_duration"], + } + _, pps = self.tester.pktgen.measure_throughput( + stream_ids=streams, options=traffic_opt ) - throughput = Mpps * 100 / float(self.wirespeed(self.nic, 64, 1)) - + Mpps = pps / 1000000.0 + line_rate = Mpps * 100 / float(self.wirespeed(self.nic, 64, 1)) + self.throughput[frame_size] = Mpps results_row = [frame_size] - results_row.append("4K pages") results_row.append(Mpps) - results_row.append("1") - results_row.append(throughput) + results_row.append(line_rate) self.result_table_add(results_row) + self.result_table_print() + + def handle_expected(self): + """ + Update expected numbers to configurate file: $DTS_CFG_FOLDER/$suite_name.cfg + """ + if load_global_setting(UPDATE_EXPECTED) == "yes": + for frame_size in self.frame_sizes: + self.expected_throughput[frame_size] = round( + self.throughput[frame_size], 3 + ) + + def handle_results(self): + """ + results handled process: + 1, save to self.test_results + 2, create test results table + """ + # save test results to self.test_result + header = self.table_header + header.append("Expected Throughput(Mpps)") + header.append("Status") + self.result_table_create(self.table_header) + for frame_size in self.frame_sizes: + wirespeed = self.wirespeed(self.nic, frame_size, self.nb_ports) + ret_data = {} + ret_data[header[0]] = str(frame_size) + _real = float(self.throughput[frame_size]) + _exp = float(self.expected_throughput[frame_size]) + ret_data[header[1]] = "{:.3f}".format(_real) + ret_data[header[2]] = "{:.3f}%".format(_real * 100 / wirespeed) + ret_data[header[3]] = "{:.3f}".format(_exp) + gap = _exp * -self.gap * 0.01 + if _real > _exp + gap: + ret_data[header[4]] = "PASS" + else: + ret_data[header[4]] = "FAIL" + self.test_result[frame_size] = deepcopy(ret_data) + + for frame_size in self.test_result.keys(): + table_row = list() + for i in range(len(header)): + table_row.append(self.test_result[frame_size][header[i]]) + self.result_table_add(table_row) + # present test results to screen + self.result_table_print() + self.verify( + "FAIL" not in self.test_result, + "Excessive gap between test results and expectations", + ) def start_testpmd_as_vhost(self): """ @@ -183,26 +221,36 @@ class TestPvpVirtioUser4kPages(TestCase): self.dut.close_session(self.vhost_user) self.dut.close_session(self.virtio_user) - def test_perf_pvp_virtio_user_split_ring_with_4K_pages(self): + def test_perf_pvp_virtio_user_split_ring_with_4K_hugepages(self): """ Basic test for virtio-user 4K pages """ + self.test_target = self.running_case + self.expected_throughput = self.get_suite_cfg()["expected_throughput"][ + self.test_target + ] self.start_testpmd_as_vhost() self.prepare_tmpfs_for_4k() self.start_testpmd_as_virtio() - self.send_and_verify() - self.result_table_print() + self.perf_test() + self.handle_expected() + self.handle_results() self.close_all_apps() - def test_perf_pvp_virtio_user_packed_ring_with_4K_pages(self): + def test_perf_pvp_virtio_user_packed_ring_with_4K_hugepages(self): """ Basic test for virtio-user 4K pages """ + self.test_target = self.running_case + self.expected_throughput = self.get_suite_cfg()["expected_throughput"][ + self.test_target + ] self.start_testpmd_as_vhost() self.prepare_tmpfs_for_4k() self.start_testpmd_as_virtio(packed=True) - self.send_and_verify() - self.result_table_print() + self.perf_test() + self.handle_expected() + self.handle_results() self.close_all_apps() def tear_down(self):