From patchwork Mon Apr 24 07:33:39 2023 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Ling, WeiX" X-Patchwork-Id: 126441 Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id A8F78429D2; Mon, 24 Apr 2023 09:35:31 +0200 (CEST) Received: from mails.dpdk.org (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id A384342BD9; Mon, 24 Apr 2023 09:35:31 +0200 (CEST) Received: from mga14.intel.com (mga14.intel.com [192.55.52.115]) by mails.dpdk.org (Postfix) with ESMTP id 77622410DE for ; Mon, 24 Apr 2023 09:35:29 +0200 (CEST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=intel.com; i=@intel.com; q=dns/txt; s=Intel; t=1682321729; x=1713857729; h=from:to:cc:subject:date:message-id:in-reply-to: references:mime-version:content-transfer-encoding; bh=6O7dzc5HeYUrrRM2T3Y6OLwBL1M9ipOWVKLZBWd1NnM=; b=G6+e/CVWwy8blwakshqXdX0BZX1R0YSze/c0wMIkF5sEzd/woI987uHP Z4kYOh1gtcM8eMGbKeBtIpONATYqwDGcYPYHYj1YcYpdBWvnbFHpZql+j AQD+UJiaOJ4wVLG7YyoZVgr6nakOnZnVDawNMqGr4tJcEaNL2PHul+rRk H2i2CFayF28HvWEQB9z1ZTsJheEGXZ14PzdjIZdn+gcOEhn2f+YjLJPjW XqcoMzH8TXNvdKBCkiAggUOiF82/ssstfMPBpZ6hI0Qw+wLCUtxpc4Mpc ncGnJzfV9F5lx4lcbkLirN+zNJY8AJ/fTlDZPfMEQGh+1kpc9N5XljmVo w==; X-IronPort-AV: E=McAfee;i="6600,9927,10689"; a="346409588" X-IronPort-AV: E=Sophos;i="5.99,222,1677571200"; d="scan'208";a="346409588" Received: from fmsmga005.fm.intel.com ([10.253.24.32]) by fmsmga103.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Apr 2023 00:35:29 -0700 X-ExtLoop1: 1 X-IronPort-AV: E=McAfee;i="6600,9927,10689"; a="1022604381" X-IronPort-AV: E=Sophos;i="5.99,222,1677571200"; d="scan'208";a="1022604381" Received: from unknown (HELO localhost.localdomain) ([10.239.252.222]) by fmsmga005-auth.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384; 24 Apr 2023 00:35:27 -0700 From: Wei Ling To: dts@dpdk.org Cc: Wei Ling Subject: [dts][PATCH V2 2/3] tests/pvp_diff_qemu_version: add case3 to test virtio1.1 and optimize verify expected Date: Mon, 24 Apr 2023 15:33:39 +0800 Message-Id: <20230424073340.2368259-3-weix.ling@intel.com> X-Mailer: git-send-email 2.25.1 In-Reply-To: <20230424073340.2368259-1-weix.ling@intel.com> References: <20230424073340.2368259-1-weix.ling@intel.com> MIME-Version: 1.0 X-BeenThere: dts@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: test suite reviews and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dts-bounces@dpdk.org 1.Add case3 to test virtio 1.1 path. 2.Modify the casename with virtio protocol version. 3.Optimize the verify the expected value from the hard code to read from conf/pvp_diff_qemu_version.cfg to adapt different platform. Signed-off-by: Wei Ling --- tests/TestSuite_pvp_diff_qemu_version.py | 236 +++++++++++++---------- 1 file changed, 136 insertions(+), 100 deletions(-) diff --git a/tests/TestSuite_pvp_diff_qemu_version.py b/tests/TestSuite_pvp_diff_qemu_version.py index 02682815..6a7830f4 100644 --- a/tests/TestSuite_pvp_diff_qemu_version.py +++ b/tests/TestSuite_pvp_diff_qemu_version.py @@ -2,23 +2,11 @@ # Copyright(c) 2019 Intel Corporation # -""" -DPDK Test suite. - -Vhost PVP performance using differnet Qemu test suite. -Can config the qemu version in config file like: -qemu = - path=qemu-2.5/bin/qemu-system-x86_64; - path=qemu-2.6/bin/qemu-system-x86_64; -""" import re import time -from scapy.utils import wrpcap - -import framework.utils as utils from framework.pktgen import PacketGeneratorHelper -from framework.settings import HEADER_SIZE +from framework.settings import HEADER_SIZE, UPDATE_EXPECTED, load_global_setting from framework.test_case import TestCase from framework.virt_common import VM @@ -34,7 +22,6 @@ class TestVhostPVPDiffQemuVersion(TestCase): self.cores_num = len( [n for n in self.dut.cores if int(n["socket"]) == self.socket] ) - self.verify(len(self.dut_ports) >= 1, "Insufficient ports for testing") self.verify( self.cores_num >= 3, "There has not enought cores to test this suite" @@ -42,7 +29,6 @@ class TestVhostPVPDiffQemuVersion(TestCase): self.cores = self.dut.get_core_list("1S/3C/1T", socket=self.socket) self.vm_dut = None self.packet_params_set() - self.logger.info( "You can config all the path of qemu version you want to" + " tested in the conf file %s.cfg" % self.suite_name @@ -53,7 +39,6 @@ class TestVhostPVPDiffQemuVersion(TestCase): ) res = self.verify_qemu_version_config() self.verify(res is True, "The path of qemu version in config file not right") - self.out_path = "/tmp" out = self.tester.send_expect("ls -d %s" % self.out_path, "# ") if "No such file or directory" in out: @@ -62,36 +47,36 @@ class TestVhostPVPDiffQemuVersion(TestCase): self.pktgen_helper = PacketGeneratorHelper() self.base_dir = self.dut.base_dir.replace("~", "/root") self.pci_info = self.dut.ports_info[0]["pci"] - self.number_of_ports = 1 + self.nb_ports = 1 self.path = self.dut.apps_name["test-pmd"] self.testpmd_name = self.path.split("/")[-1] + self.gap = self.get_suite_cfg()["accepted_tolerance"] + self.vhost_user = self.dut.new_session(suite="vhost-user") def set_up(self): """ Run before each test case. """ - self.vhost = self.dut.new_session(suite="vhost-user") self.dut.send_expect("rm -rf %s/vhost-net*" % self.base_dir, "#") self.dut.send_expect("killall -s INT %s" % self.testpmd_name, "#") self.dut.send_expect("killall -I qemu-system-x86_64", "#", 20) + self.throughput = dict() + self.test_result = [] + self.table_header = [ + "QemuVersion", + "FrameSize(B)", + "Throughput(Mpps)", + "LineRate(%)", + ] def packet_params_set(self): - self.frame_sizes = [64, 128, 256, 512, 1024, 1500] + self.frame_sizes = [64, 128, 256, 512, 1024, 1280, 1518] # get the frame_sizes from cfg file if "packet_sizes" in self.get_suite_cfg(): self.frame_sizes = self.get_suite_cfg()["packet_sizes"] - self.virtio1_mac = "52:54:00:00:00:01" self.src1 = "192.168.4.1" self.dst1 = "192.168.3.1" - self.header_row = [ - "QemuVersion", - "FrameSize(B)", - "Throughput(Mpps)", - "LineRate(%)", - "Cycle", - "Expected Throughput(Mpps)", - ] def get_qemu_list_from_config(self): """ @@ -122,7 +107,6 @@ class TestVhostPVPDiffQemuVersion(TestCase): qemu_num = len(self.qemu_list) for i in range(qemu_num): qemu_path = self.qemu_list[i]["path"] - out = self.dut.send_expect("ls %s" % qemu_path, "#") if "No such file or directory" in out: self.logger.error( @@ -137,13 +121,14 @@ class TestVhostPVPDiffQemuVersion(TestCase): % (qemu_path, self.dut.get_ip_address()) ) return False - out = self.dut.send_expect("%s --version" % qemu_path, "#") result = re.search("QEMU\s*emulator\s*version\s*(\d*.\d*)", out) version = result.group(1) # update the version info to self.qemu_list self.qemu_list[i].update({"version": "qemu-%s" % version}) - + self.qemu_versions = list() + for i in self.qemu_list: + self.qemu_versions.append(i["version"]) # print all the qemu version you config config_qemu_version = "" for i in range(len(self.qemu_list)): @@ -151,7 +136,6 @@ class TestVhostPVPDiffQemuVersion(TestCase): self.logger.info( "The suite will test the qemu version of: %s" % config_qemu_version ) - return True def rm_vm_qemu_path_config(self): @@ -167,7 +151,7 @@ class TestVhostPVPDiffQemuVersion(TestCase): self.verify(qemu_index < params_num, "Please config qemu path in conf gile") self.vm.params.pop(qemu_index) - def start_vm(self, path, modem): + def start_one_vm(self, qemu_path, qemu_mode): """ start vm """ @@ -176,15 +160,17 @@ class TestVhostPVPDiffQemuVersion(TestCase): vm_params["driver"] = "vhost-user" vm_params["opt_path"] = "%s/vhost-net" % self.base_dir vm_params["opt_mac"] = self.virtio1_mac - if modem == 1: - vm_params["opt_settings"] = "disable-modern=false,mrg_rxbuf=on" - elif modem == 0: + if qemu_mode == 0: vm_params["opt_settings"] = "disable-modern=true,mrg_rxbuf=on" + elif qemu_mode == 1: + vm_params["opt_settings"] = "disable-modern=false,mrg_rxbuf=on" + elif qemu_mode == 2: + vm_params["opt_settings"] = "disable-modern=false,mrg_rxbuf=on,packed=on" self.vm.set_vm_device(**vm_params) self.vm.load_config() self.rm_vm_qemu_path_config() # set qemu version info - self.vm.set_qemu_emulator(path) + self.vm.set_qemu_emulator(qemu_path) # Due to we have change the params info before, # so need to start vm with load_config=False try: @@ -204,9 +190,9 @@ class TestVhostPVPDiffQemuVersion(TestCase): ) para = " -- -i --nb-cores=1 --txd=1024 --rxd=1024" command_line_client = self.path + eal_params + para - self.vhost.send_expect(command_line_client, "testpmd> ", 30) - self.vhost.send_expect("set fwd mac", "testpmd> ", 30) - self.vhost.send_expect("start", "testpmd> ", 30) + self.vhost_user.send_expect(command_line_client, "testpmd> ", 30) + self.vhost_user.send_expect("set fwd mac", "testpmd> ", 30) + self.vhost_user.send_expect("start", "testpmd> ", 30) def vm_testpmd_start(self): """ @@ -220,26 +206,9 @@ class TestVhostPVPDiffQemuVersion(TestCase): self.vm_dut.send_expect("set fwd mac", "testpmd> ", 20) self.vm_dut.send_expect("start", "testpmd> ") - @property - def check_value(self): - check_dict = dict.fromkeys(self.frame_sizes) - linerate = { - 64: 0.07, - 128: 0.10, - 256: 0.17, - 512: 0.18, - 1024: 0.35, - 1280: 0.40, - 1500: 0.45, - } - for size in self.frame_sizes: - speed = self.wirespeed(self.nic, size, self.number_of_ports) - check_dict[size] = round(speed * linerate[size], 2) - return check_dict - - def send_verify(self, qemu_version, vlan_id1=0, tag="Performance"): - self.result_table_create(self.header_row) - perf_result = {} + def perf_test(self, qemu_version, vlan_id1=0): + self.result_table_create(self.table_header) + self.throughput[qemu_version] = {} for frame_size in self.frame_sizes: info = "Running test %s, and %d frame size." % ( self.running_case, @@ -264,42 +233,30 @@ class TestVhostPVPDiffQemuVersion(TestCase): tgenInput.append( (port, port, "%s/pvp_diff_qemu_version.pcap" % self.out_path) ) - self.tester.pktgen.clear_streams() streams = self.pktgen_helper.prepare_stream_from_tginput( tgenInput, 100, None, self.tester.pktgen ) # set traffic option - traffic_opt = {"delay": 5, "duration": 20} + traffic_opt = { + "delay": 5, + "duration": self.get_suite_cfg()["test_duration"], + } _, pps = self.tester.pktgen.measure_throughput( stream_ids=streams, options=traffic_opt ) Mpps = pps / 1000000.0 line_rate = Mpps * 100 / float(self.wirespeed(self.nic, frame_size, 1)) + self.throughput[qemu_version][frame_size] = Mpps # update print table info data_row = [ qemu_version, frame_size, str(Mpps), str(line_rate), - tag, - self.check_value[frame_size], ] self.result_table_add(data_row) - perf_result[frame_size] = Mpps - self.result_table_print() - for frame_size in perf_result.keys(): - self.verify( - perf_result[frame_size] > self.check_value[frame_size], - "%s of frame size %d speed verify failed, expect %s, result %s" - % ( - self.running_case, - frame_size, - self.check_value[frame_size], - perf_result[frame_size], - ), - ) def close_testpmd_and_qemu(self): """ @@ -307,57 +264,136 @@ class TestVhostPVPDiffQemuVersion(TestCase): close the qemu """ self.vm_dut.send_expect("quit", "#", 20) - self.vhost.send_expect("quit", "#", 20) + self.vhost_user.send_expect("quit", "#", 20) self.vm.stop() self.dut.send_expect("killall -I %s" % self.testpmd_name, "#", 20) self.dut.send_expect("rm -rf %s/vhost-net*" % self.base_dir, "#") - def test_perf_vhost_pvp_diffrent_qemu_version_mergeable_mac(self): + def test_perf_vhost_pvp_diff_qemu_version_virtio95_mergeable_path(self): + """ + Test Case 1: PVP multi qemu version test with virtio 0.95 mergeable path + """ + self.test_target = self.running_case + self.expected_throughput = self.get_suite_cfg()["expected_throughput"][ + self.test_target + ] + for i in range(len(self.qemu_list)): + qemu_path = self.qemu_list[i]["path"] + qemu_version = self.qemu_list[i]["version"] + self.start_vhost_testpmd() + self.logger.info("now testing the qemu path of %s" % qemu_path) + self.start_one_vm(qemu_path=qemu_path, qemu_mode=0) + self.vm_testpmd_start() + vlan_id1 = 1000 + self.perf_test(qemu_version, vlan_id1) + self.close_testpmd_and_qemu() + self.handle_expected() + self.handle_results() + + def test_perf_vhost_pvp_diff_qemu_version_virtio10_mergeable_path(self): """ - Test the performance of one vm with virtio 0.95 on mergeable path + Test Case 2: PVP test with virtio 1.0 mergeable path """ + self.test_target = self.running_case + self.expected_throughput = self.get_suite_cfg()["expected_throughput"][ + self.test_target + ] for i in range(len(self.qemu_list)): - path = self.qemu_list[i]["path"] - version = self.qemu_list[i]["version"] + qemu_path = self.qemu_list[i]["path"] + qemu_version = self.qemu_list[i]["version"] self.start_vhost_testpmd() - self.start_vm(path, 0) - # Start testpmd in vm + self.logger.info("now testing the qemu path of %s" % qemu_path) + self.start_one_vm(qemu_path=qemu_path, qemu_mode=1) self.vm_testpmd_start() - self.logger.info("now testing the qemu path of %s" % path) - time.sleep(5) vlan_id1 = 1000 - self.send_verify(version, vlan_id1, "virtio-0.95, Mergeable") + self.perf_test(qemu_version, vlan_id1) self.close_testpmd_and_qemu() + self.handle_expected() + self.handle_results() - def test_perf_vhost_pvp_diffrent_qemu_version_modern_mergeable_mac(self): + def test_perf_vhost_pvp_diff_qemu_version_virtio11_mergeable_path(self): """ - Test the performance of one vm with virtio 1.0 on mergeable path + Test Case 3: PVP test with virtio 1.1 mergeable path """ + self.test_target = self.running_case + self.expected_throughput = self.get_suite_cfg()["expected_throughput"][ + self.test_target + ] for i in range(len(self.qemu_list)): - path = self.qemu_list[i]["path"] - version = self.qemu_list[i]["version"] + qemu_path = self.qemu_list[i]["path"] + qemu_version = self.qemu_list[i]["version"] self.start_vhost_testpmd() - self.start_vm(path, 1) - # Start testpmd in vm + self.logger.info("now testing the qemu path of %s" % qemu_path) + self.start_one_vm(qemu_path=qemu_path, qemu_mode=2) self.vm_testpmd_start() - self.logger.info("now testing the qemu path of %s" % path) - time.sleep(5) vlan_id1 = 1000 - self.send_verify(version, vlan_id1, "virtio-1.0, Mergeable") + self.perf_test(qemu_version, vlan_id1) self.close_testpmd_and_qemu() + self.handle_expected() + self.handle_results() + + def handle_expected(self): + """ + Update expected numbers to configurate file: $DTS_CFG_FOLDER/$suite_name.cfg + """ + if load_global_setting(UPDATE_EXPECTED) == "yes": + for qemu_version in self.qemu_versions: + for frame_size in self.frame_sizes: + self.expected_throughput[qemu_version][frame_size] = round( + self.throughput[qemu_version][frame_size], 3 + ) + + def handle_results(self): + """ + results handled process: + 1, save to self.test_results + 2, create test results table + """ + # save test results to self.test_result + header = self.table_header + header.append("Expected Throughput(Mpps)") + header.append("Status") + self.result_table_create(self.table_header) + for qv in self.qemu_list: + qemu_version = qv["version"] + for frame_size in self.frame_sizes: + wirespeed = self.wirespeed(self.nic, frame_size, self.nb_ports) + ret_data = {} + ret_data[header[0]] = qemu_version + ret_data[header[1]] = str(frame_size) + _real = float(self.throughput[qemu_version][frame_size]) + _exp = float(self.expected_throughput[qemu_version][frame_size]) + ret_data[header[2]] = "{:.3f}".format(_real) + ret_data[header[3]] = "{:.3f}%".format(_real * 100 / wirespeed) + ret_data[header[4]] = "{:.3f}".format(_exp) + gap = _exp * -self.gap * 0.01 + if _real > _exp + gap: + ret_data[header[5]] = "PASS" + else: + ret_data[header[5]] = "FAIL" + self.test_result.append(ret_data) + + for test_result in self.test_result: + table_row = list() + for i in range(len(header)): + table_row.append(test_result[header[i]]) + self.result_table_add(table_row) + # present test results to screen + self.result_table_print() + self.verify( + "FAIL" not in self.test_result, + "Excessive gap between test results and expectations", + ) def tear_down(self): """ Run after each test case. - Clear qemu and testpmd to avoid blocking the following TCs """ - self.dut.close_session(self.vhost) self.dut.send_expect("killall -s INT qemu-system-x86_64", "#") self.dut.send_expect("killall -s INT %s" % self.testpmd_name, "#") - time.sleep(2) def tear_down_all(self): """ Run after each test suite. """ - pass + self.dut.close_session(self.vhost_user)