@@ -11,8 +11,12 @@
# - test_duration is how many seconds each combination performance will
# be recorded.
#
-# - accepted_tolerance defines the accepted tolerance between test
-# results and expected numbers.
+# - accepted_tolerance defines the accepted tolerance for the
+# fluctuation ratio of performance, it is a float(e.g. 0.05).
+# Assume:
+# F = (Real-Mpps - Expected-Mpps)/Expected-Mpps
+# T = accepted_tolerance
+# If F < -T, the case is PASS, else it is FAIL
#
# - expected_throughput is a dictionary defining expected throughput
# numbers based on NIC, and the pattern is
@@ -38,7 +42,7 @@ test_parameters = {'1C/1T': {64: [512, 2048]},
rx_desc_16byte = 'y'
test_duration = 60
throughput_stat_sample_interval = 2
-accepted_tolerance = 1
+accepted_tolerance = 0.05
expected_throughput = {
'fortville_spirit': {
'1C/1T': {64: {512: 0.00, 2048: 0.00}},
@@ -116,12 +116,12 @@ class TestNicSingleCorePerf(TestCase):
# {'$framesize':{"$nb_desc": 'throughput'}
self.throughput = {}
- # Accepted tolerance in Mpps
- self.gap = self.get_suite_cfg()['accepted_tolerance']
+ # Accepted tolerance is ratio
+ self.gap = self.get_suite_cfg().get('accepted_tolerance', 0.1)
# header to print test result table
self.table_header = ['Fwd_core', 'Frame Size', 'TXD/RXD', 'Real-Mpps', 'Rate',
- 'Expected-Mpps', 'Real - Expected', 'Status']
+ 'Expected-Mpps', 'Fluc Ratio', 'Status']
self.test_result = {}
def flows(self):
@@ -332,12 +332,16 @@ class TestNicSingleCorePerf(TestCase):
ret_data[header[3]] = "{:.3f}".format(_real)
ret_data[header[4]] = "{:.3f}%".format(_real * 100 / wirespeed)
ret_data[header[5]] = "{:.3f}".format(_exp)
- delta = _real - _exp
- ret_data[header[6]] = "{:.3f}".format(delta)
- if delta > -self.gap:
- ret_data[header[7]] = 'PASS'
+ delta = (_real - _exp)/_exp
+ if _exp != 0:
+ ret_data[header[6]] = "{:.3f}".format(delta)
+ if delta > -self.gap:
+ ret_data[header[7]] = 'PASS'
+ else:
+ ret_data[header[7]] = 'FAIL'
else:
- ret_data[header[7]] = 'FAIL'
+ ret_data[header[6]] = "N/A"
+ ret_data[header[7]] = 'PASS'
ret_datas[frame_size][nb_desc] = deepcopy(ret_data)
self.test_result[fwd_config] = deepcopy(ret_datas)
@@ -376,15 +380,9 @@ class TestNicSingleCorePerf(TestCase):
row_dict0 = dict()
row_dict0['performance'] = list()
row_dict0['parameters'] = list()
- result_throughput = float(row_in['Real-Mpps'])
- expected_throughput = float(row_in['Expected-Mpps'])
- # delta value and accepted tolerance in percentage
- delta = result_throughput - expected_throughput
- if delta > -self.gap:
- row_dict0['status'] = 'PASS'
- else:
- row_dict0['status'] = 'FAIL'
- row_dict1 = dict(name="Throughput", value=result_throughput, unit="Mpps", delta=delta)
+ row_dict0['status'] = row_in['Status']
+ row_dict1 = dict(name="Throughput", value=row_in['Real-Mpps'], unit="Mpps",
+ delta=row_in['Fluc Ratio'])
row_dict2 = dict(name="Txd/Rxd", value=row_in["TXD/RXD"], unit="descriptor")
row_dict3 = dict(name="frame_size", value=row_in["Frame Size"], unit="bytes")
row_dict4 = dict(name="Fwd_core", value=row_in["Fwd_core"])
the delta used to defines the difference between real pps and expected pps, the case fails if it exceed the absolute number specified by 'accepted_tolerance' in conf/nic_single_core_perf.cfg, the deault is 1Mpps, which is not friendly when the expected is only a few of million, so updated the accepted_tolerance to a percentage, considering the CI intergation, so the type is float. Here are main changes: * the meaning of accepted_tolerance before: threshold for (real-mpps - expected-mpps) after: threshold for (real-mpps - expected-mpps)/expected-mpps * Pass criteria before: real-mpps - expected-mpps > -accepted_tolerance after: (real-mpps - expected-mpps)/expected-mpps > -accepted_tolerance * rst table before: ... +---------------+-----------------+--------+ ... | Expected-Mpps | Real - Expected | Status | ... +===============+=================+========+ ... | 9.900 | -0.040 | PASS | ... +---------------+-----------------+--------+ after: ... +---------------+------------+--------+ ... | Expected-Mpps | Fluc Ratio | Status | ... +===============+============+========+ ... | 9.900 | -0.004 | PASS | ... +---------------+------------+--------+ * delta in JSON: before: real-mpps - expected-mpps after: (real-mpps - expected-mpps)/expected-mpps Signed-off-by: Lijuan Tu <lijuan.tu@intel.com> --- conf/nic_single_core_perf.cfg | 10 +++++--- tests/TestSuite_nic_single_core_perf.py | 32 ++++++++++++------------- 2 files changed, 22 insertions(+), 20 deletions(-)