[v3] Framework:fix the first numa node may not be '0'
Checks
| Context |
Check |
Description |
| ci/Intel-dts-suite-test |
success
|
Testing OK
|
Commit Message
On certain Virtual Machine system such as Power Virtual Machine, PowerVM,
assigns the CPUs of its virtual machines (or Logical Partitions LPARs) to
NUMA nodes, and so the VM (or LPARs) 's CPUs may be on different ID
other than '0'.
This patch should fix that.
v2: correct spelling and remove trailing whitespace
v3:
* adding a flag to check if default hugepages is already cleared
Signed-off-by: Thinh Tran <thinhtr@linux.vnet.ibm.com>
---
framework/crb.py | 14 ++++++++------
framework/dut.py | 40 +++++++++++++++++++++++-----------------
2 files changed, 31 insertions(+), 23 deletions(-)
Comments
> -----Original Message-----
> From: Thinh Tran <thinhtr@linux.vnet.ibm.com>
> Sent: 2021年10月16日 14:46
> To: dts@dpdk.org
> Cc: Ling, WeiX <weix.ling@intel.com>; Jiang, YuX <yux.jiang@intel.com>; Tu,
> Lijuan <lijuan.tu@intel.com>; Thinh Tran <thinhtr@linux.vnet.ibm.com>
> Subject: [dts] [PATCH v3] Framework:fix the first numa node may not be '0'
>
> On certain Virtual Machine system such as Power Virtual Machine, PowerVM,
> assigns the CPUs of its virtual machines (or Logical Partitions LPARs) to NUMA
> nodes, and so the VM (or LPARs) 's CPUs may be on different ID other than '0'.
> This patch should fix that.
>
> v2: correct spelling and remove trailing whitespace
>
> v3:
> * adding a flag to check if default hugepages is already cleared
>
> Signed-off-by: Thinh Tran <thinhtr@linux.vnet.ibm.com>
Applied, thanks
@@ -61,6 +61,7 @@ class Crb(object):
self.stage = 'pre-init'
self.name = name
self.trex_prefix = None
+ self.default_hugepages_cleared = False
self.logger = getLogger(name)
self.session = SSHConnection(self.get_ip_address(), name,
@@ -204,24 +205,25 @@ class Crb(object):
else:
return ''
- def set_huge_pages(self, huge_pages, numa=-1):
+ def set_huge_pages(self, huge_pages, numa=""):
"""
Set numbers of huge pages
"""
page_size = self.send_expect("awk '/Hugepagesize/ {print $2}' /proc/meminfo", "# ")
- if numa == -1:
+ if not numa:
self.send_expect('echo %d > /sys/kernel/mm/hugepages/hugepages-%skB/nr_hugepages' % (huge_pages, page_size), '# ', 5)
else:
- # sometimes we set hugepage on kernel cmdline, so we clear all nodes' default hugepages at the first time.
- if numa == 0:
+ # sometimes we set hugepage on kernel cmdline, so we clear it
+ if not self.default_hugepages_cleared:
self.send_expect('echo 0 > /sys/kernel/mm/hugepages/hugepages-%skB/nr_hugepages' % (page_size), '# ', 5)
+ self.default_hugepages_cleared = True
# some platform not support numa, example vm dut
try:
- self.send_expect('echo %d > /sys/devices/system/node/node%d/hugepages/hugepages-%skB/nr_hugepages' % (huge_pages, numa, page_size), '# ', 5)
+ self.send_expect('echo %d > /sys/devices/system/node/%s/hugepages/hugepages-%skB/nr_hugepages' % (huge_pages, numa, page_size), '# ', 5)
except:
- self.logger.warning("set %d hugepage on socket %d error" % (huge_pages, numa))
+ self.logger.warning("set %d hugepage on %s error" % (huge_pages, numa))
self.send_expect('echo %d > /sys/kernel/mm/hugepages/hugepages-%skB/nr_hugepages' % (huge_pages. page_size), '# ', 5)
def set_speedup_options(self, read_cache, skip_setup):
@@ -547,16 +547,15 @@ class Dut(Crb):
return
hugepages_size = self.send_expect("awk '/Hugepagesize/ {print $2}' /proc/meminfo", "# ")
total_huge_pages = self.get_total_huge_pages()
- total_numa_nodes = self.send_expect("ls /sys/devices/system/node | grep node* | wc -l", "# ")
- numa_service_num = self.get_def_rte_config('CONFIG_RTE_MAX_NUMA_NODES')
- try:
- int(total_numa_nodes)
- except ValueError:
+ numa_nodes = self.send_expect("ls /sys/devices/system/node | grep node*", "# ")
+ if not numa_nodes:
total_numa_nodes = -1
- if numa_service_num is not None:
- numa = min(int(total_numa_nodes), int(numa_service_num))
else:
- numa = total_numa_nodes
+ numa_nodes = numa_nodes.splitlines()
+ total_numa_nodes = len(numa_nodes)
+ self.logger.info(numa_nodes)
+
+
force_socket = False
if int(hugepages_size) < (1024 * 1024):
@@ -578,16 +577,23 @@ class Dut(Crb):
arch_huge_pages = hugepages if hugepages > 0 else 2048
if total_huge_pages != arch_huge_pages:
- # before all hugepage average distribution by all socket,
- # but sometimes create mbuf pool on socket 0 failed when setup testpmd,
- # so set all huge page on socket 0
- if force_socket:
- self.set_huge_pages(arch_huge_pages, 0)
+ if total_numa_nodes == -1 :
+ self.set_huge_pages(arch_huge_pages)
else:
- for numa_id in range(0, int(numa)):
- self.set_huge_pages(arch_huge_pages, numa_id)
- if numa == -1:
- self.set_huge_pages(arch_huge_pages)
+ # before all hugepage average distribution by all socket,
+ # but sometimes create mbuf pool on socket 0 failed when
+ # setup testpmd, so set all huge page on first socket
+ if force_socket:
+ self.set_huge_pages(arch_huge_pages, numa_nodes[0])
+ self.logger.info("force_socket on %s" % numa_nodes[0])
+ else:
+ numa_service_num = self.get_def_rte_config('CONFIG_RTE_MAX_NUMA_NODES')
+ if numa_service_num is not None:
+ total_numa_nodes = min(total_numa_nodes, int(numa_service_num))
+
+ # set huge pages to configured total_numa_nodes
+ for numa_node in numa_nodes[:total_numa_nodes]:
+ self.set_huge_pages(arch_huge_pages, numa_node)
self.mount_huge_pages()
self.hugepage_path = self.strip_hugepage_path()