[v2,12/13] app/test: replace .sh scripts with .py scripts
Checks
Commit Message
- Add python scripts to get coremask and check hugepage config
- Remove two corresponding .sh scripts
- Replace calling of .sh with corresponding .py in meson.build
Signed-off-by: Jie Zhou <jizh@linux.microsoft.com>
---
app/test/get-coremask.sh | 13 -------------
app/test/get_coremask.py | 8 ++++++++
app/test/has-hugepage.sh | 11 -----------
app/test/has_hugepage.py | 41 ++++++++++++++++++++++++++++++++++++++++
app/test/meson.build | 7 ++++---
5 files changed, 53 insertions(+), 27 deletions(-)
delete mode 100755 app/test/get-coremask.sh
create mode 100644 app/test/get_coremask.py
delete mode 100755 app/test/has-hugepage.sh
create mode 100644 app/test/has_hugepage.py
Comments
2021-08-18 10:13 (UTC-0700), Jie Zhou:
> - Add python scripts to get coremask and check hugepage config
> - Remove two corresponding .sh scripts
> - Replace calling of .sh with corresponding .py in meson.build
There are multiple violations of PEP8 in new files,
please run a formatter on them, e.g. "black" or "yapf".
I also suggest running "pylint" on those files.
> Signed-off-by: Jie Zhou <jizh@linux.microsoft.com>
> ---
> app/test/get-coremask.sh | 13 -------------
> app/test/get_coremask.py | 8 ++++++++
> app/test/has-hugepage.sh | 11 -----------
> app/test/has_hugepage.py | 41 ++++++++++++++++++++++++++++++++++++++++
> app/test/meson.build | 7 ++++---
> 5 files changed, 53 insertions(+), 27 deletions(-)
> delete mode 100755 app/test/get-coremask.sh
> create mode 100644 app/test/get_coremask.py
> delete mode 100755 app/test/has-hugepage.sh
> create mode 100644 app/test/has_hugepage.py
>
> diff --git a/app/test/get-coremask.sh b/app/test/get-coremask.sh
> deleted file mode 100755
> index bb8cf404d2..0000000000
> --- a/app/test/get-coremask.sh
> +++ /dev/null
> @@ -1,13 +0,0 @@
> -#! /bin/sh -e
> -# SPDX-License-Identifier: BSD-3-Clause
> -# Copyright(c) 2019 Intel Corporation
> -
> -if [ "$(uname)" = "Linux" ] ; then
> - cat /sys/devices/system/cpu/present
> -elif [ "$(uname)" = "FreeBSD" ] ; then
> - ncpus=$(/sbin/sysctl -n hw.ncpu)
> - echo 0-$(expr $ncpus - 1)
> -else
> -# fallback
> - echo 0-3
> -fi
> diff --git a/app/test/get_coremask.py b/app/test/get_coremask.py
> new file mode 100644
> index 0000000000..3d0947e8ff
> --- /dev/null
> +++ b/app/test/get_coremask.py
> @@ -0,0 +1,8 @@
> +#! /usr/bin/env python3
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright (c) 2021 Microsoft Corporation
> +
> +import multiprocessing
> +
> +c = multiprocessing.cpu_count()
> +print("0-"+str(c-1))
Fallback case is lost, error handling missing.
Or is it intended for this script to fail fast?
> diff --git a/app/test/has-hugepage.sh b/app/test/has-hugepage.sh
> deleted file mode 100755
> index d600fad319..0000000000
> --- a/app/test/has-hugepage.sh
> +++ /dev/null
> @@ -1,11 +0,0 @@
> -#! /bin/sh
> -# SPDX-License-Identifier: BSD-3-Clause
> -# Copyright 2020 Mellanox Technologies, Ltd
> -
> -if [ "$(uname)" = "Linux" ] ; then
> - cat /proc/sys/vm/nr_hugepages || echo 0
> -elif [ "$(uname)" = "FreeBSD" ] ; then
> - echo 1 # assume FreeBSD always has hugepages
> -else
> - echo 0
> -fi
> diff --git a/app/test/has_hugepage.py b/app/test/has_hugepage.py
> new file mode 100644
> index 0000000000..decf47df45
> --- /dev/null
> +++ b/app/test/has_hugepage.py
> @@ -0,0 +1,41 @@
> +#! /usr/bin/env python3
> +# SPDX-License-Identifier: BSD-3-Clause
> +# Copyright (c) 2021 Microsoft Corporation
> +
> +import os
> +import platform
> +
> +osName = platform.system()
> +if osName == "Linux":
> + file_o=open("/proc/sys/vm/nr_hugepages")
> + content=file_o.read()
> + print(content)
> + file_o.close()
> +elif osName == "FreeBSD":
> + # Assume FreeBSD always has hugepages enabled
> + print("1")
> +elif osName == "Windows":
> + # On Windows, check if the Administrator has "Lock pages in memory" security setting
> + # to determine if large page is enabled or not
This script is intended to check not whether there are sufficient privileges
but whether the system supports hugepages at all.
I suggest checking ctypes.windll.kernel32.GetLargePageMinimum() > 0.
> +
> + # Export the USER_RIGHTS security settings
> + # Use os.popen instead of os.system to suppress the output of secedit to stdout
> + userRightsfile = "userrights.inf"
> + os.popen('secedit /export /areas USER_RIGHTS /cfg "' + userRightsfile + '"')
> +
> + # Parse the exported user rights setting to determine if Administrator
> + # SeLockMemoryPrivilege being set or not
> + largepageEnabledStr = 'SeLockMemoryPrivilege = Administrator'
> + enabled = 0
> + # On different OS versions tested, the exported inf file has utf-16 encoding
> + with open(userRightsfile, encoding = 'utf-16') as f:
> + urcontent = f.readlines()
> + for line in urcontent:
> + if largepageEnabledStr in line:
> + enabled = 1
> + break
> +
> + f.close()
> + print(enabled)
> +else:
> + print("0")
> diff --git a/app/test/meson.build b/app/test/meson.build
> index a7611686ad..7240304916 100644
> --- a/app/test/meson.build
> +++ b/app/test/meson.build
> @@ -450,7 +450,7 @@ dpdk_test = executable('dpdk-test',
> driver_install_path),
> install: true)
>
> -has_hugepage = run_command('has-hugepage.sh').stdout().strip() != '0'
> +has_hugepage = run_command('has_hugepage.py').stdout().strip() != '0'
Python scripts should be called using `py3` variable,
see buildtools/ for examples.
> message('hugepage availability: @0@'.format(has_hugepage))
>
> # some perf tests (eg: memcpy perf autotest)take very long
> @@ -458,8 +458,9 @@ message('hugepage availability: @0@'.format(has_hugepage))
> timeout_seconds = 600
> timeout_seconds_fast = 10
>
> -get_coremask = find_program('get-coremask.sh')
> -num_cores_arg = '-l ' + run_command(get_coremask).stdout().strip()
> +list_of_cores = run_command('get_coremask.py').stdout().strip()
> +message('list of cores: @0@'.format(list_of_cores))
> +num_cores_arg = '-l ' + list_of_cores
>
> default_test_args = [num_cores_arg]
>
On Sun, Aug 29, 2021 at 01:17:38AM +0300, Dmitry Kozlyuk wrote:
> 2021-08-18 10:13 (UTC-0700), Jie Zhou:
> > - Add python scripts to get coremask and check hugepage config
> > - Remove two corresponding .sh scripts
> > - Replace calling of .sh with corresponding .py in meson.build
>
> There are multiple violations of PEP8 in new files,
> please run a formatter on them, e.g. "black" or "yapf".
> I also suggest running "pylint" on those files.
>
> > Signed-off-by: Jie Zhou <jizh@linux.microsoft.com>
> > ---
> > app/test/get-coremask.sh | 13 -------------
> > app/test/get_coremask.py | 8 ++++++++
> > app/test/has-hugepage.sh | 11 -----------
> > app/test/has_hugepage.py | 41 ++++++++++++++++++++++++++++++++++++++++
> > app/test/meson.build | 7 ++++---
> > 5 files changed, 53 insertions(+), 27 deletions(-)
> > delete mode 100755 app/test/get-coremask.sh
> > create mode 100644 app/test/get_coremask.py
> > delete mode 100755 app/test/has-hugepage.sh
> > create mode 100644 app/test/has_hugepage.py
> >
> > diff --git a/app/test/get-coremask.sh b/app/test/get-coremask.sh
> > deleted file mode 100755
> > index bb8cf404d2..0000000000
> > --- a/app/test/get-coremask.sh
> > +++ /dev/null
> > @@ -1,13 +0,0 @@
> > -#! /bin/sh -e
> > -# SPDX-License-Identifier: BSD-3-Clause
> > -# Copyright(c) 2019 Intel Corporation
> > -
> > -if [ "$(uname)" = "Linux" ] ; then
> > - cat /sys/devices/system/cpu/present
> > -elif [ "$(uname)" = "FreeBSD" ] ; then
> > - ncpus=$(/sbin/sysctl -n hw.ncpu)
> > - echo 0-$(expr $ncpus - 1)
> > -else
> > -# fallback
> > - echo 0-3
> > -fi
> > diff --git a/app/test/get_coremask.py b/app/test/get_coremask.py
> > new file mode 100644
> > index 0000000000..3d0947e8ff
> > --- /dev/null
> > +++ b/app/test/get_coremask.py
> > @@ -0,0 +1,8 @@
> > +#! /usr/bin/env python3
> > +# SPDX-License-Identifier: BSD-3-Clause
> > +# Copyright (c) 2021 Microsoft Corporation
> > +
> > +import multiprocessing
> > +
> > +c = multiprocessing.cpu_count()
> > +print("0-"+str(c-1))
>
> Fallback case is lost, error handling missing.
> Or is it intended for this script to fail fast?
>
> > diff --git a/app/test/has-hugepage.sh b/app/test/has-hugepage.sh
> > deleted file mode 100755
> > index d600fad319..0000000000
> > --- a/app/test/has-hugepage.sh
> > +++ /dev/null
> > @@ -1,11 +0,0 @@
> > -#! /bin/sh
> > -# SPDX-License-Identifier: BSD-3-Clause
> > -# Copyright 2020 Mellanox Technologies, Ltd
> > -
> > -if [ "$(uname)" = "Linux" ] ; then
> > - cat /proc/sys/vm/nr_hugepages || echo 0
> > -elif [ "$(uname)" = "FreeBSD" ] ; then
> > - echo 1 # assume FreeBSD always has hugepages
> > -else
> > - echo 0
> > -fi
> > diff --git a/app/test/has_hugepage.py b/app/test/has_hugepage.py
> > new file mode 100644
> > index 0000000000..decf47df45
> > --- /dev/null
> > +++ b/app/test/has_hugepage.py
> > @@ -0,0 +1,41 @@
> > +#! /usr/bin/env python3
> > +# SPDX-License-Identifier: BSD-3-Clause
> > +# Copyright (c) 2021 Microsoft Corporation
> > +
> > +import os
> > +import platform
> > +
> > +osName = platform.system()
> > +if osName == "Linux":
> > + file_o=open("/proc/sys/vm/nr_hugepages")
> > + content=file_o.read()
> > + print(content)
> > + file_o.close()
> > +elif osName == "FreeBSD":
> > + # Assume FreeBSD always has hugepages enabled
> > + print("1")
> > +elif osName == "Windows":
> > + # On Windows, check if the Administrator has "Lock pages in memory" security setting
> > + # to determine if large page is enabled or not
>
> This script is intended to check not whether there are sufficient privileges
> but whether the system supports hugepages at all.
> I suggest checking ctypes.windll.kernel32.GetLargePageMinimum() > 0.
>
> > +
> > + # Export the USER_RIGHTS security settings
> > + # Use os.popen instead of os.system to suppress the output of secedit to stdout
> > + userRightsfile = "userrights.inf"
> > + os.popen('secedit /export /areas USER_RIGHTS /cfg "' + userRightsfile + '"')
> > +
> > + # Parse the exported user rights setting to determine if Administrator
> > + # SeLockMemoryPrivilege being set or not
> > + largepageEnabledStr = 'SeLockMemoryPrivilege = Administrator'
> > + enabled = 0
> > + # On different OS versions tested, the exported inf file has utf-16 encoding
> > + with open(userRightsfile, encoding = 'utf-16') as f:
> > + urcontent = f.readlines()
> > + for line in urcontent:
> > + if largepageEnabledStr in line:
> > + enabled = 1
> > + break
> > +
> > + f.close()
> > + print(enabled)
> > +else:
> > + print("0")
> > diff --git a/app/test/meson.build b/app/test/meson.build
> > index a7611686ad..7240304916 100644
> > --- a/app/test/meson.build
> > +++ b/app/test/meson.build
> > @@ -450,7 +450,7 @@ dpdk_test = executable('dpdk-test',
> > driver_install_path),
> > install: true)
> >
> > -has_hugepage = run_command('has-hugepage.sh').stdout().strip() != '0'
> > +has_hugepage = run_command('has_hugepage.py').stdout().strip() != '0'
>
> Python scripts should be called using `py3` variable,
> see buildtools/ for examples.
>
> > message('hugepage availability: @0@'.format(has_hugepage))
> >
> > # some perf tests (eg: memcpy perf autotest)take very long
> > @@ -458,8 +458,9 @@ message('hugepage availability: @0@'.format(has_hugepage))
> > timeout_seconds = 600
> > timeout_seconds_fast = 10
> >
> > -get_coremask = find_program('get-coremask.sh')
> > -num_cores_arg = '-l ' + run_command(get_coremask).stdout().strip()
> > +list_of_cores = run_command('get_coremask.py').stdout().strip()
> > +message('list of cores: @0@'.format(list_of_cores))
> > +num_cores_arg = '-l ' + list_of_cores
> >
> > default_test_args = [num_cores_arg]
> >
Thanks Dmitry. Will address all the comments in V3.
deleted file mode 100755
@@ -1,13 +0,0 @@
-#! /bin/sh -e
-# SPDX-License-Identifier: BSD-3-Clause
-# Copyright(c) 2019 Intel Corporation
-
-if [ "$(uname)" = "Linux" ] ; then
- cat /sys/devices/system/cpu/present
-elif [ "$(uname)" = "FreeBSD" ] ; then
- ncpus=$(/sbin/sysctl -n hw.ncpu)
- echo 0-$(expr $ncpus - 1)
-else
-# fallback
- echo 0-3
-fi
new file mode 100644
@@ -0,0 +1,8 @@
+#! /usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2021 Microsoft Corporation
+
+import multiprocessing
+
+c = multiprocessing.cpu_count()
+print("0-"+str(c-1))
deleted file mode 100755
@@ -1,11 +0,0 @@
-#! /bin/sh
-# SPDX-License-Identifier: BSD-3-Clause
-# Copyright 2020 Mellanox Technologies, Ltd
-
-if [ "$(uname)" = "Linux" ] ; then
- cat /proc/sys/vm/nr_hugepages || echo 0
-elif [ "$(uname)" = "FreeBSD" ] ; then
- echo 1 # assume FreeBSD always has hugepages
-else
- echo 0
-fi
new file mode 100644
@@ -0,0 +1,41 @@
+#! /usr/bin/env python3
+# SPDX-License-Identifier: BSD-3-Clause
+# Copyright (c) 2021 Microsoft Corporation
+
+import os
+import platform
+
+osName = platform.system()
+if osName == "Linux":
+ file_o=open("/proc/sys/vm/nr_hugepages")
+ content=file_o.read()
+ print(content)
+ file_o.close()
+elif osName == "FreeBSD":
+ # Assume FreeBSD always has hugepages enabled
+ print("1")
+elif osName == "Windows":
+ # On Windows, check if the Administrator has "Lock pages in memory" security setting
+ # to determine if large page is enabled or not
+
+ # Export the USER_RIGHTS security settings
+ # Use os.popen instead of os.system to suppress the output of secedit to stdout
+ userRightsfile = "userrights.inf"
+ os.popen('secedit /export /areas USER_RIGHTS /cfg "' + userRightsfile + '"')
+
+ # Parse the exported user rights setting to determine if Administrator
+ # SeLockMemoryPrivilege being set or not
+ largepageEnabledStr = 'SeLockMemoryPrivilege = Administrator'
+ enabled = 0
+ # On different OS versions tested, the exported inf file has utf-16 encoding
+ with open(userRightsfile, encoding = 'utf-16') as f:
+ urcontent = f.readlines()
+ for line in urcontent:
+ if largepageEnabledStr in line:
+ enabled = 1
+ break
+
+ f.close()
+ print(enabled)
+else:
+ print("0")
@@ -450,7 +450,7 @@ dpdk_test = executable('dpdk-test',
driver_install_path),
install: true)
-has_hugepage = run_command('has-hugepage.sh').stdout().strip() != '0'
+has_hugepage = run_command('has_hugepage.py').stdout().strip() != '0'
message('hugepage availability: @0@'.format(has_hugepage))
# some perf tests (eg: memcpy perf autotest)take very long
@@ -458,8 +458,9 @@ message('hugepage availability: @0@'.format(has_hugepage))
timeout_seconds = 600
timeout_seconds_fast = 10
-get_coremask = find_program('get-coremask.sh')
-num_cores_arg = '-l ' + run_command(get_coremask).stdout().strip()
+list_of_cores = run_command('get_coremask.py').stdout().strip()
+message('list of cores: @0@'.format(list_of_cores))
+num_cores_arg = '-l ' + list_of_cores
default_test_args = [num_cores_arg]