Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/41481/?format=api
{ "id": 41481, "url": "http://patchwork.dpdk.org/api/patches/41481/?format=api", "web_url": "http://patchwork.dpdk.org/project/dpdk/patch/02efa4a4e6fab105ae883a855ae1985afc208a31.1529940601.git.anatoly.burakov@intel.com/", "project": { "id": 1, "url": "http://patchwork.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<02efa4a4e6fab105ae883a855ae1985afc208a31.1529940601.git.anatoly.burakov@intel.com>", "list_archive_url": "https://inbox.dpdk.org/dev/02efa4a4e6fab105ae883a855ae1985afc208a31.1529940601.git.anatoly.burakov@intel.com", "date": "2018-06-25T15:59:46", "name": "[RFC,9/9] usertools/lib: add GRUB utility library for hugepage config", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "851f6cd86b3079310d5ceecce11402951951f53c", "submitter": { "id": 4, "url": "http://patchwork.dpdk.org/api/people/4/?format=api", "name": "Anatoly Burakov", "email": "anatoly.burakov@intel.com" }, "delegate": { "id": 1, "url": "http://patchwork.dpdk.org/api/users/1/?format=api", "username": "tmonjalo", "first_name": "Thomas", "last_name": "Monjalon", "email": "thomas@monjalon.net" }, "mbox": "http://patchwork.dpdk.org/project/dpdk/patch/02efa4a4e6fab105ae883a855ae1985afc208a31.1529940601.git.anatoly.burakov@intel.com/mbox/", "series": [ { "id": 225, "url": "http://patchwork.dpdk.org/api/series/225/?format=api", "web_url": "http://patchwork.dpdk.org/project/dpdk/list/?series=225", "date": "2018-06-25T15:59:39", "name": "Modularize and enhance DPDK Python scripts", "version": 1, "mbox": "http://patchwork.dpdk.org/series/225/mbox/" } ], "comments": "http://patchwork.dpdk.org/api/patches/41481/comments/", "check": "fail", "checks": "http://patchwork.dpdk.org/api/patches/41481/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@dpdk.org", "Delivered-To": "patchwork@dpdk.org", "Received": [ "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 589B64F91;\n\tMon, 25 Jun 2018 17:59:58 +0200 (CEST)", "from mga09.intel.com (mga09.intel.com [134.134.136.24])\n\tby dpdk.org (Postfix) with ESMTP id 0ACCA1559\n\tfor <dev@dpdk.org>; Mon, 25 Jun 2018 17:59:51 +0200 (CEST)", "from orsmga002.jf.intel.com ([10.7.209.21])\n\tby orsmga102.jf.intel.com with ESMTP/TLS/DHE-RSA-AES256-GCM-SHA384;\n\t25 Jun 2018 08:59:50 -0700", "from irvmail001.ir.intel.com ([163.33.26.43])\n\tby orsmga002.jf.intel.com with ESMTP; 25 Jun 2018 08:59:48 -0700", "from sivswdev01.ir.intel.com (sivswdev01.ir.intel.com\n\t[10.237.217.45])\n\tby irvmail001.ir.intel.com (8.14.3/8.13.6/MailSET/Hub) with ESMTP id\n\tw5PFxloN032524; Mon, 25 Jun 2018 16:59:47 +0100", "from sivswdev01.ir.intel.com (localhost [127.0.0.1])\n\tby sivswdev01.ir.intel.com with ESMTP id w5PFxlqd026652;\n\tMon, 25 Jun 2018 16:59:47 +0100", "(from aburakov@localhost)\n\tby sivswdev01.ir.intel.com with LOCAL id w5PFxlCe026644;\n\tMon, 25 Jun 2018 16:59:47 +0100" ], "X-Amp-Result": "SKIPPED(no attachment in message)", "X-Amp-File-Uploaded": "False", "X-ExtLoop1": "1", "X-IronPort-AV": "E=Sophos;i=\"5.51,270,1526367600\"; d=\"scan'208\";a=\"69856448\"", "From": "Anatoly Burakov <anatoly.burakov@intel.com>", "To": "dev@dpdk.org", "Cc": "john.mcnamara@intel.com, bruce.richardson@intel.com,\n\tpablo.de.lara.guarch@intel.com, david.hunt@intel.com,\n\tmohammad.abdul.awal@intel.com", "Date": "Mon, 25 Jun 2018 16:59:46 +0100", "Message-Id": "<02efa4a4e6fab105ae883a855ae1985afc208a31.1529940601.git.anatoly.burakov@intel.com>", "X-Mailer": "git-send-email 1.7.0.7", "In-Reply-To": [ "<cover.1529940601.git.anatoly.burakov@intel.com>", "<cover.1529940601.git.anatoly.burakov@intel.com>" ], "References": [ "<cover.1529940601.git.anatoly.burakov@intel.com>", "<cover.1529940601.git.anatoly.burakov@intel.com>" ], "Subject": "[dpdk-dev] [RFC 9/9] usertools/lib: add GRUB utility library for\n\thugepage config", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n\t<mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n\t<mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "This library is highly experimental and can kill kittens, but its\nmain purpose is to automatically set up GRUB command-line to\nallocate a given number of hugepages at boot time. It works in\na similar way HugeUtil library does, but instead of committing\nchanges to fstab or runtime configuration, it commits its\nchanges to GRUB default command-line and updates all GRUB entries\nafterwards. I got it to a state where it's safe to use on my\nsystem, but see the part above about killing kittens - you have\nbeen warned :)\n\nNo example scripts will currently be provided.\n\nSigned-off-by: Anatoly Burakov <anatoly.burakov@intel.com>\n---\n usertools/DPDKConfigLib/GrubHugeUtil.py | 175 ++++++++++++++++++++++++\n 1 file changed, 175 insertions(+)\n create mode 100755 usertools/DPDKConfigLib/GrubHugeUtil.py", "diff": "diff --git a/usertools/DPDKConfigLib/GrubHugeUtil.py b/usertools/DPDKConfigLib/GrubHugeUtil.py\nnew file mode 100755\nindex 000000000..4b8e349b8\n--- /dev/null\n+++ b/usertools/DPDKConfigLib/GrubHugeUtil.py\n@@ -0,0 +1,175 @@\n+#!/usr/bin/env python\n+# SPDX-License-Identifier: BSD-3-Clause\n+# Copyright(c) 2018 Intel Corporation\n+\n+\n+from .PlatformInfo import *\n+from .Util import *\n+import re\n+import os\n+\n+__KERNEL_TRANSPARENT_HP = \"/sys/kernel/mm/transparent_hugepage/enabled\"\n+_GRUB_CMDLINE_PARAM_NAME = \"GRUB_CMDLINE_LINUX_DEFAULT\"\n+\n+# local copy of platform info\n+info = PlatformInfo()\n+\n+def _find_linux_default_cmdline():\n+ with open(\"/etc/default/grub\") as f:\n+ for line in f:\n+ line = line.strip()\n+ if line.startswith(_GRUB_CMDLINE_PARAM_NAME):\n+ return line\n+ else:\n+ raise RuntimeError(\"Invalid GRUB default configuration format\")\n+\n+\n+def _parse_linux_default_cmdline(line):\n+ # get value to the right of equals sign, strip whitespace and quotes,\n+ # split into separate keys and make a list of values\n+ _, cmdline = kv_split(line, \"=\")\n+ # remove quotes\n+ if cmdline[0] == cmdline[-1] == '\"':\n+ cmdline = cmdline[1:-1]\n+\n+ return [kv_split(v, \"=\") for v in cmdline.split()]\n+\n+\n+def _generate_linux_default_cmdline(cmdline):\n+ lines = []\n+ cmdline_idx = -1\n+ with open(\"/etc/default/grub\") as f:\n+ for idx, line in enumerate(f):\n+ line = line.strip()\n+ lines.extend([line])\n+ if line.startswith(_GRUB_CMDLINE_PARAM_NAME):\n+ cmdline_idx = idx\n+ if cmdline_idx == -1:\n+ raise RuntimeError(\"Invalid GRUB default configuration format\")\n+\n+ # write the lines back, replacing one we want\n+ with open(\"/etc/default/grub\", \"w\") as f:\n+ for idx, line in enumerate(lines):\n+ if idx == cmdline_idx:\n+ line = cmdline\n+ f.write(line + \"\\n\")\n+\n+\n+def _find_transparent_hugepage():\n+ if not os.path.exists(__KERNEL_TRANSPARENT_HP):\n+ return None\n+ value = read_file(__KERNEL_TRANSPARENT_HP)\n+ m = re.search(r\"\\[([a-z]+)\\]\", value)\n+ if not m:\n+ raise RuntimeError(\"BUG: Bad regular expression\")\n+ return m.group(1)\n+\n+\n+class GrubHugepageConfig:\n+ def __init__(self):\n+ self.update()\n+\n+ def update(self):\n+ self.reset()\n+\n+ hugepage_sizes = info.hugepage_sizes_supported\n+ if len(hugepage_sizes) == 0:\n+ raise RuntimeError(\"Hugepages appear to be unsupported\")\n+ cmdline = _find_linux_default_cmdline()\n+ values = _parse_linux_default_cmdline(cmdline)\n+\n+ # parse values in the list\n+ self.default_hugepagesz = info.default_hugepage_size\n+ self.transparent_hugepage = _find_transparent_hugepage()\n+ sizes = []\n+ nrs = []\n+ for k, v in values:\n+ if k == \"default_hugepagesz\":\n+ self.default_hugepagesz = human_readable_to_kilobytes(v)\n+ elif k == \"transparent_hugepage\":\n+ self.transparent_hugepage = v\n+ elif k == \"hugepagesz\":\n+ sizes.append(human_readable_to_kilobytes(v))\n+ elif k == \"hugepages\":\n+ nrs.append(v)\n+ if len(sizes) != len(nrs):\n+ raise RuntimeError(\"GRUB hugepage configuration is wrong\")\n+ detected_hugepages = dict(zip(sizes, map(int, nrs)))\n+ self.nr_hugepages = {size: detected_hugepages.get(size, 0)\n+ for size in hugepage_sizes}\n+\n+ def commit(self):\n+ # perform sanity checks - we can't afford invalid data making it into\n+ # bootloader config, as that might render user's machine unbootable, so\n+ # tread really really carefully\n+\n+ # first, check if user didn't add any unexpected hugepage sizes\n+ configured_sizes = set(self.nr_hugepages.keys())\n+ supported_sizes = set(info.hugepage_sizes_supported)\n+\n+ if configured_sizes != supported_sizes:\n+ diff = configured_sizes.difference(supported_sizes)\n+ raise ValueError(\"Unsupported hugepage sizes: %s\" %\n+ [kilobytes_to_human_readable(s) for s in diff])\n+\n+ # check if default hugepage is one of the supported ones\n+ if self.default_hugepagesz is not None and\\\n+ self.default_hugepagesz not in configured_sizes:\n+ s = kilobytes_to_human_readable(self.default_hugepagesz)\n+ raise ValueError(\"Unsupported default hugepage size: %i\" % s)\n+\n+ # transparent hugepages support was added in recent kernels, so check\n+ # if user is trying to set this\n+ if _find_transparent_hugepage() is None and \\\n+ self.transparent_hugepage is not None:\n+ raise ValueError(\"Transparent hugepages are not unsupported\")\n+\n+ # OK, parameters look to be valid - let's roll\n+\n+ # read and parse current cmdline\n+ cmdline = _find_linux_default_cmdline()\n+\n+ values = _parse_linux_default_cmdline(cmdline)\n+\n+ # clear out old data\n+ klist = [\"transparent_hugepage\", \"default_hugepagesz\",\n+ \"hugepage\", \"hugepagesz\"]\n+ # iterate over a copy so that we could delete items\n+ for k, v in values[:]:\n+ if k in klist:\n+ values.remove((k, v))\n+\n+ # generate new cmdline\n+ cmdline = \" \".join([(\"%s=%s\" % (k, v)) if v is not None else k\n+ for k, v in values])\n+\n+ # now, populate cmdline with new data\n+ new_items = []\n+ for sz, nr in self.nr_hugepages.items():\n+ sz = kilobytes_to_human_readable(sz)\n+ new_items += \"hugepagesz=%s hugepages=%i\" % (sz, nr)\n+ if self.default_hugepagesz is not None:\n+ new_items += \"default_hugepagesz=%i\" % self.default_hugepagesz\n+ if self.transparent_hugepage is not None:\n+ new_items += \"transparent_hugepage=%s\" % self.transparent_hugepage\n+\n+ cmdline = \"%s %s\" % (cmdline, \" \".join(new_items))\n+\n+ # strip any extraneous whitespace we may have added\n+ cmdline = re.sub(r\"\\s\\s+\", \" \", cmdline).strip()\n+\n+ # now, put everything back together\n+ cmdline = '%s=\"%s\"' % (_GRUB_CMDLINE_PARAM_NAME, cmdline)\n+\n+ # write it to config\n+ _generate_linux_default_cmdline(cmdline)\n+\n+ # finally, update GRUB\n+ if not run([\"update-grub\"]):\n+ raise RuntimeError(\"Failed to update GRUB\")\n+ self.update()\n+\n+ def reset(self):\n+ self.nr_hugepages = {} # pagesz: number\n+ self.default_hugepagesz = None\n+ self.transparent_hugepage = None\n", "prefixes": [ "RFC", "9/9" ] }