Patch Detail

HTTP 200 OK
Allow: GET, PUT, PATCH, HEAD, OPTIONS
Content-Type: application/json
Vary: Accept

{
    "id": 95486,
    "url": "http://patchwork.dpdk.org/api/patches/95486/?format=api",
    "web_url": "http://patchwork.dpdk.org/project/dpdk/patch/afb0da82b0b05f8580ff6a5b6ff22e3a7842aa15.1625654882.git.anatoly.burakov@intel.com/",
    "project": {
        "id": 1,
        "url": "http://patchwork.dpdk.org/api/projects/1/?format=api",
        "name": "DPDK",
        "link_name": "dpdk",
        "list_id": "dev.dpdk.org",
        "list_email": "dev@dpdk.org",
        "web_url": "http://core.dpdk.org",
        "scm_url": "git://dpdk.org/dpdk",
        "webscm_url": "http://git.dpdk.org/dpdk",
        "list_archive_url": "https://inbox.dpdk.org/dev",
        "list_archive_url_format": "https://inbox.dpdk.org/dev/{}",
        "commit_url_format": ""
    },
    "msgid": "<afb0da82b0b05f8580ff6a5b6ff22e3a7842aa15.1625654882.git.anatoly.burakov@intel.com>",
    "list_archive_url": "https://inbox.dpdk.org/dev/afb0da82b0b05f8580ff6a5b6ff22e3a7842aa15.1625654882.git.anatoly.burakov@intel.com",
    "date": "2021-07-07T10:48:18",
    "name": "[v7,5/7] power: support callbacks for multiple Rx queues",
    "commit_ref": null,
    "pull_url": null,
    "state": "superseded",
    "archived": true,
    "hash": "21c58f094162f176cfeb0664779e66285a6df540",
    "submitter": {
        "id": 4,
        "url": "http://patchwork.dpdk.org/api/people/4/?format=api",
        "name": "Anatoly Burakov",
        "email": "anatoly.burakov@intel.com"
    },
    "delegate": {
        "id": 1,
        "url": "http://patchwork.dpdk.org/api/users/1/?format=api",
        "username": "tmonjalo",
        "first_name": "Thomas",
        "last_name": "Monjalon",
        "email": "thomas@monjalon.net"
    },
    "mbox": "http://patchwork.dpdk.org/project/dpdk/patch/afb0da82b0b05f8580ff6a5b6ff22e3a7842aa15.1625654882.git.anatoly.burakov@intel.com/mbox/",
    "series": [
        {
            "id": 17694,
            "url": "http://patchwork.dpdk.org/api/series/17694/?format=api",
            "web_url": "http://patchwork.dpdk.org/project/dpdk/list/?series=17694",
            "date": "2021-07-07T10:48:13",
            "name": "Enhancements for PMD power management",
            "version": 7,
            "mbox": "http://patchwork.dpdk.org/series/17694/mbox/"
        }
    ],
    "comments": "http://patchwork.dpdk.org/api/patches/95486/comments/",
    "check": "warning",
    "checks": "http://patchwork.dpdk.org/api/patches/95486/checks/",
    "tags": {},
    "related": [],
    "headers": {
        "Return-Path": "<dev-bounces@dpdk.org>",
        "X-Original-To": "patchwork@inbox.dpdk.org",
        "Delivered-To": "patchwork@inbox.dpdk.org",
        "Received": [
            "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 10993A0C4A;\n\tWed,  7 Jul 2021 12:48:59 +0200 (CEST)",
            "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 472A7414D7;\n\tWed,  7 Jul 2021 12:48:37 +0200 (CEST)",
            "from mga04.intel.com (mga04.intel.com [192.55.52.120])\n by mails.dpdk.org (Postfix) with ESMTP id 2F437414D4\n for <dev@dpdk.org>; Wed,  7 Jul 2021 12:48:35 +0200 (CEST)",
            "from fmsmga005.fm.intel.com ([10.253.24.32])\n by fmsmga104.fm.intel.com with ESMTP/TLS/ECDHE-RSA-AES256-GCM-SHA384;\n 07 Jul 2021 03:48:34 -0700",
            "from silpixa00399498.ir.intel.com (HELO\n silpixa00399498.ger.corp.intel.com) ([10.237.223.53])\n by fmsmga005.fm.intel.com with ESMTP; 07 Jul 2021 03:48:32 -0700"
        ],
        "X-IronPort-AV": [
            "E=McAfee;i=\"6200,9189,10037\"; a=\"207456952\"",
            "E=Sophos;i=\"5.83,331,1616482800\"; d=\"scan'208\";a=\"207456952\"",
            "E=Sophos;i=\"5.83,331,1616482800\"; d=\"scan'208\";a=\"647848640\""
        ],
        "X-ExtLoop1": "1",
        "From": "Anatoly Burakov <anatoly.burakov@intel.com>",
        "To": "dev@dpdk.org,\n\tDavid Hunt <david.hunt@intel.com>",
        "Cc": "konstantin.ananyev@intel.com,\n\tciara.loftus@intel.com",
        "Date": "Wed,  7 Jul 2021 10:48:18 +0000",
        "Message-Id": "\n <afb0da82b0b05f8580ff6a5b6ff22e3a7842aa15.1625654882.git.anatoly.burakov@intel.com>",
        "X-Mailer": "git-send-email 2.25.1",
        "In-Reply-To": "<cover.1625654882.git.anatoly.burakov@intel.com>",
        "References": "<cover.1625498488.git.anatoly.burakov@intel.com>\n <cover.1625654882.git.anatoly.burakov@intel.com>",
        "MIME-Version": "1.0",
        "Content-Transfer-Encoding": "8bit",
        "Subject": "[dpdk-dev] [PATCH v7 5/7] power: support callbacks for multiple Rx\n queues",
        "X-BeenThere": "dev@dpdk.org",
        "X-Mailman-Version": "2.1.29",
        "Precedence": "list",
        "List-Id": "DPDK patches and discussions <dev.dpdk.org>",
        "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>",
        "List-Archive": "<http://mails.dpdk.org/archives/dev/>",
        "List-Post": "<mailto:dev@dpdk.org>",
        "List-Help": "<mailto:dev-request@dpdk.org?subject=help>",
        "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>",
        "Errors-To": "dev-bounces@dpdk.org",
        "Sender": "\"dev\" <dev-bounces@dpdk.org>"
    },
    "content": "Currently, there is a hard limitation on the PMD power management\nsupport that only allows it to support a single queue per lcore. This is\nnot ideal as most DPDK use cases will poll multiple queues per core.\n\nThe PMD power management mechanism relies on ethdev Rx callbacks, so it\nis very difficult to implement such support because callbacks are\neffectively stateless and have no visibility into what the other ethdev\ndevices are doing. This places limitations on what we can do within the\nframework of Rx callbacks, but the basics of this implementation are as\nfollows:\n\n- Replace per-queue structures with per-lcore ones, so that any device\n  polled from the same lcore can share data\n- Any queue that is going to be polled from a specific lcore has to be\n  added to the list of queues to poll, so that the callback is aware of\n  other queues being polled by the same lcore\n- Both the empty poll counter and the actual power saving mechanism is\n  shared between all queues polled on a particular lcore, and is only\n  activated when all queues in the list were polled and were determined\n  to have no traffic.\n- The limitation on UMWAIT-based polling is not removed because UMWAIT\n  is incapable of monitoring more than one address.\n\nAlso, while we're at it, update and improve the docs.\n\nSigned-off-by: Anatoly Burakov <anatoly.burakov@intel.com>\n---\n\nNotes:\n    v7:\n    - Fix bug where initial sleep target was always set to zero\n    - Fix logic in handling of n_queues_ready_to_sleep counter\n    - Update documentation on hardware requirements\n    \n    v6:\n    - Track each individual queue sleep status (Konstantin)\n    - Fix segfault (Dave)\n    \n    v5:\n    - Remove the \"power save queue\" API and replace it with mechanism suggested by\n      Konstantin\n    \n    v3:\n    - Move the list of supported NICs to NIC feature table\n    \n    v2:\n    - Use a TAILQ for queues instead of a static array\n    - Address feedback from Konstantin\n    - Add additional checks for stopped queues\n\n doc/guides/nics/features.rst           |  10 +\n doc/guides/prog_guide/power_man.rst    |  69 ++--\n doc/guides/rel_notes/release_21_08.rst |   3 +\n lib/power/rte_power_pmd_mgmt.c         | 456 +++++++++++++++++++------\n 4 files changed, 402 insertions(+), 136 deletions(-)",
    "diff": "diff --git a/doc/guides/nics/features.rst b/doc/guides/nics/features.rst\nindex 403c2b03a3..a96e12d155 100644\n--- a/doc/guides/nics/features.rst\n+++ b/doc/guides/nics/features.rst\n@@ -912,6 +912,16 @@ Supports to get Rx/Tx packet burst mode information.\n * **[implements] eth_dev_ops**: ``rx_burst_mode_get``, ``tx_burst_mode_get``.\n * **[related] API**: ``rte_eth_rx_burst_mode_get()``, ``rte_eth_tx_burst_mode_get()``.\n \n+.. _nic_features_get_monitor_addr:\n+\n+PMD power management using monitor addresses\n+--------------------------------------------\n+\n+Supports getting a monitoring condition to use together with Ethernet PMD power\n+management (see :doc:`../prog_guide/power_man` for more details).\n+\n+* **[implements] eth_dev_ops**: ``get_monitor_addr``\n+\n .. _nic_features_other:\n \n Other dev ops not represented by a Feature\ndiff --git a/doc/guides/prog_guide/power_man.rst b/doc/guides/prog_guide/power_man.rst\nindex c70ae128ac..0e66878892 100644\n--- a/doc/guides/prog_guide/power_man.rst\n+++ b/doc/guides/prog_guide/power_man.rst\n@@ -198,34 +198,45 @@ Ethernet PMD Power Management API\n Abstract\n ~~~~~~~~\n \n-Existing power management mechanisms require developers\n-to change application design or change code to make use of it.\n-The PMD power management API provides a convenient alternative\n-by utilizing Ethernet PMD RX callbacks,\n-and triggering power saving whenever empty poll count reaches a certain number.\n-\n-Monitor\n-   This power saving scheme will put the CPU into optimized power state\n-   and use the ``rte_power_monitor()`` function\n-   to monitor the Ethernet PMD RX descriptor address,\n-   and wake the CPU up whenever there's new traffic.\n-\n-Pause\n-   This power saving scheme will avoid busy polling\n-   by either entering power-optimized sleep state\n-   with ``rte_power_pause()`` function,\n-   or, if it's not available, use ``rte_pause()``.\n-\n-Frequency scaling\n-   This power saving scheme will use ``librte_power`` library\n-   functionality to scale the core frequency up/down\n-   depending on traffic volume.\n-\n-.. note::\n-\n-   Currently, this power management API is limited to mandatory mapping\n-   of 1 queue to 1 core (multiple queues are supported,\n-   but they must be polled from different cores).\n+Existing power management mechanisms require developers to change application\n+design or change code to make use of it. The PMD power management API provides a\n+convenient alternative by utilizing Ethernet PMD RX callbacks, and triggering\n+power saving whenever empty poll count reaches a certain number.\n+\n+* Monitor\n+   This power saving scheme will put the CPU into optimized power state and\n+   monitor the Ethernet PMD RX descriptor address, waking the CPU up whenever\n+   there's new traffic. Support for this scheme may not be available on all\n+   platforms, and further limitations may apply (see below).\n+\n+* Pause\n+   This power saving scheme will avoid busy polling by either entering\n+   power-optimized sleep state with ``rte_power_pause()`` function, or, if it's\n+   not supported by the underlying platform, use ``rte_pause()``.\n+\n+* Frequency scaling\n+   This power saving scheme will use ``librte_power`` library functionality to\n+   scale the core frequency up/down depending on traffic volume.\n+\n+The \"monitor\" mode is only supported in the following configurations and scenarios:\n+\n+* On Linux* x86_64, `rte_power_monitor()` requires WAITPKG instruction set being\n+  supported by the CPU. Please refer to your platform documentation for further\n+  information.\n+\n+* If ``rte_cpu_get_intrinsics_support()`` function indicates that\n+  ``rte_power_monitor()`` is supported by the platform, then monitoring will be\n+  limited to a mapping of 1 core 1 queue (thus, each Rx queue will have to be\n+  monitored from a different lcore).\n+\n+* If ``rte_cpu_get_intrinsics_support()`` function indicates that the\n+  ``rte_power_monitor()`` function is not supported, then monitor mode will not\n+  be supported.\n+\n+* Not all Ethernet drivers support monitoring, even if the underlying\n+  platform may support the necessary CPU instructions. Please refer to\n+  :doc:`../nics/overview` for more information.\n+\n \n API Overview for Ethernet PMD Power Management\n ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n@@ -242,3 +253,5 @@ References\n \n *   The :doc:`../sample_app_ug/vm_power_management`\n     chapter in the :doc:`../sample_app_ug/index` section.\n+\n+*   The :doc:`../nics/overview` chapter in the :doc:`../nics/index` section\ndiff --git a/doc/guides/rel_notes/release_21_08.rst b/doc/guides/rel_notes/release_21_08.rst\nindex 4b84c89c0b..fce50f0cd6 100644\n--- a/doc/guides/rel_notes/release_21_08.rst\n+++ b/doc/guides/rel_notes/release_21_08.rst\n@@ -85,6 +85,9 @@ New Features\n   usecases. Configuration happens via standard rawdev enq/deq operations. See\n   the :doc:`../rawdevs/cnxk_bphy` rawdev guide for more details on this driver.\n \n+* rte_power: The experimental PMD power management API now supports managing\n+  multiple Ethernet Rx queues per lcore.\n+\n \n Removed Items\n -------------\ndiff --git a/lib/power/rte_power_pmd_mgmt.c b/lib/power/rte_power_pmd_mgmt.c\nindex 9b95cf1794..ceaf386d2b 100644\n--- a/lib/power/rte_power_pmd_mgmt.c\n+++ b/lib/power/rte_power_pmd_mgmt.c\n@@ -33,18 +33,98 @@ enum pmd_mgmt_state {\n \tPMD_MGMT_ENABLED\n };\n \n-struct pmd_queue_cfg {\n+union queue {\n+\tuint32_t val;\n+\tstruct {\n+\t\tuint16_t portid;\n+\t\tuint16_t qid;\n+\t};\n+};\n+\n+struct queue_list_entry {\n+\tTAILQ_ENTRY(queue_list_entry) next;\n+\tunion queue queue;\n+\tuint64_t n_empty_polls;\n+\tuint64_t n_sleeps;\n+\tconst struct rte_eth_rxtx_callback *cb;\n+};\n+\n+struct pmd_core_cfg {\n+\tTAILQ_HEAD(queue_list_head, queue_list_entry) head;\n+\t/**< List of queues associated with this lcore */\n+\tsize_t n_queues;\n+\t/**< How many queues are in the list? */\n \tvolatile enum pmd_mgmt_state pwr_mgmt_state;\n \t/**< State of power management for this queue */\n \tenum rte_power_pmd_mgmt_type cb_mode;\n \t/**< Callback mode for this queue */\n-\tconst struct rte_eth_rxtx_callback *cur_cb;\n-\t/**< Callback instance */\n-\tuint64_t empty_poll_stats;\n-\t/**< Number of empty polls */\n+\tuint64_t n_queues_ready_to_sleep;\n+\t/**< Number of queues ready to enter power optimized state */\n+\tuint64_t sleep_target;\n+\t/**< Prevent a queue from triggering sleep multiple times */\n } __rte_cache_aligned;\n+static struct pmd_core_cfg lcore_cfgs[RTE_MAX_LCORE];\n \n-static struct pmd_queue_cfg port_cfg[RTE_MAX_ETHPORTS][RTE_MAX_QUEUES_PER_PORT];\n+static inline bool\n+queue_equal(const union queue *l, const union queue *r)\n+{\n+\treturn l->val == r->val;\n+}\n+\n+static inline void\n+queue_copy(union queue *dst, const union queue *src)\n+{\n+\tdst->val = src->val;\n+}\n+\n+static struct queue_list_entry *\n+queue_list_find(const struct pmd_core_cfg *cfg, const union queue *q)\n+{\n+\tstruct queue_list_entry *cur;\n+\n+\tTAILQ_FOREACH(cur, &cfg->head, next) {\n+\t\tif (queue_equal(&cur->queue, q))\n+\t\t\treturn cur;\n+\t}\n+\treturn NULL;\n+}\n+\n+static int\n+queue_list_add(struct pmd_core_cfg *cfg, const union queue *q)\n+{\n+\tstruct queue_list_entry *qle;\n+\n+\t/* is it already in the list? */\n+\tif (queue_list_find(cfg, q) != NULL)\n+\t\treturn -EEXIST;\n+\n+\tqle = malloc(sizeof(*qle));\n+\tif (qle == NULL)\n+\t\treturn -ENOMEM;\n+\tmemset(qle, 0, sizeof(*qle));\n+\n+\tqueue_copy(&qle->queue, q);\n+\tTAILQ_INSERT_TAIL(&cfg->head, qle, next);\n+\tcfg->n_queues++;\n+\n+\treturn 0;\n+}\n+\n+static struct queue_list_entry *\n+queue_list_take(struct pmd_core_cfg *cfg, const union queue *q)\n+{\n+\tstruct queue_list_entry *found;\n+\n+\tfound = queue_list_find(cfg, q);\n+\tif (found == NULL)\n+\t\treturn NULL;\n+\n+\tTAILQ_REMOVE(&cfg->head, found, next);\n+\tcfg->n_queues--;\n+\n+\t/* freeing is responsibility of the caller */\n+\treturn found;\n+}\n \n static void\n calc_tsc(void)\n@@ -74,21 +154,75 @@ calc_tsc(void)\n \t}\n }\n \n+static inline void\n+queue_reset(struct pmd_core_cfg *cfg, struct queue_list_entry *qcfg)\n+{\n+\tconst bool is_ready_to_sleep = qcfg->n_sleeps == cfg->sleep_target;\n+\n+\t/* reset empty poll counter for this queue */\n+\tqcfg->n_empty_polls = 0;\n+\t/* reset the queue sleep counter as well */\n+\tqcfg->n_sleeps = 0;\n+\t/* remove the queue from list of queues ready to sleep */\n+\tif (is_ready_to_sleep)\n+\t\tcfg->n_queues_ready_to_sleep--;\n+\t/*\n+\t * no need change the lcore sleep target counter because this lcore will\n+\t * reach the n_sleeps anyway, and the other cores are already counted so\n+\t * there's no need to do anything else.\n+\t */\n+}\n+\n+static inline bool\n+queue_can_sleep(struct pmd_core_cfg *cfg, struct queue_list_entry *qcfg)\n+{\n+\t/* this function is called - that means we have an empty poll */\n+\tqcfg->n_empty_polls++;\n+\n+\t/* if we haven't reached threshold for empty polls, we can't sleep */\n+\tif (qcfg->n_empty_polls <= EMPTYPOLL_MAX)\n+\t\treturn false;\n+\n+\t/*\n+\t * we've reached a point where we are able to sleep, but we still need\n+\t * to check if this queue has already been marked for sleeping.\n+\t */\n+\tif (qcfg->n_sleeps == cfg->sleep_target)\n+\t\treturn true;\n+\n+\t/* mark this queue as ready for sleep */\n+\tqcfg->n_sleeps = cfg->sleep_target;\n+\tcfg->n_queues_ready_to_sleep++;\n+\n+\treturn true;\n+}\n+\n+static inline bool\n+lcore_can_sleep(struct pmd_core_cfg *cfg)\n+{\n+\t/* are all queues ready to sleep? */\n+\tif (cfg->n_queues_ready_to_sleep != cfg->n_queues)\n+\t\treturn false;\n+\n+\t/* we've reached an iteration where we can sleep, reset sleep counter */\n+\tcfg->n_queues_ready_to_sleep = 0;\n+\tcfg->sleep_target++;\n+\n+\treturn true;\n+}\n+\n static uint16_t\n clb_umwait(uint16_t port_id, uint16_t qidx, struct rte_mbuf **pkts __rte_unused,\n-\t\tuint16_t nb_rx, uint16_t max_pkts __rte_unused,\n-\t\tvoid *addr __rte_unused)\n+\t\tuint16_t nb_rx, uint16_t max_pkts __rte_unused, void *arg)\n {\n+\tstruct queue_list_entry *queue_conf = arg;\n \n-\tstruct pmd_queue_cfg *q_conf;\n-\n-\tq_conf = &port_cfg[port_id][qidx];\n-\n+\t/* this callback can't do more than one queue, omit multiqueue logic */\n \tif (unlikely(nb_rx == 0)) {\n-\t\tq_conf->empty_poll_stats++;\n-\t\tif (unlikely(q_conf->empty_poll_stats > EMPTYPOLL_MAX)) {\n+\t\tqueue_conf->n_empty_polls++;\n+\t\tif (unlikely(queue_conf->n_empty_polls > EMPTYPOLL_MAX)) {\n \t\t\tstruct rte_power_monitor_cond pmc;\n-\t\t\tuint16_t ret;\n+\t\t\tint ret;\n \n \t\t\t/* use monitoring condition to sleep */\n \t\t\tret = rte_eth_get_monitor_addr(port_id, qidx,\n@@ -97,60 +231,77 @@ clb_umwait(uint16_t port_id, uint16_t qidx, struct rte_mbuf **pkts __rte_unused,\n \t\t\t\trte_power_monitor(&pmc, UINT64_MAX);\n \t\t}\n \t} else\n-\t\tq_conf->empty_poll_stats = 0;\n+\t\tqueue_conf->n_empty_polls = 0;\n \n \treturn nb_rx;\n }\n \n static uint16_t\n-clb_pause(uint16_t port_id, uint16_t qidx, struct rte_mbuf **pkts __rte_unused,\n-\t\tuint16_t nb_rx, uint16_t max_pkts __rte_unused,\n-\t\tvoid *addr __rte_unused)\n+clb_pause(uint16_t port_id __rte_unused, uint16_t qidx __rte_unused,\n+\t\tstruct rte_mbuf **pkts __rte_unused, uint16_t nb_rx,\n+\t\tuint16_t max_pkts __rte_unused, void *arg)\n {\n-\tstruct pmd_queue_cfg *q_conf;\n+\tconst unsigned int lcore = rte_lcore_id();\n+\tstruct queue_list_entry *queue_conf = arg;\n+\tstruct pmd_core_cfg *lcore_conf;\n+\tconst bool empty = nb_rx == 0;\n \n-\tq_conf = &port_cfg[port_id][qidx];\n+\tlcore_conf = &lcore_cfgs[lcore];\n \n-\tif (unlikely(nb_rx == 0)) {\n-\t\tq_conf->empty_poll_stats++;\n-\t\t/* sleep for 1 microsecond */\n-\t\tif (unlikely(q_conf->empty_poll_stats > EMPTYPOLL_MAX)) {\n-\t\t\t/* use tpause if we have it */\n-\t\t\tif (global_data.intrinsics_support.power_pause) {\n-\t\t\t\tconst uint64_t cur = rte_rdtsc();\n-\t\t\t\tconst uint64_t wait_tsc =\n-\t\t\t\t\t\tcur + global_data.tsc_per_us;\n-\t\t\t\trte_power_pause(wait_tsc);\n-\t\t\t} else {\n-\t\t\t\tuint64_t i;\n-\t\t\t\tfor (i = 0; i < global_data.pause_per_us; i++)\n-\t\t\t\t\trte_pause();\n-\t\t\t}\n+\tif (likely(!empty))\n+\t\t/* early exit */\n+\t\tqueue_reset(lcore_conf, queue_conf);\n+\telse {\n+\t\t/* can this queue sleep? */\n+\t\tif (!queue_can_sleep(lcore_conf, queue_conf))\n+\t\t\treturn nb_rx;\n+\n+\t\t/* can this lcore sleep? */\n+\t\tif (!lcore_can_sleep(lcore_conf))\n+\t\t\treturn nb_rx;\n+\n+\t\t/* sleep for 1 microsecond, use tpause if we have it */\n+\t\tif (global_data.intrinsics_support.power_pause) {\n+\t\t\tconst uint64_t cur = rte_rdtsc();\n+\t\t\tconst uint64_t wait_tsc =\n+\t\t\t\t\tcur + global_data.tsc_per_us;\n+\t\t\trte_power_pause(wait_tsc);\n+\t\t} else {\n+\t\t\tuint64_t i;\n+\t\t\tfor (i = 0; i < global_data.pause_per_us; i++)\n+\t\t\t\trte_pause();\n \t\t}\n-\t} else\n-\t\tq_conf->empty_poll_stats = 0;\n+\t}\n \n \treturn nb_rx;\n }\n \n static uint16_t\n-clb_scale_freq(uint16_t port_id, uint16_t qidx,\n+clb_scale_freq(uint16_t port_id __rte_unused, uint16_t qidx __rte_unused,\n \t\tstruct rte_mbuf **pkts __rte_unused, uint16_t nb_rx,\n-\t\tuint16_t max_pkts __rte_unused, void *_  __rte_unused)\n+\t\tuint16_t max_pkts __rte_unused, void *arg)\n {\n-\tstruct pmd_queue_cfg *q_conf;\n+\tconst unsigned int lcore = rte_lcore_id();\n+\tconst bool empty = nb_rx == 0;\n+\tstruct pmd_core_cfg *lcore_conf = &lcore_cfgs[lcore];\n+\tstruct queue_list_entry *queue_conf = arg;\n \n-\tq_conf = &port_cfg[port_id][qidx];\n+\tif (likely(!empty)) {\n+\t\t/* early exit */\n+\t\tqueue_reset(lcore_conf, queue_conf);\n \n-\tif (unlikely(nb_rx == 0)) {\n-\t\tq_conf->empty_poll_stats++;\n-\t\tif (unlikely(q_conf->empty_poll_stats > EMPTYPOLL_MAX))\n-\t\t\t/* scale down freq */\n-\t\t\trte_power_freq_min(rte_lcore_id());\n-\t} else {\n-\t\tq_conf->empty_poll_stats = 0;\n-\t\t/* scale up freq */\n+\t\t/* scale up freq immediately */\n \t\trte_power_freq_max(rte_lcore_id());\n+\t} else {\n+\t\t/* can this queue sleep? */\n+\t\tif (!queue_can_sleep(lcore_conf, queue_conf))\n+\t\t\treturn nb_rx;\n+\n+\t\t/* can this lcore sleep? */\n+\t\tif (!lcore_can_sleep(lcore_conf))\n+\t\t\treturn nb_rx;\n+\n+\t\trte_power_freq_min(rte_lcore_id());\n \t}\n \n \treturn nb_rx;\n@@ -167,11 +318,80 @@ queue_stopped(const uint16_t port_id, const uint16_t queue_id)\n \treturn qinfo.queue_state == RTE_ETH_QUEUE_STATE_STOPPED;\n }\n \n+static int\n+cfg_queues_stopped(struct pmd_core_cfg *queue_cfg)\n+{\n+\tconst struct queue_list_entry *entry;\n+\n+\tTAILQ_FOREACH(entry, &queue_cfg->head, next) {\n+\t\tconst union queue *q = &entry->queue;\n+\t\tint ret = queue_stopped(q->portid, q->qid);\n+\t\tif (ret != 1)\n+\t\t\treturn ret;\n+\t}\n+\treturn 1;\n+}\n+\n+static int\n+check_scale(unsigned int lcore)\n+{\n+\tenum power_management_env env;\n+\n+\t/* only PSTATE and ACPI modes are supported */\n+\tif (!rte_power_check_env_supported(PM_ENV_ACPI_CPUFREQ) &&\n+\t    !rte_power_check_env_supported(PM_ENV_PSTATE_CPUFREQ)) {\n+\t\tRTE_LOG(DEBUG, POWER, \"Neither ACPI nor PSTATE modes are supported\\n\");\n+\t\treturn -ENOTSUP;\n+\t}\n+\t/* ensure we could initialize the power library */\n+\tif (rte_power_init(lcore))\n+\t\treturn -EINVAL;\n+\n+\t/* ensure we initialized the correct env */\n+\tenv = rte_power_get_env();\n+\tif (env != PM_ENV_ACPI_CPUFREQ && env != PM_ENV_PSTATE_CPUFREQ) {\n+\t\tRTE_LOG(DEBUG, POWER, \"Neither ACPI nor PSTATE modes were initialized\\n\");\n+\t\treturn -ENOTSUP;\n+\t}\n+\n+\t/* we're done */\n+\treturn 0;\n+}\n+\n+static int\n+check_monitor(struct pmd_core_cfg *cfg, const union queue *qdata)\n+{\n+\tstruct rte_power_monitor_cond dummy;\n+\n+\t/* check if rte_power_monitor is supported */\n+\tif (!global_data.intrinsics_support.power_monitor) {\n+\t\tRTE_LOG(DEBUG, POWER, \"Monitoring intrinsics are not supported\\n\");\n+\t\treturn -ENOTSUP;\n+\t}\n+\n+\tif (cfg->n_queues > 0) {\n+\t\tRTE_LOG(DEBUG, POWER, \"Monitoring multiple queues is not supported\\n\");\n+\t\treturn -ENOTSUP;\n+\t}\n+\n+\t/* check if the device supports the necessary PMD API */\n+\tif (rte_eth_get_monitor_addr(qdata->portid, qdata->qid,\n+\t\t\t&dummy) == -ENOTSUP) {\n+\t\tRTE_LOG(DEBUG, POWER, \"The device does not support rte_eth_get_monitor_addr\\n\");\n+\t\treturn -ENOTSUP;\n+\t}\n+\n+\t/* we're done */\n+\treturn 0;\n+}\n+\n int\n rte_power_ethdev_pmgmt_queue_enable(unsigned int lcore_id, uint16_t port_id,\n \t\tuint16_t queue_id, enum rte_power_pmd_mgmt_type mode)\n {\n-\tstruct pmd_queue_cfg *queue_cfg;\n+\tconst union queue qdata = {.portid = port_id, .qid = queue_id};\n+\tstruct pmd_core_cfg *lcore_cfg;\n+\tstruct queue_list_entry *queue_cfg;\n \tstruct rte_eth_dev_info info;\n \trte_rx_callback_fn clb;\n \tint ret;\n@@ -202,9 +422,19 @@ rte_power_ethdev_pmgmt_queue_enable(unsigned int lcore_id, uint16_t port_id,\n \t\tgoto end;\n \t}\n \n-\tqueue_cfg = &port_cfg[port_id][queue_id];\n+\tlcore_cfg = &lcore_cfgs[lcore_id];\n \n-\tif (queue_cfg->pwr_mgmt_state != PMD_MGMT_DISABLED) {\n+\t/* check if other queues are stopped as well */\n+\tret = cfg_queues_stopped(lcore_cfg);\n+\tif (ret != 1) {\n+\t\t/* error means invalid queue, 0 means queue wasn't stopped */\n+\t\tret = ret < 0 ? -EINVAL : -EBUSY;\n+\t\tgoto end;\n+\t}\n+\n+\t/* if callback was already enabled, check current callback type */\n+\tif (lcore_cfg->pwr_mgmt_state != PMD_MGMT_DISABLED &&\n+\t\t\tlcore_cfg->cb_mode != mode) {\n \t\tret = -EINVAL;\n \t\tgoto end;\n \t}\n@@ -214,53 +444,20 @@ rte_power_ethdev_pmgmt_queue_enable(unsigned int lcore_id, uint16_t port_id,\n \n \tswitch (mode) {\n \tcase RTE_POWER_MGMT_TYPE_MONITOR:\n-\t{\n-\t\tstruct rte_power_monitor_cond dummy;\n-\n-\t\t/* check if rte_power_monitor is supported */\n-\t\tif (!global_data.intrinsics_support.power_monitor) {\n-\t\t\tRTE_LOG(DEBUG, POWER, \"Monitoring intrinsics are not supported\\n\");\n-\t\t\tret = -ENOTSUP;\n+\t\t/* check if we can add a new queue */\n+\t\tret = check_monitor(lcore_cfg, &qdata);\n+\t\tif (ret < 0)\n \t\t\tgoto end;\n-\t\t}\n \n-\t\t/* check if the device supports the necessary PMD API */\n-\t\tif (rte_eth_get_monitor_addr(port_id, queue_id,\n-\t\t\t\t&dummy) == -ENOTSUP) {\n-\t\t\tRTE_LOG(DEBUG, POWER, \"The device does not support rte_eth_get_monitor_addr\\n\");\n-\t\t\tret = -ENOTSUP;\n-\t\t\tgoto end;\n-\t\t}\n \t\tclb = clb_umwait;\n \t\tbreak;\n-\t}\n \tcase RTE_POWER_MGMT_TYPE_SCALE:\n-\t{\n-\t\tenum power_management_env env;\n-\t\t/* only PSTATE and ACPI modes are supported */\n-\t\tif (!rte_power_check_env_supported(PM_ENV_ACPI_CPUFREQ) &&\n-\t\t\t\t!rte_power_check_env_supported(\n-\t\t\t\t\tPM_ENV_PSTATE_CPUFREQ)) {\n-\t\t\tRTE_LOG(DEBUG, POWER, \"Neither ACPI nor PSTATE modes are supported\\n\");\n-\t\t\tret = -ENOTSUP;\n+\t\t/* check if we can add a new queue */\n+\t\tret = check_scale(lcore_id);\n+\t\tif (ret < 0)\n \t\t\tgoto end;\n-\t\t}\n-\t\t/* ensure we could initialize the power library */\n-\t\tif (rte_power_init(lcore_id)) {\n-\t\t\tret = -EINVAL;\n-\t\t\tgoto end;\n-\t\t}\n-\t\t/* ensure we initialized the correct env */\n-\t\tenv = rte_power_get_env();\n-\t\tif (env != PM_ENV_ACPI_CPUFREQ &&\n-\t\t\t\tenv != PM_ENV_PSTATE_CPUFREQ) {\n-\t\t\tRTE_LOG(DEBUG, POWER, \"Neither ACPI nor PSTATE modes were initialized\\n\");\n-\t\t\tret = -ENOTSUP;\n-\t\t\tgoto end;\n-\t\t}\n \t\tclb = clb_scale_freq;\n \t\tbreak;\n-\t}\n \tcase RTE_POWER_MGMT_TYPE_PAUSE:\n \t\t/* figure out various time-to-tsc conversions */\n \t\tif (global_data.tsc_per_us == 0)\n@@ -273,13 +470,27 @@ rte_power_ethdev_pmgmt_queue_enable(unsigned int lcore_id, uint16_t port_id,\n \t\tret = -EINVAL;\n \t\tgoto end;\n \t}\n+\t/* add this queue to the list */\n+\tret = queue_list_add(lcore_cfg, &qdata);\n+\tif (ret < 0) {\n+\t\tRTE_LOG(DEBUG, POWER, \"Failed to add queue to list: %s\\n\",\n+\t\t\t\tstrerror(-ret));\n+\t\tgoto end;\n+\t}\n+\t/* new queue is always added last */\n+\tqueue_cfg = TAILQ_LAST(&lcore_cfg->head, queue_list_head);\n+\n+\t/* when enabling first queue, ensure sleep target is not 0 */\n+\tif (lcore_cfg->n_queues == 1 && lcore_cfg->sleep_target == 0)\n+\t\tlcore_cfg->sleep_target = 1;\n \n \t/* initialize data before enabling the callback */\n-\tqueue_cfg->empty_poll_stats = 0;\n-\tqueue_cfg->cb_mode = mode;\n-\tqueue_cfg->pwr_mgmt_state = PMD_MGMT_ENABLED;\n-\tqueue_cfg->cur_cb = rte_eth_add_rx_callback(port_id, queue_id,\n-\t\t\tclb, NULL);\n+\tif (lcore_cfg->n_queues == 1) {\n+\t\tlcore_cfg->cb_mode = mode;\n+\t\tlcore_cfg->pwr_mgmt_state = PMD_MGMT_ENABLED;\n+\t}\n+\tqueue_cfg->cb = rte_eth_add_rx_callback(port_id, queue_id,\n+\t\t\tclb, queue_cfg);\n \n \tret = 0;\n end:\n@@ -290,7 +501,9 @@ int\n rte_power_ethdev_pmgmt_queue_disable(unsigned int lcore_id,\n \t\tuint16_t port_id, uint16_t queue_id)\n {\n-\tstruct pmd_queue_cfg *queue_cfg;\n+\tconst union queue qdata = {.portid = port_id, .qid = queue_id};\n+\tstruct pmd_core_cfg *lcore_cfg;\n+\tstruct queue_list_entry *queue_cfg;\n \tint ret;\n \n \tRTE_ETH_VALID_PORTID_OR_ERR_RET(port_id, -EINVAL);\n@@ -306,24 +519,40 @@ rte_power_ethdev_pmgmt_queue_disable(unsigned int lcore_id,\n \t}\n \n \t/* no need to check queue id as wrong queue id would not be enabled */\n-\tqueue_cfg = &port_cfg[port_id][queue_id];\n+\tlcore_cfg = &lcore_cfgs[lcore_id];\n \n-\tif (queue_cfg->pwr_mgmt_state != PMD_MGMT_ENABLED)\n+\t/* check if other queues are stopped as well */\n+\tret = cfg_queues_stopped(lcore_cfg);\n+\tif (ret != 1) {\n+\t\t/* error means invalid queue, 0 means queue wasn't stopped */\n+\t\treturn ret < 0 ? -EINVAL : -EBUSY;\n+\t}\n+\n+\tif (lcore_cfg->pwr_mgmt_state != PMD_MGMT_ENABLED)\n \t\treturn -EINVAL;\n \n-\t/* stop any callbacks from progressing */\n-\tqueue_cfg->pwr_mgmt_state = PMD_MGMT_DISABLED;\n+\t/*\n+\t * There is no good/easy way to do this without race conditions, so we\n+\t * are just going to throw our hands in the air and hope that the user\n+\t * has read the documentation and has ensured that ports are stopped at\n+\t * the time we enter the API functions.\n+\t */\n+\tqueue_cfg = queue_list_take(lcore_cfg, &qdata);\n+\tif (queue_cfg == NULL)\n+\t\treturn -ENOENT;\n \n-\tswitch (queue_cfg->cb_mode) {\n+\t/* if we've removed all queues from the lists, set state to disabled */\n+\tif (lcore_cfg->n_queues == 0)\n+\t\tlcore_cfg->pwr_mgmt_state = PMD_MGMT_DISABLED;\n+\n+\tswitch (lcore_cfg->cb_mode) {\n \tcase RTE_POWER_MGMT_TYPE_MONITOR: /* fall-through */\n \tcase RTE_POWER_MGMT_TYPE_PAUSE:\n-\t\trte_eth_remove_rx_callback(port_id, queue_id,\n-\t\t\t\tqueue_cfg->cur_cb);\n+\t\trte_eth_remove_rx_callback(port_id, queue_id, queue_cfg->cb);\n \t\tbreak;\n \tcase RTE_POWER_MGMT_TYPE_SCALE:\n \t\trte_power_freq_max(lcore_id);\n-\t\trte_eth_remove_rx_callback(port_id, queue_id,\n-\t\t\t\tqueue_cfg->cur_cb);\n+\t\trte_eth_remove_rx_callback(port_id, queue_id, queue_cfg->cb);\n \t\trte_power_exit(lcore_id);\n \t\tbreak;\n \t}\n@@ -332,7 +561,18 @@ rte_power_ethdev_pmgmt_queue_disable(unsigned int lcore_id,\n \t * ports before calling any of these API's, so we can assume that the\n \t * callbacks can be freed. we're intentionally casting away const-ness.\n \t */\n-\trte_free((void *)queue_cfg->cur_cb);\n+\trte_free((void *)queue_cfg->cb);\n+\tfree(queue_cfg);\n \n \treturn 0;\n }\n+\n+RTE_INIT(rte_power_ethdev_pmgmt_init) {\n+\tsize_t i;\n+\n+\t/* initialize all tailqs */\n+\tfor (i = 0; i < RTE_DIM(lcore_cfgs); i++) {\n+\t\tstruct pmd_core_cfg *cfg = &lcore_cfgs[i];\n+\t\tTAILQ_INIT(&cfg->head);\n+\t}\n+}\n",
    "prefixes": [
        "v7",
        "5/7"
    ]
}