[v1,3/3] examples/l3fwd-power: enable PMD power mgmt on Arm
Checks
Commit Message
For Arm aarch, power monitor uses WFE instruction to enable, which can
not exit automatically within the time limit. This means
'rte_power_monitor_wakeup' API needs to be called to wake up sleep cores
if there is no store operation to monitored address.
Furthermore, we disable power monitor feature on the main core so that it
can be used to wake up other sleeping cores when it receives SIGINT
siginal.
Signed-off-by: Feifei Wang <feifei.wang2@arm.com>
Reviewed-by: Ruifeng Wang <ruifeng.wang@arm.com>
---
examples/l3fwd-power/main.c | 30 +++++++++++++++++++++++++++++-
1 file changed, 29 insertions(+), 1 deletion(-)
Comments
On 25/08/2022 07:42, Feifei Wang wrote:
> For Arm aarch, power monitor uses WFE instruction to enable, which can
> not exit automatically within the time limit. This means
> 'rte_power_monitor_wakeup' API needs to be called to wake up sleep cores
> if there is no store operation to monitored address.
>
> Furthermore, we disable power monitor feature on the main core so that it
> can be used to wake up other sleeping cores when it receives SIGINT
> siginal.
>
> Signed-off-by: Feifei Wang <feifei.wang2@arm.com>
> Reviewed-by: Ruifeng Wang <ruifeng.wang@arm.com>
> ---
> examples/l3fwd-power/main.c | 30 +++++++++++++++++++++++++++++-
> 1 file changed, 29 insertions(+), 1 deletion(-)
>
> diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
> index 887c6eae3f..2bd0d700f0 100644
> --- a/examples/l3fwd-power/main.c
> +++ b/examples/l3fwd-power/main.c
> @@ -432,8 +432,16 @@ static void
> signal_exit_now(int sigtype)
> {
>
> - if (sigtype == SIGINT)
> + if (sigtype == SIGINT) {
> +#if defined(RTE_ARCH_ARM64)
> + /**
> + * wake_up api does not need input parameter on Arm,
> + * so 0 is meaningless here.
> + */
> + rte_power_monitor_wakeup(0);
> +#endif
> quit_signal = true;
> + }
>
> }
>
> @@ -2885,6 +2893,25 @@ main(int argc, char **argv)
> "Error setting scaling freq max: err=%d, lcore %d\n",
> ret, lcore_id);
>
> +#if defined(RTE_ARCH_ARM64)
> + /* Ensure the main lcore does not enter the power-monitor state,
> + * so that it can be used to wake up other lcores on ARM.
> + * This is due to WFE instruction has no timeout wake-up mechanism,
> + * and if users want to exit actively, the main lcore is needed
> + * to send SEV instruction to wake up other lcores.
> + */
> + unsigned int main_lcore = rte_get_main_lcore();
> + if (lcore_id != main_lcore ||
> + pmgmt_type != RTE_POWER_MGMT_TYPE_MONITOR) {
> + ret = rte_power_ethdev_pmgmt_queue_enable(
> + lcore_id, portid, queueid,
> + pmgmt_type);
> + if (ret < 0)
> + rte_exit(EXIT_FAILURE,
> + "rte_power_ethdev_pmgmt_queue_enable: err=%d, port=%d\n",
> + ret, portid);
> + }
> +#else
> ret = rte_power_ethdev_pmgmt_queue_enable(
> lcore_id, portid, queueid,
> pmgmt_type);
> @@ -2892,6 +2919,7 @@ main(int argc, char **argv)
> rte_exit(EXIT_FAILURE,
> "rte_power_ethdev_pmgmt_queue_enable: err=%d, port=%d\n",
> ret, portid);
> +#endif
> }
> }
> }
Hi Feifei,
Acked-by: David Hunt <david.hunt@intel.com>
On Mon, Aug 29, 2022 at 2:48 PM Hunt, David <david.hunt@intel.com> wrote:
> On 25/08/2022 07:42, Feifei Wang wrote:
> > For Arm aarch, power monitor uses WFE instruction to enable, which can
> > not exit automatically within the time limit. This means
> > 'rte_power_monitor_wakeup' API needs to be called to wake up sleep cores
> > if there is no store operation to monitored address.
> >
> > Furthermore, we disable power monitor feature on the main core so that it
> > can be used to wake up other sleeping cores when it receives SIGINT
> > siginal.
> >
> > Signed-off-by: Feifei Wang <feifei.wang2@arm.com>
> > Reviewed-by: Ruifeng Wang <ruifeng.wang@arm.com>
> > ---
> > examples/l3fwd-power/main.c | 30 +++++++++++++++++++++++++++++-
> > 1 file changed, 29 insertions(+), 1 deletion(-)
> >
> > diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
> > index 887c6eae3f..2bd0d700f0 100644
> > --- a/examples/l3fwd-power/main.c
> > +++ b/examples/l3fwd-power/main.c
> > @@ -432,8 +432,16 @@ static void
> > signal_exit_now(int sigtype)
> > {
> >
> > - if (sigtype == SIGINT)
> > + if (sigtype == SIGINT) {
> > +#if defined(RTE_ARCH_ARM64)
Having a arch specific behavior in the application shows that there is
something wrong either in the API, or in the Arm implementation of the
API.
I don't think this is a good solution.
Can't we find a better alternative? By changing the API probably?
> > + /**
> > + * wake_up api does not need input parameter on Arm,
> > + * so 0 is meaningless here.
> > + */
> > + rte_power_monitor_wakeup(0);
> > +#endif
> > quit_signal = true;
> > + }
> >
> > }
> >
> > @@ -2885,6 +2893,25 @@ main(int argc, char **argv)
> > "Error setting scaling freq max: err=%d, lcore %d\n",
> > ret, lcore_id);
> >
> > +#if defined(RTE_ARCH_ARM64)
> > + /* Ensure the main lcore does not enter the power-monitor state,
> > + * so that it can be used to wake up other lcores on ARM.
> > + * This is due to WFE instruction has no timeout wake-up mechanism,
> > + * and if users want to exit actively, the main lcore is needed
> > + * to send SEV instruction to wake up other lcores.
> > + */
> > + unsigned int main_lcore = rte_get_main_lcore();
> > + if (lcore_id != main_lcore ||
> > + pmgmt_type != RTE_POWER_MGMT_TYPE_MONITOR) {
> > + ret = rte_power_ethdev_pmgmt_queue_enable(
> > + lcore_id, portid, queueid,
> > + pmgmt_type);
> > + if (ret < 0)
> > + rte_exit(EXIT_FAILURE,
> > + "rte_power_ethdev_pmgmt_queue_enable: err=%d, port=%d\n",
> > + ret, portid);
> > + }
> > +#else
> > ret = rte_power_ethdev_pmgmt_queue_enable(
> > lcore_id, portid, queueid,
> > pmgmt_type);
> > @@ -2892,6 +2919,7 @@ main(int argc, char **argv)
> > rte_exit(EXIT_FAILURE,
> > "rte_power_ethdev_pmgmt_queue_enable: err=%d, port=%d\n",
> > ret, portid);
> > +#endif
> > }
> > }
> > }
>
>
> Hi Feifei,
>
> Acked-by: David Hunt <david.hunt@intel.com>
> -----邮件原件-----
> 发件人: David Marchand <david.marchand@redhat.com>
> 发送时间: Monday, October 3, 2022 3:12 PM
> 收件人: Hunt, David <david.hunt@intel.com>; Feifei Wang
> <Feifei.Wang2@arm.com>; Ruifeng Wang <Ruifeng.Wang@arm.com>
> 抄送: dev@dpdk.org; nd <nd@arm.com>; thomas@monjalon.net
> 主题: Re: [PATCH v1 3/3] examples/l3fwd-power: enable PMD power mgmt
> on Arm
>
> On Mon, Aug 29, 2022 at 2:48 PM Hunt, David <david.hunt@intel.com> wrote:
> > On 25/08/2022 07:42, Feifei Wang wrote:
> > > For Arm aarch, power monitor uses WFE instruction to enable, which
> > > can not exit automatically within the time limit. This means
> > > 'rte_power_monitor_wakeup' API needs to be called to wake up sleep
> > > cores if there is no store operation to monitored address.
> > >
> > > Furthermore, we disable power monitor feature on the main core so
> > > that it can be used to wake up other sleeping cores when it receives
> > > SIGINT siginal.
> > >
> > > Signed-off-by: Feifei Wang <feifei.wang2@arm.com>
> > > Reviewed-by: Ruifeng Wang <ruifeng.wang@arm.com>
> > > ---
> > > examples/l3fwd-power/main.c | 30
> +++++++++++++++++++++++++++++-
> > > 1 file changed, 29 insertions(+), 1 deletion(-)
> > >
> > > diff --git a/examples/l3fwd-power/main.c
> > > b/examples/l3fwd-power/main.c index 887c6eae3f..2bd0d700f0 100644
> > > --- a/examples/l3fwd-power/main.c
> > > +++ b/examples/l3fwd-power/main.c
> > > @@ -432,8 +432,16 @@ static void
> > > signal_exit_now(int sigtype)
> > > {
> > >
> > > - if (sigtype == SIGINT)
> > > + if (sigtype == SIGINT) {
> > > +#if defined(RTE_ARCH_ARM64)
>
> Having a arch specific behavior in the application shows that there is
> something wrong either in the API, or in the Arm implementation of the API.
> I don't think this is a good solution.
>
> Can't we find a better alternative? By changing the API probably?
Sorry I do not understand ' shows that there is something wrong either in the API'
Here we call ' rte_power_monitor_wakeup' API is due to that we need to wake
up all cores from WFE instructions in arm, and then l3fwd can exit correctly.
This is due to that arm arch is different from x86, if there is no packets received, x86's
'UMONITOR' can automatically exit from energy-saving state after waiting for a period of time.
But arm's 'WFE' can not exit automatically. It will wait 'SEV' instructions in wake_up API to wake
up it.
Finally, if user want to exit l3fwd by 'SIGINT' in arm, main core should firstly call 'wake_up' API
to force worker cores to exit from energy-saving state.
Otherwise, the worker will stay in the energy-saving state forever if no packet is received.
>
>
> > > + /**
> > > + * wake_up api does not need input parameter on Arm,
> > > + * so 0 is meaningless here.
> > > + */
> > > + rte_power_monitor_wakeup(0); #endif
> > > quit_signal = true;
> > > + }
> > >
> > > }
> > >
> > > @@ -2885,6 +2893,25 @@ main(int argc, char **argv)
> > > "Error setting scaling freq max: err=%d,
> lcore %d\n",
> > > ret,
> > > lcore_id);
> > >
> > > +#if defined(RTE_ARCH_ARM64)
> > > + /* Ensure the main lcore does not enter the power-
> monitor state,
> > > + * so that it can be used to wake up other lcores on ARM.
> > > + * This is due to WFE instruction has no timeout wake-up
> mechanism,
> > > + * and if users want to exit actively, the main lcore is
> needed
> > > + * to send SEV instruction to wake up other lcores.
> > > + */
> > > + unsigned int main_lcore = rte_get_main_lcore();
> > > + if (lcore_id != main_lcore ||
> > > + pmgmt_type !=
> RTE_POWER_MGMT_TYPE_MONITOR) {
> > > + ret = rte_power_ethdev_pmgmt_queue_enable(
> > > + lcore_id, portid, queueid,
> > > + pmgmt_type);
> > > + if (ret < 0)
> > > + rte_exit(EXIT_FAILURE,
> > > + "rte_power_ethdev_pmgmt_queue_enable:
> err=%d, port=%d\n",
> > > + ret, portid);
> > > + }
> > > +#else
> > > ret = rte_power_ethdev_pmgmt_queue_enable(
> > > lcore_id, portid, queueid,
> > > pmgmt_type); @@
> > > -2892,6 +2919,7 @@ main(int argc, char **argv)
> > > rte_exit(EXIT_FAILURE,
> > > "rte_power_ethdev_pmgmt_queue_enable:
> err=%d, port=%d\n",
> > > ret, portid);
> > > +#endif
> > > }
> > > }
> > > }
> >
> >
> > Hi Feifei,
> >
> > Acked-by: David Hunt <david.hunt@intel.com>
>
>
> --
> David Marchand
11/10/2022 09:56, Feifei Wang:
> David Marchand <david.marchand@redhat.com>
> > > On 25/08/2022 07:42, Feifei Wang wrote:
> > > > --- a/examples/l3fwd-power/main.c
> > > > +++ b/examples/l3fwd-power/main.c
> > > > @@ -432,8 +432,16 @@ static void
> > > > signal_exit_now(int sigtype)
> > > > {
> > > >
> > > > - if (sigtype == SIGINT)
> > > > + if (sigtype == SIGINT) {
> > > > +#if defined(RTE_ARCH_ARM64)
> >
> > Having a arch specific behavior in the application shows that there is
> > something wrong either in the API, or in the Arm implementation of the API.
> > I don't think this is a good solution.
> >
> > Can't we find a better alternative? By changing the API probably?
> Sorry I do not understand ' shows that there is something wrong either in the API'
David means the application developer should not have to be aware
of the arch differences.
When you use an API, you don't check how it is implemented,
and you are not supposed to use #ifdef.
The API must be arch-agnostic.
> Here we call ' rte_power_monitor_wakeup' API is due to that we need to wake
> up all cores from WFE instructions in arm, and then l3fwd can exit correctly.
>
> This is due to that arm arch is different from x86, if there is no packets received, x86's
> 'UMONITOR' can automatically exit from energy-saving state after waiting for a period of time.
> But arm's 'WFE' can not exit automatically. It will wait 'SEV' instructions in wake_up API to wake
> up it.
>
> Finally, if user want to exit l3fwd by 'SIGINT' in arm, main core should firstly call 'wake_up' API
> to force worker cores to exit from energy-saving state.
> Otherwise, the worker will stay in the energy-saving state forever if no packet is received.
Please find a way to have a common API,
even if the API implementation is empty in x86 case.
> >
> >
> > > > + /**
> > > > + * wake_up api does not need input parameter on Arm,
> > > > + * so 0 is meaningless here.
> > > > + */
> > > > + rte_power_monitor_wakeup(0); #endif
> > > > quit_signal = true;
> > > > + }
On Thu, 25 Aug 2022 14:42:51 +0800
Feifei Wang <feifei.wang2@arm.com> wrote:
> diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-power/main.c
> index 887c6eae3f..2bd0d700f0 100644
> --- a/examples/l3fwd-power/main.c
> +++ b/examples/l3fwd-power/main.c
> @@ -432,8 +432,16 @@ static void
> signal_exit_now(int sigtype)
> {
>
> - if (sigtype == SIGINT)
> + if (sigtype == SIGINT) {
> +#if defined(RTE_ARCH_ARM64)
> + /**
> + * wake_up api does not need input parameter on Arm,
> + * so 0 is meaningless here.
> + */
> + rte_power_monitor_wakeup(0);
> +#endif
> quit_signal = true;
> + }
>
This method is problematic. There is no guarantee that power
monitor is async signal safe.
> -----邮件原件-----
> 发件人: Thomas Monjalon <thomas@monjalon.net>
> 发送时间: Friday, October 21, 2022 4:42 AM
> 收件人: David Marchand <david.marchand@redhat.com>
> 抄送: Hunt, David <david.hunt@intel.com>; Ruifeng Wang
> <Ruifeng.Wang@arm.com>; dev@dpdk.org; nd <nd@arm.com>; Feifei
> Wang <Feifei.Wang2@arm.com>
> 主题: Re: 回复: [PATCH v1 3/3] examples/l3fwd-power: enable PMD power
> mgmt on Arm
>
> 11/10/2022 09:56, Feifei Wang:
> > David Marchand <david.marchand@redhat.com>
> > > > On 25/08/2022 07:42, Feifei Wang wrote:
> > > > > --- a/examples/l3fwd-power/main.c
> > > > > +++ b/examples/l3fwd-power/main.c
> > > > > @@ -432,8 +432,16 @@ static void
> > > > > signal_exit_now(int sigtype)
> > > > > {
> > > > >
> > > > > - if (sigtype == SIGINT)
> > > > > + if (sigtype == SIGINT) {
> > > > > +#if defined(RTE_ARCH_ARM64)
> > >
> > > Having a arch specific behavior in the application shows that there
> > > is something wrong either in the API, or in the Arm implementation of
> the API.
> > > I don't think this is a good solution.
> > >
> > > Can't we find a better alternative? By changing the API probably?
> > Sorry I do not understand ' shows that there is something wrong either in
> the API'
>
> David means the application developer should not have to be aware of the
> arch differences.
> When you use an API, you don't check how it is implemented, and you are
> not supposed to use #ifdef.
> The API must be arch-agnostic.
Ok, Understand. Thanks for the explanation.
>
> > Here we call ' rte_power_monitor_wakeup' API is due to that we need to
> > wake up all cores from WFE instructions in arm, and then l3fwd can exit
> correctly.
> >
> > This is due to that arm arch is different from x86, if there is no
> > packets received, x86's 'UMONITOR' can automatically exit from energy-
> saving state after waiting for a period of time.
> > But arm's 'WFE' can not exit automatically. It will wait 'SEV'
> > instructions in wake_up API to wake up it.
> >
> > Finally, if user want to exit l3fwd by 'SIGINT' in arm, main core
> > should firstly call 'wake_up' API to force worker cores to exit from energy-
> saving state.
> > Otherwise, the worker will stay in the energy-saving state forever if no
> packet is received.
>
> Please find a way to have a common API,
> even if the API implementation is empty in x86 case.
Yes, I think what we need to do is not a create a new API, it is to look
for a correct location to call 'rte_power_monitor_wakeup'.
>
> > >
> > >
> > > > > + /**
> > > > > + * wake_up api does not need input parameter on Arm,
> > > > > + * so 0 is meaningless here.
> > > > > + */
> > > > > + rte_power_monitor_wakeup(0); #endif
> > > > > quit_signal = true;
> > > > > + }
>
>
> -----邮件原件-----
> 发件人: Stephen Hemminger <stephen@networkplumber.org>
> 发送时间: Friday, October 21, 2022 6:10 AM
> 收件人: Feifei Wang <Feifei.Wang2@arm.com>
> 抄送: David Hunt <david.hunt@intel.com>; dev@dpdk.org; nd
> <nd@arm.com>; Ruifeng Wang <Ruifeng.Wang@arm.com>
> 主题: Re: [PATCH v1 3/3] examples/l3fwd-power: enable PMD power mgmt
> on Arm
>
> On Thu, 25 Aug 2022 14:42:51 +0800
> Feifei Wang <feifei.wang2@arm.com> wrote:
>
> > diff --git a/examples/l3fwd-power/main.c b/examples/l3fwd-
> power/main.c
> > index 887c6eae3f..2bd0d700f0 100644
> > --- a/examples/l3fwd-power/main.c
> > +++ b/examples/l3fwd-power/main.c
> > @@ -432,8 +432,16 @@ static void
> > signal_exit_now(int sigtype)
> > {
> >
> > - if (sigtype == SIGINT)
> > + if (sigtype == SIGINT) {
> > +#if defined(RTE_ARCH_ARM64)
> > + /**
> > + * wake_up api does not need input parameter on Arm,
> > + * so 0 is meaningless here.
> > + */
> > + rte_power_monitor_wakeup(0);
> > +#endif
> > quit_signal = true;
> > + }
> >
>
> This method is problematic. There is no guarantee that power monitor is
> async signal safe.
Agree with this. We will put 'rte_power_monitor_wakeup' out of signal_exit.
And put it after that main_lcore exit 'main_empty_poll_loop':
-----------------------------------------------------------------------------------------------------------------
rte_eal_mp_remote_launch(main_telemetry_loop, NULL, CALL_MAIN);
%wake up all worker calls from low power state.
rte_power_monitor_wakeup(0);
if (app_mode == APP_MODE_EMPTY_POLL || app_mode == APP_MODE_TELEMETRY)
launch_timer(rte_lcore_id());
RTE_LCORE_FOREACH_WORKER(lcore_id) {
if (rte_eal_wait_lcore(lcore_id) < 0)
return -1;
}
-----------------------------------------------------------------------------------------------------------------
@@ -432,8 +432,16 @@ static void
signal_exit_now(int sigtype)
{
- if (sigtype == SIGINT)
+ if (sigtype == SIGINT) {
+#if defined(RTE_ARCH_ARM64)
+ /**
+ * wake_up api does not need input parameter on Arm,
+ * so 0 is meaningless here.
+ */
+ rte_power_monitor_wakeup(0);
+#endif
quit_signal = true;
+ }
}
@@ -2885,6 +2893,25 @@ main(int argc, char **argv)
"Error setting scaling freq max: err=%d, lcore %d\n",
ret, lcore_id);
+#if defined(RTE_ARCH_ARM64)
+ /* Ensure the main lcore does not enter the power-monitor state,
+ * so that it can be used to wake up other lcores on ARM.
+ * This is due to WFE instruction has no timeout wake-up mechanism,
+ * and if users want to exit actively, the main lcore is needed
+ * to send SEV instruction to wake up other lcores.
+ */
+ unsigned int main_lcore = rte_get_main_lcore();
+ if (lcore_id != main_lcore ||
+ pmgmt_type != RTE_POWER_MGMT_TYPE_MONITOR) {
+ ret = rte_power_ethdev_pmgmt_queue_enable(
+ lcore_id, portid, queueid,
+ pmgmt_type);
+ if (ret < 0)
+ rte_exit(EXIT_FAILURE,
+ "rte_power_ethdev_pmgmt_queue_enable: err=%d, port=%d\n",
+ ret, portid);
+ }
+#else
ret = rte_power_ethdev_pmgmt_queue_enable(
lcore_id, portid, queueid,
pmgmt_type);
@@ -2892,6 +2919,7 @@ main(int argc, char **argv)
rte_exit(EXIT_FAILURE,
"rte_power_ethdev_pmgmt_queue_enable: err=%d, port=%d\n",
ret, portid);
+#endif
}
}
}