[2/4] net/mlx5: add reference counter for DV/DR structures
Checks
Commit Message
This patch introduces the reference counter for DV/DR
flow engine structure, which we are going to share
between master and representors in E-Switch configurations
over multiport Infiniband device.
Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
---
drivers/net/mlx5/mlx5.c | 26 ++++++++++++++++++++++++--
drivers/net/mlx5/mlx5.h | 4 ++++
2 files changed, 28 insertions(+), 2 deletions(-)
Comments
Tuesday, April 2, 2019 9:23 AM, Viacheslav Ovsiienko:
> Subject: [PATCH 2/4] net/mlx5: add reference counter for DV/DR structures
Same comment about the title.
>
> This patch introduces the reference counter for DV/DR flow engine
> structure, which we are going to share between master and representors in
> E-Switch configurations over multiport Infiniband device.
>
> Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
> ---
> drivers/net/mlx5/mlx5.c | 26 ++++++++++++++++++++++++--
> drivers/net/mlx5/mlx5.h | 4 ++++
> 2 files changed, 28 insertions(+), 2 deletions(-)
>
> diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index
> 9de122d..79e0c17 100644
> --- a/drivers/net/mlx5/mlx5.c
> +++ b/drivers/net/mlx5/mlx5.c
> @@ -299,7 +299,8 @@ struct mlx5_dev_spawn_data { #ifdef
> HAVE_MLX5DV_DR
> /**
> * Initialize DV/DR related data within private structure.
> - * This is preparation step for the data sharing.
> + * Routine checks the reference counter and does actual
> + * resources creation/iniialization only if counter is zero.
> *
> * @param[in] priv
> * Pointer to the private device data structure.
> @@ -314,6 +315,14 @@ struct mlx5_dev_spawn_data {
> int err = 0;
> void *ns;
>
> + assert(sh);
> + if (sh->dv_refcnt) {
> + /* Shared DV/DR structures is already initialized. */
> + sh->dv_refcnt++;
> + priv->dv_shared = 1;
> + return 0;
> + }
> + /* Reference counter is zero, we should initialize structures. */
> ns = mlx5dv_dr_create_ns(sh->ctx,
> MLX5DV_DR_NS_DOMAIN_INGRESS_BYPASS);
> if (!ns) {
> DRV_LOG(ERR, "ingress mlx5dv_dr_create_ns failed"); @@ -
> 328,6 +337,8 @@ struct mlx5_dev_spawn_data {
> goto error;
> }
> priv->tx_ns = ns;
> + sh->dv_refcnt++;
> + priv->dv_shared = 1;
> return 0;
>
> error:
> @@ -352,6 +363,16 @@ struct mlx5_dev_spawn_data { static void
> mlx5_free_shared_dv(struct mlx5_priv *priv) {
> + struct mlx5_ibv_shared *sh;
> +
> + if (!priv->dv_shared)
> + return;
> + priv->dv_shared = 0;
> + sh = priv->sh;
> + assert(sh);
> + assert(sh->dv_refcnt);
> + if (sh->dv_refcnt && --sh->dv_refcnt)
> + return;
> if (priv->rx_ns) {
> mlx5dv_dr_destroy_ns(priv->rx_ns);
> priv->rx_ns = NULL;
> @@ -1491,7 +1512,8 @@ struct mlx5_dev_spawn_data {
> error:
> if (priv) {
> #ifdef HAVE_MLX5DV_DR
> - mlx5_free_shared_dv(priv);
> + if (priv->sh)
> + mlx5_free_shared_dv(priv);
> #endif
> if (priv->nl_socket_route >= 0)
> close(priv->nl_socket_route);
> diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index
> a3d5f8e..56a2c61 100644
> --- a/drivers/net/mlx5/mlx5.h
> +++ b/drivers/net/mlx5/mlx5.h
> @@ -213,6 +213,9 @@ struct mlx5_ibv_shared {
> char ibdev_name[IBV_SYSFS_NAME_MAX]; /* IB device name. */
> char ibdev_path[IBV_SYSFS_PATH_MAX]; /* IB device path for
> secondary */
> struct ibv_device_attr_ex device_attr; /* Device properties. */
> + /* Shared DV/DR flow data section. */
> + uint32_t dv_refcnt; /* DV/DR data reference counter. */
> + /* Shared interrupt handler section. */
> pthread_mutex_t intr_mutex; /* Interrupt config mutex. */
> uint32_t intr_cnt; /* Interrupt handler reference counter. */
> struct rte_intr_handle intr_handle; /* Interrupt handler for device.
> */ @@ -244,6 +247,7 @@ struct mlx5_priv {
> unsigned int isolated:1; /* Whether isolated mode is enabled. */
> unsigned int representor:1; /* Device is a port representor. */
> unsigned int master:1; /* Device is a E-Switch master. */
> + unsigned int dv_shared:1; /* DV/DR data is shared. */
Why this flags is needed? Aren't we always going to share?
> uint16_t domain_id; /* Switch domain identifier. */
> uint16_t vport_id; /* Associated VF vport index (if any). */
> int32_t representor_id; /* Port representor identifier. */
> --
> 1.8.3.1
> -----Original Message-----
> From: Shahaf Shuler
> Sent: Tuesday, April 2, 2019 22:10
> To: Slava Ovsiienko <viacheslavo@mellanox.com>; dev@dpdk.org
> Subject: RE: [PATCH 2/4] net/mlx5: add reference counter for DV/DR
> structures
>
> Tuesday, April 2, 2019 9:23 AM, Viacheslav Ovsiienko:
> > Subject: [PATCH 2/4] net/mlx5: add reference counter for DV/DR
> > structures
>
> Same comment about the title.
>
> >
> > This patch introduces the reference counter for DV/DR flow engine
> > structure, which we are going to share between master and representors
> > in E-Switch configurations over multiport Infiniband device.
> >
> > Signed-off-by: Viacheslav Ovsiienko <viacheslavo@mellanox.com>
> > ---
> > drivers/net/mlx5/mlx5.c | 26 ++++++++++++++++++++++++--
> > drivers/net/mlx5/mlx5.h | 4 ++++
> > 2 files changed, 28 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c index
> > 9de122d..79e0c17 100644
> > --- a/drivers/net/mlx5/mlx5.c
> > +++ b/drivers/net/mlx5/mlx5.c
> > @@ -299,7 +299,8 @@ struct mlx5_dev_spawn_data { #ifdef
> > HAVE_MLX5DV_DR
> > /**
> > * Initialize DV/DR related data within private structure.
> > - * This is preparation step for the data sharing.
> > + * Routine checks the reference counter and does actual
> > + * resources creation/iniialization only if counter is zero.
> > *
> > * @param[in] priv
> > * Pointer to the private device data structure.
> > @@ -314,6 +315,14 @@ struct mlx5_dev_spawn_data {
> > int err = 0;
> > void *ns;
> >
> > + assert(sh);
> > + if (sh->dv_refcnt) {
> > + /* Shared DV/DR structures is already initialized. */
> > + sh->dv_refcnt++;
> > + priv->dv_shared = 1;
> > + return 0;
> > + }
> > + /* Reference counter is zero, we should initialize structures. */
> > ns = mlx5dv_dr_create_ns(sh->ctx,
> > MLX5DV_DR_NS_DOMAIN_INGRESS_BYPASS);
> > if (!ns) {
> > DRV_LOG(ERR, "ingress mlx5dv_dr_create_ns failed"); @@ -
> > 328,6 +337,8 @@ struct mlx5_dev_spawn_data {
> > goto error;
> > }
> > priv->tx_ns = ns;
> > + sh->dv_refcnt++;
> > + priv->dv_shared = 1;
> > return 0;
> >
> > error:
> > @@ -352,6 +363,16 @@ struct mlx5_dev_spawn_data { static void
> > mlx5_free_shared_dv(struct mlx5_priv *priv) {
> > + struct mlx5_ibv_shared *sh;
> > +
> > + if (!priv->dv_shared)
> > + return;
> > + priv->dv_shared = 0;
> > + sh = priv->sh;
> > + assert(sh);
> > + assert(sh->dv_refcnt);
> > + if (sh->dv_refcnt && --sh->dv_refcnt)
> > + return;
> > if (priv->rx_ns) {
> > mlx5dv_dr_destroy_ns(priv->rx_ns);
> > priv->rx_ns = NULL;
> > @@ -1491,7 +1512,8 @@ struct mlx5_dev_spawn_data {
> > error:
> > if (priv) {
> > #ifdef HAVE_MLX5DV_DR
> > - mlx5_free_shared_dv(priv);
> > + if (priv->sh)
> > + mlx5_free_shared_dv(priv);
> > #endif
> > if (priv->nl_socket_route >= 0)
> > close(priv->nl_socket_route);
> > diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h index
> > a3d5f8e..56a2c61 100644
> > --- a/drivers/net/mlx5/mlx5.h
> > +++ b/drivers/net/mlx5/mlx5.h
> > @@ -213,6 +213,9 @@ struct mlx5_ibv_shared {
> > char ibdev_name[IBV_SYSFS_NAME_MAX]; /* IB device name. */
> > char ibdev_path[IBV_SYSFS_PATH_MAX]; /* IB device path for
> secondary
> > */
> > struct ibv_device_attr_ex device_attr; /* Device properties. */
> > + /* Shared DV/DR flow data section. */
> > + uint32_t dv_refcnt; /* DV/DR data reference counter. */
> > + /* Shared interrupt handler section. */
> > pthread_mutex_t intr_mutex; /* Interrupt config mutex. */
> > uint32_t intr_cnt; /* Interrupt handler reference counter. */
> > struct rte_intr_handle intr_handle; /* Interrupt handler for device.
> > */ @@ -244,6 +247,7 @@ struct mlx5_priv {
> > unsigned int isolated:1; /* Whether isolated mode is enabled. */
> > unsigned int representor:1; /* Device is a port representor. */
> > unsigned int master:1; /* Device is a E-Switch master. */
> > + unsigned int dv_shared:1; /* DV/DR data is shared. */
>
> Why this flags is needed? Aren't we always going to share?
I think we can get rid of this flag, because it is used for correct cleanup
in device spawning routines If something goes wrong (on error exit)
>
>
> > uint16_t domain_id; /* Switch domain identifier. */
> > uint16_t vport_id; /* Associated VF vport index (if any). */
> > int32_t representor_id; /* Port representor identifier. */
> > --
> > 1.8.3.1
@@ -299,7 +299,8 @@ struct mlx5_dev_spawn_data {
#ifdef HAVE_MLX5DV_DR
/**
* Initialize DV/DR related data within private structure.
- * This is preparation step for the data sharing.
+ * Routine checks the reference counter and does actual
+ * resources creation/iniialization only if counter is zero.
*
* @param[in] priv
* Pointer to the private device data structure.
@@ -314,6 +315,14 @@ struct mlx5_dev_spawn_data {
int err = 0;
void *ns;
+ assert(sh);
+ if (sh->dv_refcnt) {
+ /* Shared DV/DR structures is already initialized. */
+ sh->dv_refcnt++;
+ priv->dv_shared = 1;
+ return 0;
+ }
+ /* Reference counter is zero, we should initialize structures. */
ns = mlx5dv_dr_create_ns(sh->ctx, MLX5DV_DR_NS_DOMAIN_INGRESS_BYPASS);
if (!ns) {
DRV_LOG(ERR, "ingress mlx5dv_dr_create_ns failed");
@@ -328,6 +337,8 @@ struct mlx5_dev_spawn_data {
goto error;
}
priv->tx_ns = ns;
+ sh->dv_refcnt++;
+ priv->dv_shared = 1;
return 0;
error:
@@ -352,6 +363,16 @@ struct mlx5_dev_spawn_data {
static void
mlx5_free_shared_dv(struct mlx5_priv *priv)
{
+ struct mlx5_ibv_shared *sh;
+
+ if (!priv->dv_shared)
+ return;
+ priv->dv_shared = 0;
+ sh = priv->sh;
+ assert(sh);
+ assert(sh->dv_refcnt);
+ if (sh->dv_refcnt && --sh->dv_refcnt)
+ return;
if (priv->rx_ns) {
mlx5dv_dr_destroy_ns(priv->rx_ns);
priv->rx_ns = NULL;
@@ -1491,7 +1512,8 @@ struct mlx5_dev_spawn_data {
error:
if (priv) {
#ifdef HAVE_MLX5DV_DR
- mlx5_free_shared_dv(priv);
+ if (priv->sh)
+ mlx5_free_shared_dv(priv);
#endif
if (priv->nl_socket_route >= 0)
close(priv->nl_socket_route);
@@ -213,6 +213,9 @@ struct mlx5_ibv_shared {
char ibdev_name[IBV_SYSFS_NAME_MAX]; /* IB device name. */
char ibdev_path[IBV_SYSFS_PATH_MAX]; /* IB device path for secondary */
struct ibv_device_attr_ex device_attr; /* Device properties. */
+ /* Shared DV/DR flow data section. */
+ uint32_t dv_refcnt; /* DV/DR data reference counter. */
+ /* Shared interrupt handler section. */
pthread_mutex_t intr_mutex; /* Interrupt config mutex. */
uint32_t intr_cnt; /* Interrupt handler reference counter. */
struct rte_intr_handle intr_handle; /* Interrupt handler for device. */
@@ -244,6 +247,7 @@ struct mlx5_priv {
unsigned int isolated:1; /* Whether isolated mode is enabled. */
unsigned int representor:1; /* Device is a port representor. */
unsigned int master:1; /* Device is a E-Switch master. */
+ unsigned int dv_shared:1; /* DV/DR data is shared. */
uint16_t domain_id; /* Switch domain identifier. */
uint16_t vport_id; /* Associated VF vport index (if any). */
int32_t representor_id; /* Port representor identifier. */