net/bnxt: remove EEM system memory support
diff mbox series

Message ID 20200729004256.46924-1-ajit.khaparde@broadcom.com
State Accepted, archived
Delegated to: Ajit Khaparde
Headers show
Series
  • net/bnxt: remove EEM system memory support
Related show

Checks

Context Check Description
ci/iol-mellanox-Performance success Performance Testing PASS
ci/iol-testing success Testing PASS
ci/iol-intel-Performance success Performance Testing PASS
ci/Intel-compilation success Compilation OK
ci/checkpatch success coding style OK

Commit Message

Ajit Khaparde July 29, 2020, 12:42 a.m. UTC
From: Randy Schacher <stuart.schacher@broadcom.com>

Remove the memory management scheme for Extended Exact Match
using system memory. Using host memory scheme instead which
was the default anyway.

Fixes: b2da02480cb7 ("net/bnxt: support EEM system memory")

Signed-off-by: Randy Schacher <stuart.schacher@broadcom.com>
Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
Reviewed-by: Peter Spreadborough <peter.spreadborough@broadcom.com>
Reviewed-by: Farah Smith <farah.smith@broadcom.com>
---
 config/common_base                      |   1 -
 drivers/net/bnxt/Makefile               |   3 -
 drivers/net/bnxt/bnxt_hwrm.c            |  27 --
 drivers/net/bnxt/meson.build            |   1 -
 drivers/net/bnxt/tf_core/Makefile       |   4 -
 drivers/net/bnxt/tf_core/tf_device.c    |   4 -
 drivers/net/bnxt/tf_core/tf_em.h        |  22 +-
 drivers/net/bnxt/tf_core/tf_em_common.c |  60 ---
 drivers/net/bnxt/tf_core/tf_em_system.c | 521 ------------------------
 drivers/net/bnxt/tf_core/tf_tbl.h       |   7 -
 drivers/net/bnxt/tf_core/tfp.c          |  12 -
 11 files changed, 2 insertions(+), 660 deletions(-)
 delete mode 100644 drivers/net/bnxt/tf_core/tf_em_system.c

Comments

Ajit Khaparde July 29, 2020, 2:01 p.m. UTC | #1
On Tue, Jul 28, 2020 at 5:43 PM Ajit Khaparde <ajit.khaparde@broadcom.com>
wrote:

> From: Randy Schacher <stuart.schacher@broadcom.com>
>
> Remove the memory management scheme for Extended Exact Match
> using system memory. Using host memory scheme instead which
> was the default anyway.
>
> Fixes: b2da02480cb7 ("net/bnxt: support EEM system memory")
>
> Signed-off-by: Randy Schacher <stuart.schacher@broadcom.com>
> Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
> Reviewed-by: Peter Spreadborough <peter.spreadborough@broadcom.com>
> Reviewed-by: Farah Smith <farah.smith@broadcom.com>
>
Patch applied to dpdk-next-net-brcm. Thanks


> ---
>  config/common_base                      |   1 -
>  drivers/net/bnxt/Makefile               |   3 -
>  drivers/net/bnxt/bnxt_hwrm.c            |  27 --
>  drivers/net/bnxt/meson.build            |   1 -
>  drivers/net/bnxt/tf_core/Makefile       |   4 -
>  drivers/net/bnxt/tf_core/tf_device.c    |   4 -
>  drivers/net/bnxt/tf_core/tf_em.h        |  22 +-
>  drivers/net/bnxt/tf_core/tf_em_common.c |  60 ---
>  drivers/net/bnxt/tf_core/tf_em_system.c | 521 ------------------------
>  drivers/net/bnxt/tf_core/tf_tbl.h       |   7 -
>  drivers/net/bnxt/tf_core/tfp.c          |  12 -
>  11 files changed, 2 insertions(+), 660 deletions(-)
>  delete mode 100644 drivers/net/bnxt/tf_core/tf_em_system.c
>
> diff --git a/config/common_base b/config/common_base
> index f76585f16..971351a09 100644
> --- a/config/common_base
> +++ b/config/common_base
> @@ -220,7 +220,6 @@ CONFIG_RTE_LIBRTE_BNX2X_DEBUG_PERIODIC=n
>  # Compile burst-oriented Broadcom BNXT PMD driver
>  #
>  CONFIG_RTE_LIBRTE_BNXT_PMD=y
> -CONFIG_RTE_LIBRTE_BNXT_PMD_SYSTEM=n
>
>  #
>  # Compile burst-oriented Chelsio Terminator (CXGBE) PMD
> diff --git a/drivers/net/bnxt/Makefile b/drivers/net/bnxt/Makefile
> index 2c4cdd1c4..0c5b74918 100644
> --- a/drivers/net/bnxt/Makefile
> +++ b/drivers/net/bnxt/Makefile
> @@ -52,9 +52,6 @@ CFLAGS += -I$(SRCDIR) -I$(SRCDIR)/tf_ulp
> -I$(SRCDIR)/tf_core -I$(SRCDIR)/hcapi
>  include $(SRCDIR)/tf_ulp/Makefile
>  include $(SRCDIR)/tf_core/Makefile
>  include $(SRCDIR)/hcapi/Makefile
> -ifeq ($(CONFIG_RTE_LIBRTE_BNXT_PMD_SYSTEM), y)
> -CFLAGS += -DTF_USE_SYSTEM_MEM
> -endif
>  endif
>
>  #
> diff --git a/drivers/net/bnxt/bnxt_hwrm.c b/drivers/net/bnxt/bnxt_hwrm.c
> index f5f0dfe73..8296d1d44 100644
> --- a/drivers/net/bnxt/bnxt_hwrm.c
> +++ b/drivers/net/bnxt/bnxt_hwrm.c
> @@ -5534,30 +5534,3 @@ int bnxt_hwrm_cfa_vfr_free(struct bnxt *bp,
> uint16_t vf_idx)
>         PMD_DRV_LOG(DEBUG, "VFR %d freed\n", vf_idx);
>         return rc;
>  }
> -
> -#ifdef RTE_LIBRTE_BNXT_PMD_SYSTEM
> -int
> -bnxt_hwrm_oem_cmd(struct bnxt *bp, uint32_t entry_num)
> -{
> -       struct hwrm_oem_cmd_input req = {0};
> -       struct hwrm_oem_cmd_output *resp = bp->hwrm_cmd_resp_addr;
> -       struct bnxt_dmabuf_info oem_data;
> -       int rc = 0;
> -
> -       HWRM_PREP(&req, HWRM_OEM_CMD, BNXT_USE_CHIMP_MB);
> -       req.IANA = 0x14e4;
> -
> -       memset(&oem_data, 0, sizeof(struct bnxt_dmabuf_info));
> -       oem_data.entry_num = (entry_num);
> -       memcpy(&req.oem_data[0], &oem_data, sizeof(struct
> bnxt_dmabuf_info));
> -
> -       rc = bnxt_hwrm_send_message(bp, &req, sizeof(req),
> BNXT_USE_CHIMP_MB);
> -       HWRM_CHECK_RESULT();
> -
> -       bp->dmabuf.entry_num = entry_num;
> -
> -       HWRM_UNLOCK();
> -
> -       return rc;
> -}
> -#endif /* RTE_LIBRTE_BNXT_PMD_SYSTEM */
> diff --git a/drivers/net/bnxt/meson.build b/drivers/net/bnxt/meson.build
> index 190469e29..a89d1f97c 100644
> --- a/drivers/net/bnxt/meson.build
> +++ b/drivers/net/bnxt/meson.build
> @@ -29,7 +29,6 @@ sources = files('bnxt_cpr.c',
>         'tf_core/rand.c',
>         'tf_core/stack.c',
>          'tf_core/tf_em_common.c',
> -        'tf_core/tf_em_host.c',
>          'tf_core/tf_em_internal.c',
>         'tf_core/tf_rm.c',
>         'tf_core/tf_tbl.c',
> diff --git a/drivers/net/bnxt/tf_core/Makefile
> b/drivers/net/bnxt/tf_core/Makefile
> index cf6aaec6c..84840366a 100644
> --- a/drivers/net/bnxt/tf_core/Makefile
> +++ b/drivers/net/bnxt/tf_core/Makefile
> @@ -16,11 +16,7 @@ SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_core/tf_msg.c
>  SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_core/tf_tbl.c
>  SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_core/tf_em_common.c
>  SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_core/tf_em_internal.c
> -ifeq ($(CONFIG_RTE_LIBRTE_BNXT_PMD_SYSTEM), n)
>  SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_core/tf_em_host.c
> -else
> -SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD_SYSTEM) += tf_core/tf_em_system.c
> -endif
>  SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_core/tf_session.c
>  SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_core/tf_device.c
>  SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_core/tf_device_p4.c
> diff --git a/drivers/net/bnxt/tf_core/tf_device.c
> b/drivers/net/bnxt/tf_core/tf_device.c
> index f08f7eba7..112944095 100644
> --- a/drivers/net/bnxt/tf_core/tf_device.c
> +++ b/drivers/net/bnxt/tf_core/tf_device.c
> @@ -92,11 +92,7 @@ tf_dev_bind_p4(struct tf *tfp,
>         em_cfg.num_elements = TF_EM_TBL_TYPE_MAX;
>         em_cfg.cfg = tf_em_ext_p4;
>         em_cfg.resources = resources;
> -#ifdef TF_USE_SYSTEM_MEM
> -       em_cfg.mem_type = TF_EEM_MEM_TYPE_SYSTEM;
> -#else
>         em_cfg.mem_type = TF_EEM_MEM_TYPE_HOST;
> -#endif
>         rc = tf_em_ext_common_bind(tfp, &em_cfg);
>         if (rc) {
>                 TFP_DRV_LOG(ERR,
> diff --git a/drivers/net/bnxt/tf_core/tf_em.h
> b/drivers/net/bnxt/tf_core/tf_em.h
> index ae2e64dd7..2a67e4760 100644
> --- a/drivers/net/bnxt/tf_core/tf_em.h
> +++ b/drivers/net/bnxt/tf_core/tf_em.h
> @@ -9,15 +9,6 @@
>  #include "tf_core.h"
>  #include "tf_session.h"
>
> -#ifdef TF_USE_SYSTEM_MEM
> -/**
> - * Select EEM sysmem mmap export to be done at init
> - * or on the first write to EEM.
> - */
> -#define TF_EM_SYSMEM_DELAY_EXPORT 1
> -#else
> -#define TF_EM_SYSMEM_DELAY_EXPORT 0
> -#endif
>
>  #define SUPPORT_CFA_HW_P4 1
>  #define SUPPORT_CFA_HW_P58 0
> @@ -85,13 +76,8 @@
>  /*
>   * System memory always uses 4K pages
>   */
> -#ifdef TF_USE_SYSTEM_MEM
> -#define TF_EM_PAGE_SIZE (1 << TF_EM_PAGE_SIZE_4K)
> -#define TF_EM_PAGE_ALIGNMENT (1 << TF_EM_PAGE_SIZE_4K)
> -#else
>  #define TF_EM_PAGE_SIZE        (1 << TF_EM_PAGE_SHIFT)
>  #define TF_EM_PAGE_ALIGNMENT (1 << TF_EM_PAGE_SHIFT)
> -#endif
>
>  /*
>   * Used to build GFID:
> @@ -176,13 +162,9 @@ struct tf_em_cfg_parms {
>   *
>   * @ref tf_em_ext_common_unbind
>   *
> - * @ref tf_em_ext_host_alloc
> - *
> - * @ref tf_em_ext_host_free
> - *
> - * @ref tf_em_ext_system_alloc
> + * @ref tf_em_ext_alloc
>   *
> - * @ref tf_em_ext_system_free
> + * @ref tf_em_ext_free
>   *
>   * @ref tf_em_ext_common_free
>   *
> diff --git a/drivers/net/bnxt/tf_core/tf_em_common.c
> b/drivers/net/bnxt/tf_core/tf_em_common.c
> index 0037fce42..0d8b90871 100644
> --- a/drivers/net/bnxt/tf_core/tf_em_common.c
> +++ b/drivers/net/bnxt/tf_core/tf_em_common.c
> @@ -783,34 +783,6 @@ tf_insert_eem_entry(struct tf_tbl_scope_cb
> *tbl_scope_cb,
>         uint64_t big_hash;
>         int rc;
>
> -#if (TF_EM_SYSMEM_DELAY_EXPORT == 1)
> -       if (!tbl_scope_cb->valid) {
> -               rc = offload_system_mmap(tbl_scope_cb);
> -
> -               if (rc) {
> -                       struct tf_rm_free_parms fparms = { 0 };
> -                       uint32_t rm_tbl_scope_id;
> -
> -                       TFP_DRV_LOG(ERR,
> -                                   "System alloc mmap failed\n");
> -
> -                       rm_tbl_scope_id =
> -                               tf_tbl_scope_adjust(parms->tbl_scope_id);
> -
> -                       if (rm_tbl_scope_id == TF_TBL_SCOPE_INVALID)
> -                               return -EINVAL;
> -
> -                       /* Free Table control block */
> -                       fparms.rm_db = eem_db[TF_DIR_RX];
> -                       fparms.db_index = TF_EM_TBL_TYPE_TBL_SCOPE;
> -                       fparms.index = parms->tbl_scope_id;
> -                       tf_rm_free(&fparms);
> -                       return -EINVAL;
> -               }
> -
> -               tbl_scope_cb->valid = true;
> -       }
> -#endif
>         /* Get mask to use on hash */
>         mask =
> tf_em_get_key_mask(tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY0_TABLE].num_entries);
>
> @@ -1128,38 +1100,6 @@ int tf_tbl_ext_common_set(struct tf *tfp,
>                 return -EINVAL;
>         }
>
> -#if (TF_EM_SYSMEM_DELAY_EXPORT == 1)
> -       if (!tbl_scope_cb->valid) {
> -               rc = offload_system_mmap(tbl_scope_cb);
> -
> -               if (rc) {
> -                       struct tf_rm_free_parms fparms = { 0 };
> -                       uint32_t rm_tbl_scope_id;
> -
> -                       /* TODO: support allocation of table scope from
> -                        * min in HCAPI RM.  For now call adjust function
> -                        * on value obtained from RM.
> -                        */
> -                       rm_tbl_scope_id =
> -                               tf_tbl_scope_adjust(parms->tbl_scope_id);
> -
> -                       if (rm_tbl_scope_id == TF_TBL_SCOPE_INVALID)
> -                               return -EINVAL;
> -
> -                       TFP_DRV_LOG(ERR,
> -                                   "System alloc mmap failed\n");
> -                       /* Free Table control block */
> -                       fparms.rm_db = eem_db[TF_DIR_RX];
> -                       fparms.db_index = TF_EM_TBL_TYPE_TBL_SCOPE;
> -                       fparms.index = rm_tbl_scope_id;
> -                       tf_rm_free(&fparms);
> -                       return -EINVAL;
> -               }
> -
> -               tbl_scope_cb->valid = true;
> -       }
> -#endif
> -
>         op.opcode = HCAPI_CFA_HWOPS_PUT;
>         key_tbl.base0 =
>                 (uint8_t
> *)&tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_RECORD_TABLE];
> diff --git a/drivers/net/bnxt/tf_core/tf_em_system.c
> b/drivers/net/bnxt/tf_core/tf_em_system.c
> deleted file mode 100644
> index 32ee1ee69..000000000
> --- a/drivers/net/bnxt/tf_core/tf_em_system.c
> +++ /dev/null
> @@ -1,521 +0,0 @@
> -/* SPDX-License-Identifier: BSD-3-Clause
> - * Copyright(c) 2019-2020 Broadcom
> - * All rights reserved.
> - */
> -
> -#include <string.h>
> -#include <sys/types.h>
> -#include <sys/stat.h>
> -#include <fcntl.h>
> -#include <stdbool.h>
> -#include <math.h>
> -#include <sys/param.h>
> -#include <sys/mman.h>
> -#include <sys/ioctl.h>
> -#include <unistd.h>
> -
> -#include <rte_common.h>
> -#include <rte_errno.h>
> -#include <rte_log.h>
> -
> -#include "tf_core.h"
> -#include "tf_util.h"
> -#include "tf_common.h"
> -#include "tf_em.h"
> -#include "tf_em_common.h"
> -#include "tf_msg.h"
> -#include "tfp.h"
> -#include "lookup3.h"
> -#include "tf_ext_flow_handle.h"
> -
> -#include "bnxt.h"
> -
> -enum tf_em_req_type {
> -       TF_EM_BNXT_LFC_CFA_EEM_DMABUF_EXPORT_REQ = 5,
> -};
> -
> -struct tf_em_bnxt_lfc_req_hdr {
> -       uint32_t ver;
> -       uint32_t bus;
> -       uint32_t devfn;
> -       enum tf_em_req_type req_type;
> -};
> -
> -struct tf_em_bnxt_lfc_cfa_eem_std_hdr {
> -       uint16_t version;
> -       uint16_t size;
> -       uint32_t flags;
> -       #define TF_EM_BNXT_LFC_EEM_CFG_PRIMARY_FUNC     (1 << 0)
> -};
> -
> -struct tf_em_bnxt_lfc_dmabuf_fd {
> -       int fd[TF_DIR_MAX][TF_MAX_TABLE];
> -};
> -
> -#ifndef __user
> -#define __user
> -#endif
> -
> -struct tf_em_bnxt_lfc_cfa_eem_dmabuf_export_req {
> -       struct tf_em_bnxt_lfc_cfa_eem_std_hdr std;
> -       uint8_t dir;
> -       uint32_t flags;
> -       void __user *dma_fd;
> -};
> -
> -struct tf_em_bnxt_lfc_req {
> -       struct tf_em_bnxt_lfc_req_hdr hdr;
> -       union {
> -               struct tf_em_bnxt_lfc_cfa_eem_dmabuf_export_req
> -                      eem_dmabuf_export_req;
> -               uint64_t hreq;
> -       } req;
> -};
> -
> -#define TF_EEM_BNXT_LFC_IOCTL_MAGIC     0x98
> -#define BNXT_LFC_REQ    \
> -       _IOW(TF_EEM_BNXT_LFC_IOCTL_MAGIC, 1, struct tf_em_bnxt_lfc_req)
> -
> -/**
> - * EM DBs.
> - */
> -extern void *eem_db[TF_DIR_MAX];
> -
> -extern struct tf_tbl_scope_cb tbl_scopes[TF_NUM_TBL_SCOPE];
> -
> -static void
> -tf_em_dmabuf_mem_unmap(struct hcapi_cfa_em_table *tbl)
> -{
> -       struct hcapi_cfa_em_page_tbl *tp;
> -       int level;
> -       uint32_t page_no, pg_count;
> -
> -       for (level = (tbl->num_lvl - 1); level < tbl->num_lvl; level++) {
> -               tp = &tbl->pg_tbl[level];
> -
> -               pg_count = tbl->page_cnt[level];
> -               for (page_no = 0; page_no < pg_count; page_no++) {
> -                       if (tp->pg_va_tbl != NULL &&
> -                           tp->pg_va_tbl[page_no] != NULL &&
> -                           tp->pg_size != 0) {
> -                               (void)munmap(tp->pg_va_tbl[page_no],
> -                                            tp->pg_size);
> -                       }
> -               }
> -
> -               tfp_free((void *)tp->pg_va_tbl);
> -               tfp_free((void *)tp->pg_pa_tbl);
> -       }
> -}
> -
> -/**
> - * Unregisters EM Ctx in Firmware
> - *
> - * [in] tfp
> - *   Pointer to a TruFlow handle
> - *
> - * [in] tbl_scope_cb
> - *   Pointer to a table scope control block
> - *
> - * [in] dir
> - *   Receive or transmit direction
> - */
> -static void
> -tf_em_ctx_unreg(struct tf_tbl_scope_cb *tbl_scope_cb,
> -               int dir)
> -{
> -       struct hcapi_cfa_em_ctx_mem_info *ctxp =
> -               &tbl_scope_cb->em_ctx_info[dir];
> -       struct hcapi_cfa_em_table *tbl;
> -       int i;
> -
> -       for (i = TF_KEY0_TABLE; i < TF_MAX_TABLE; i++) {
> -               tbl = &ctxp->em_tables[i];
> -                       tf_em_dmabuf_mem_unmap(tbl);
> -       }
> -}
> -
> -static int tf_export_tbl_scope(int lfc_fd,
> -                              int *fd,
> -                              int bus,
> -                              int devfn)
> -{
> -       struct tf_em_bnxt_lfc_req tf_lfc_req;
> -       struct tf_em_bnxt_lfc_dmabuf_fd *dma_fd;
> -       struct tfp_calloc_parms  mparms;
> -       int rc;
> -
> -       memset(&tf_lfc_req, 0, sizeof(struct tf_em_bnxt_lfc_req));
> -       tf_lfc_req.hdr.ver = 1;
> -       tf_lfc_req.hdr.bus = bus;
> -       tf_lfc_req.hdr.devfn = devfn;
> -       tf_lfc_req.hdr.req_type = TF_EM_BNXT_LFC_CFA_EEM_DMABUF_EXPORT_REQ;
> -       tf_lfc_req.req.eem_dmabuf_export_req.flags = O_ACCMODE;
> -       tf_lfc_req.req.eem_dmabuf_export_req.std.version = 1;
> -
> -       mparms.nitems = 1;
> -       mparms.size = sizeof(struct tf_em_bnxt_lfc_dmabuf_fd);
> -       mparms.alignment = 0;
> -       tfp_calloc(&mparms);
> -       dma_fd = (struct tf_em_bnxt_lfc_dmabuf_fd *)mparms.mem_va;
> -       tf_lfc_req.req.eem_dmabuf_export_req.dma_fd = dma_fd;
> -
> -       rc = ioctl(lfc_fd, BNXT_LFC_REQ, &tf_lfc_req);
> -       if (rc) {
> -               TFP_DRV_LOG(ERR,
> -                           "EXT EEM export chanel_fd %d, rc=%d\n",
> -                           lfc_fd,
> -                           rc);
> -               tfp_free(dma_fd);
> -               return rc;
> -       }
> -
> -       memcpy(fd, dma_fd->fd, sizeof(dma_fd->fd));
> -       tfp_free(dma_fd);
> -
> -       return rc;
> -}
> -
> -static int
> -tf_em_dmabuf_mem_map(struct hcapi_cfa_em_table *tbl,
> -                    int dmabuf_fd)
> -{
> -       struct hcapi_cfa_em_page_tbl *tp;
> -       int level;
> -       uint32_t page_no;
> -       uint32_t pg_count;
> -       uint32_t offset;
> -       struct tfp_calloc_parms parms;
> -
> -       for (level = (tbl->num_lvl - 1); level < tbl->num_lvl; level++) {
> -               tp = &tbl->pg_tbl[level];
> -
> -               pg_count = tbl->page_cnt[level];
> -               offset = 0;
> -
> -               parms.nitems = pg_count;
> -               parms.size = sizeof(void *);
> -               parms.alignment = 0;
> -
> -               if ((tfp_calloc(&parms)) != 0)
> -                       return -ENOMEM;
> -
> -               tp->pg_va_tbl = parms.mem_va;
> -               parms.nitems = pg_count;
> -               parms.size = sizeof(void *);
> -               parms.alignment = 0;
> -
> -               if ((tfp_calloc(&parms)) != 0) {
> -                       tfp_free((void *)tp->pg_va_tbl);
> -                       return -ENOMEM;
> -               }
> -
> -               tp->pg_pa_tbl = parms.mem_va;
> -               tp->pg_count = 0;
> -               tp->pg_size =  TF_EM_PAGE_SIZE;
> -
> -               for (page_no = 0; page_no < pg_count; page_no++) {
> -                       tp->pg_va_tbl[page_no] = mmap(NULL,
> -                                                     TF_EM_PAGE_SIZE,
> -                                                     PROT_READ |
> PROT_WRITE,
> -                                                     MAP_SHARED,
> -                                                     dmabuf_fd,
> -                                                     offset);
> -                       if (tp->pg_va_tbl[page_no] == (void *)-1) {
> -                               TFP_DRV_LOG(ERR,
> -               "MMap memory error. level:%d page:%d pg_count:%d - %s\n",
> -                                           level,
> -                                    page_no,
> -                                           pg_count,
> -                                           strerror(errno));
> -                               return -ENOMEM;
> -                       }
> -                       offset += tp->pg_size;
> -                       tp->pg_count++;
> -               }
> -       }
> -
> -       return 0;
> -}
> -
> -static int tf_mmap_tbl_scope(struct tf_tbl_scope_cb *tbl_scope_cb,
> -                            enum tf_dir dir,
> -                            int tbl_type,
> -                            int dmabuf_fd)
> -{
> -       struct hcapi_cfa_em_table *tbl;
> -
> -       if (tbl_type == TF_EFC_TABLE)
> -               return 0;
> -
> -       tbl = &tbl_scope_cb->em_ctx_info[dir].em_tables[tbl_type];
> -       return tf_em_dmabuf_mem_map(tbl, dmabuf_fd);
> -}
> -
> -#define TF_LFC_DEVICE "/dev/bnxt_lfc"
> -
> -static int
> -tf_prepare_dmabuf_bnxt_lfc_device(struct tf_tbl_scope_cb *tbl_scope_cb)
> -{
> -       int lfc_fd;
> -
> -       lfc_fd = open(TF_LFC_DEVICE, O_RDWR);
> -       if (!lfc_fd) {
> -               TFP_DRV_LOG(ERR,
> -                           "EEM: open %s device error\n",
> -                           TF_LFC_DEVICE);
> -               return -ENOENT;
> -       }
> -
> -       tbl_scope_cb->lfc_fd = lfc_fd;
> -
> -       return 0;
> -}
> -
> -int offload_system_mmap(struct tf_tbl_scope_cb *tbl_scope_cb)
> -{
> -       int rc;
> -       int dmabuf_fd;
> -       enum tf_dir dir;
> -       enum hcapi_cfa_em_table_type tbl_type;
> -
> -       rc = tf_prepare_dmabuf_bnxt_lfc_device(tbl_scope_cb);
> -       if (rc) {
> -               TFP_DRV_LOG(ERR, "EEM: Prepare bnxt_lfc channel failed\n");
> -               return rc;
> -       }
> -
> -       rc = tf_export_tbl_scope(tbl_scope_cb->lfc_fd,
> -                                (int *)tbl_scope_cb->fd,
> -                                tbl_scope_cb->bus,
> -                                tbl_scope_cb->devfn);
> -       if (rc) {
> -               TFP_DRV_LOG(ERR,
> -                           "export dmabuf fd failed\n");
> -               return rc;
> -       }
> -
> -       tbl_scope_cb->valid = true;
> -
> -       for (dir = 0; dir < TF_DIR_MAX; dir++) {
> -               for (tbl_type = TF_KEY0_TABLE; tbl_type <
> -                            TF_MAX_TABLE; tbl_type++) {
> -                       if (tbl_type == TF_EFC_TABLE)
> -                               continue;
> -
> -                       dmabuf_fd = tbl_scope_cb->fd[(dir ? 0 :
> 1)][tbl_type];
> -                       rc = tf_mmap_tbl_scope(tbl_scope_cb,
> -                                              dir,
> -                                              tbl_type,
> -                                              dmabuf_fd);
> -                       if (rc) {
> -                               TFP_DRV_LOG(ERR,
> -                                           "dir:%d tbl:%d mmap failed rc
> %d\n",
> -                                           dir,
> -                                           tbl_type,
> -                                           rc);
> -                               break;
> -                       }
> -               }
> -       }
> -       return 0;
> -}
> -
> -static int
> -tf_destroy_dmabuf_bnxt_lfc_device(struct tf_tbl_scope_cb *tbl_scope_cb)
> -{
> -       close(tbl_scope_cb->lfc_fd);
> -
> -       return 0;
> -}
> -
> -static int
> -tf_dmabuf_alloc(struct tf *tfp, struct tf_tbl_scope_cb *tbl_scope_cb)
> -{
> -       int rc;
> -
> -       rc = tfp_msg_hwrm_oem_cmd(tfp,
> -
>  tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY0_TABLE].num_entries);
> -       if (rc)
> -               PMD_DRV_LOG(ERR, "EEM: Failed to prepare system memory
> rc:%d\n",
> -                           rc);
> -
> -       return 0;
> -}
> -
> -static int
> -tf_dmabuf_free(struct tf *tfp, struct tf_tbl_scope_cb *tbl_scope_cb)
> -{
> -       int rc;
> -
> -       rc = tfp_msg_hwrm_oem_cmd(tfp, 0);
> -       if (rc)
> -               TFP_DRV_LOG(ERR, "EEM: Failed to cleanup system memory\n");
> -
> -       tf_destroy_dmabuf_bnxt_lfc_device(tbl_scope_cb);
> -
> -       return 0;
> -}
> -
> -int
> -tf_em_ext_alloc(struct tf *tfp,
> -               struct tf_alloc_tbl_scope_parms *parms)
> -{
> -       int rc;
> -       struct tf_session *tfs;
> -       struct tf_tbl_scope_cb *tbl_scope_cb;
> -       struct tf_free_tbl_scope_parms free_parms;
> -       int dir;
> -       int i;
> -       struct hcapi_cfa_em_table *em_tables;
> -
> -       TF_CHECK_PARMS2(tfp, parms);
> -
> -       /* Retrieve the session information */
> -       rc = tf_session_get_session(tfp, &tfs);
> -       if (rc) {
> -               TFP_DRV_LOG(ERR,
> -                           "Failed to lookup session, rc:%s\n",
> -                           strerror(-rc));
> -               return rc;
> -       }
> -
> -       rc = tf_tbl_scope_alloc(&parms->tbl_scope_id);
> -       if (rc) {
> -               TFP_DRV_LOG(ERR,
> -                           "Failed to allocate table scope\n");
> -               return rc;
> -       }
> -
> -       tbl_scope_cb = &tbl_scopes[parms->tbl_scope_id];
> -       tbl_scope_cb->index = parms->tbl_scope_id;
> -       tbl_scope_cb->tbl_scope_id = parms->tbl_scope_id;
> -       tbl_scope_cb->bus = tfs->session_id.internal.bus;
> -       tbl_scope_cb->devfn = tfs->session_id.internal.device;
> -
> -       for (dir = 0; dir < TF_DIR_MAX; dir++) {
> -               rc = tf_msg_em_qcaps(tfp,
> -                                    dir,
> -                                    &tbl_scope_cb->em_caps[dir]);
> -               if (rc) {
> -                       TFP_DRV_LOG(ERR,
> -                                   "EEM: Unable to query for EEM
> capability,"
> -                                   " rc:%s\n",
> -                                   strerror(-rc));
> -                       goto cleanup;
> -               }
> -       }
> -
> -       /*
> -        * Validate and setup table sizes
> -        */
> -       if (tf_em_validate_num_entries(tbl_scope_cb, parms))
> -               goto cleanup;
> -
> -       rc = tf_dmabuf_alloc(tfp, tbl_scope_cb);
> -       if (rc) {
> -               TFP_DRV_LOG(ERR,
> -                           "System DMA buff alloc failed\n");
> -               return -EIO;
> -       }
> -
> -       for (dir = 0; dir < TF_DIR_MAX; dir++) {
> -               for (i = TF_KEY0_TABLE; i < TF_MAX_TABLE; i++) {
> -                       if (i == TF_EFC_TABLE)
> -                               continue;
> -
> -                       em_tables =
> -
>  &tbl_scope_cb->em_ctx_info[dir].em_tables[i];
> -
> -                       rc = tf_em_size_table(em_tables, TF_EM_PAGE_SIZE);
> -                       if (rc) {
> -                               TFP_DRV_LOG(ERR, "Size table failed\n");
> -                               goto cleanup;
> -                       }
> -               }
> -
> -               em_tables = tbl_scope_cb->em_ctx_info[dir].em_tables;
> -               rc = tf_create_tbl_pool_external(dir,
> -                                       tbl_scope_cb,
> -
>  em_tables[TF_RECORD_TABLE].num_entries,
> -
>  em_tables[TF_RECORD_TABLE].entry_size);
> -
> -               if (rc) {
> -                       TFP_DRV_LOG(ERR,
> -                                   "%s TBL: Unable to allocate idx pools
> %s\n",
> -                                   tf_dir_2_str(dir),
> -                                   strerror(-rc));
> -                       goto cleanup_full;
> -               }
> -       }
> -
> -#if (TF_EM_SYSMEM_DELAY_EXPORT == 0)
> -       rc = offload_system_mmap(tbl_scope_cb);
> -
> -       if (rc) {
> -               TFP_DRV_LOG(ERR,
> -                           "System alloc mmap failed\n");
> -               goto cleanup_full;
> -       }
> -#endif
> -
> -       return rc;
> -
> -cleanup_full:
> -       free_parms.tbl_scope_id = parms->tbl_scope_id;
> -       tf_em_ext_free(tfp, &free_parms);
> -       return -EINVAL;
> -
> -cleanup:
> -       /* Free Table control block */
> -       tf_tbl_scope_free(parms->tbl_scope_id);
> -       return -EINVAL;
> -}
> -
> -int
> -tf_em_ext_free(struct tf *tfp,
> -              struct tf_free_tbl_scope_parms *parms)
> -{
> -       int rc;
> -       struct tf_session *tfs;
> -       struct tf_tbl_scope_cb *tbl_scope_cb;
> -       int dir;
> -
> -       TF_CHECK_PARMS2(tfp, parms);
> -
> -       /* Retrieve the session information */
> -       rc = tf_session_get_session(tfp, &tfs);
> -       if (rc) {
> -               TFP_DRV_LOG(ERR,
> -                           "Failed to lookup session, rc:%s\n",
> -                           strerror(-rc));
> -               return rc;
> -       }
> -
> -       tbl_scope_cb = &tbl_scopes[parms->tbl_scope_id];
> -
> -       rc = tf_tbl_scope_free(parms->tbl_scope_id);
> -       if (rc) {
> -               TFP_DRV_LOG(ERR,
> -                           "Failed to free table scope\n");
> -       }
> -
> -       for (dir = 0; dir < TF_DIR_MAX; dir++) {
> -               /* Free associated external pools
> -                */
> -               tf_destroy_tbl_pool_external(dir,
> -                                            tbl_scope_cb);
> -
> -               /* Unmap memory */
> -               tf_em_ctx_unreg(tbl_scope_cb, dir);
> -
> -               tf_msg_em_op(tfp,
> -                            dir,
> -                            HWRM_TF_EXT_EM_OP_INPUT_OP_EXT_EM_DISABLE);
> -       }
> -
> -       tf_dmabuf_free(tfp, tbl_scope_cb);
> -       tbl_scope_cb->valid = false;
> -
> -       return rc;
> -}
> diff --git a/drivers/net/bnxt/tf_core/tf_tbl.h
> b/drivers/net/bnxt/tf_core/tf_tbl.h
> index 930fcc324..2a5d24c94 100644
> --- a/drivers/net/bnxt/tf_core/tf_tbl.h
> +++ b/drivers/net/bnxt/tf_core/tf_tbl.h
> @@ -38,13 +38,6 @@ struct tf_em_caps {
>   */
>  struct tf_tbl_scope_cb {
>         uint32_t tbl_scope_id;
> -#ifdef TF_USE_SYSTEM_MEM
> -       int lfc_fd;
> -       uint32_t bus;
> -       uint32_t devfn;
> -       int fd[TF_DIR_MAX][TF_MAX_TABLE];
> -       bool valid;
> -#endif
>         int index;
>         struct hcapi_cfa_em_ctx_mem_info em_ctx_info[TF_DIR_MAX];
>         struct tf_em_caps em_caps[TF_DIR_MAX];
> diff --git a/drivers/net/bnxt/tf_core/tfp.c
> b/drivers/net/bnxt/tf_core/tfp.c
> index 3eade3127..426a182a9 100644
> --- a/drivers/net/bnxt/tf_core/tfp.c
> +++ b/drivers/net/bnxt/tf_core/tfp.c
> @@ -87,18 +87,6 @@ tfp_send_msg_tunneled(struct tf *tfp,
>         return rc;
>  }
>
> -#ifdef TF_USE_SYSTEM_MEM
> -int
> -tfp_msg_hwrm_oem_cmd(struct tf *tfp,
> -                    uint32_t max_flows)
> -{
> -       return bnxt_hwrm_oem_cmd(container_of(tfp,
> -                                             struct bnxt,
> -                                             tfp),
> -                                max_flows);
> -}
> -#endif /* TF_USE_SYSTEM_MEM */
> -
>  /**
>   * Allocates zero'ed memory from the heap.
>   *
> --
> 2.21.1 (Apple Git-122.3)
>
>
Ferruh Yigit July 29, 2020, 4:24 p.m. UTC | #2
On 7/29/2020 1:42 AM, Ajit Khaparde wrote:
> From: Randy Schacher <stuart.schacher@broadcom.com>
> 
> Remove the memory management scheme for Extended Exact Match
> using system memory. Using host memory scheme instead which
> was the default anyway.
> 
> Fixes: b2da02480cb7 ("net/bnxt: support EEM system memory")
> 
> Signed-off-by: Randy Schacher <stuart.schacher@broadcom.com>
> Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
> Reviewed-by: Peter Spreadborough <peter.spreadborough@broadcom.com>
> Reviewed-by: Farah Smith <farah.smith@broadcom.com>

<...>

> @@ -29,7 +29,6 @@ sources = files('bnxt_cpr.c',
>  	'tf_core/rand.c',
>  	'tf_core/stack.c',
>          'tf_core/tf_em_common.c',
> -        'tf_core/tf_em_host.c',
>          'tf_core/tf_em_internal.c',
>  	'tf_core/tf_rm.c',
>  	'tf_core/tf_tbl.c',

I was wondering why 'tf_em_host.c' removed from meson, while 'tf_em_system.c'
file is removed and how this is working,
but it seems 'tf_em_host.c' was duplicate in meson and you are fixing it J

Since duplication is created in this release, no need to backport it, and it is
OK have it part of this patch.
Ajit Khaparde July 29, 2020, 9:24 p.m. UTC | #3
On Wed, Jul 29, 2020 at 9:25 AM Ferruh Yigit <ferruh.yigit@intel.com> wrote:

> On 7/29/2020 1:42 AM, Ajit Khaparde wrote:
> > From: Randy Schacher <stuart.schacher@broadcom.com>
> >
> > Remove the memory management scheme for Extended Exact Match
> > using system memory. Using host memory scheme instead which
> > was the default anyway.
> >
> > Fixes: b2da02480cb7 ("net/bnxt: support EEM system memory")
> >
> > Signed-off-by: Randy Schacher <stuart.schacher@broadcom.com>
> > Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com>
> > Reviewed-by: Peter Spreadborough <peter.spreadborough@broadcom.com>
> > Reviewed-by: Farah Smith <farah.smith@broadcom.com>
>
> <...>
>
> > @@ -29,7 +29,6 @@ sources = files('bnxt_cpr.c',
> >       'tf_core/rand.c',
> >       'tf_core/stack.c',
> >          'tf_core/tf_em_common.c',
> > -        'tf_core/tf_em_host.c',
> >          'tf_core/tf_em_internal.c',
> >       'tf_core/tf_rm.c',
> >       'tf_core/tf_tbl.c',
>
> I was wondering why 'tf_em_host.c' removed from meson, while
> 'tf_em_system.c'
> file is removed and how this is working,
> but it seems 'tf_em_host.c' was duplicate in meson and you are fixing it J
>
> Since duplication is created in this release, no need to backport it, and
> it is
> OK have it part of this patch.
>
Ferruh, Do you want me to respin the patch? Thanks
Ferruh Yigit July 29, 2020, 9:57 p.m. UTC | #4
On 7/29/2020 10:24 PM, Ajit Khaparde wrote:
> 
> 
> On Wed, Jul 29, 2020 at 9:25 AM Ferruh Yigit <ferruh.yigit@intel.com
> <mailto:ferruh.yigit@intel.com>> wrote:
> 
>     On 7/29/2020 1:42 AM, Ajit Khaparde wrote:
>     > From: Randy Schacher <stuart.schacher@broadcom.com
>     <mailto:stuart.schacher@broadcom.com>>
>     >
>     > Remove the memory management scheme for Extended Exact Match
>     > using system memory. Using host memory scheme instead which
>     > was the default anyway.
>     >
>     > Fixes: b2da02480cb7 ("net/bnxt: support EEM system memory")
>     >
>     > Signed-off-by: Randy Schacher <stuart.schacher@broadcom.com
>     <mailto:stuart.schacher@broadcom.com>>
>     > Signed-off-by: Ajit Khaparde <ajit.khaparde@broadcom.com
>     <mailto:ajit.khaparde@broadcom.com>>
>     > Reviewed-by: Peter Spreadborough <peter.spreadborough@broadcom.com
>     <mailto:peter.spreadborough@broadcom.com>>
>     > Reviewed-by: Farah Smith <farah.smith@broadcom.com
>     <mailto:farah.smith@broadcom.com>>
> 
>     <...>
> 
>     > @@ -29,7 +29,6 @@ sources = files('bnxt_cpr.c',
>     >       'tf_core/rand.c',
>     >       'tf_core/stack.c',
>     >          'tf_core/tf_em_common.c',
>     > -        'tf_core/tf_em_host.c',
>     >          'tf_core/tf_em_internal.c',
>     >       'tf_core/tf_rm.c',
>     >       'tf_core/tf_tbl.c',
> 
>     I was wondering why 'tf_em_host.c' removed from meson, while 'tf_em_system.c'
>     file is removed and how this is working,
>     but it seems 'tf_em_host.c' was duplicate in meson and you are fixing it J
> 
>     Since duplication is created in this release, no need to backport it, and it is
>     OK have it part of this patch.
> 
> Ferruh, Do you want me to respin the patch? Thanks

I think no needed, thanks, already pulled to next-net as it is.

Patch
diff mbox series

diff --git a/config/common_base b/config/common_base
index f76585f16..971351a09 100644
--- a/config/common_base
+++ b/config/common_base
@@ -220,7 +220,6 @@  CONFIG_RTE_LIBRTE_BNX2X_DEBUG_PERIODIC=n
 # Compile burst-oriented Broadcom BNXT PMD driver
 #
 CONFIG_RTE_LIBRTE_BNXT_PMD=y
-CONFIG_RTE_LIBRTE_BNXT_PMD_SYSTEM=n
 
 #
 # Compile burst-oriented Chelsio Terminator (CXGBE) PMD
diff --git a/drivers/net/bnxt/Makefile b/drivers/net/bnxt/Makefile
index 2c4cdd1c4..0c5b74918 100644
--- a/drivers/net/bnxt/Makefile
+++ b/drivers/net/bnxt/Makefile
@@ -52,9 +52,6 @@  CFLAGS += -I$(SRCDIR) -I$(SRCDIR)/tf_ulp -I$(SRCDIR)/tf_core -I$(SRCDIR)/hcapi
 include $(SRCDIR)/tf_ulp/Makefile
 include $(SRCDIR)/tf_core/Makefile
 include $(SRCDIR)/hcapi/Makefile
-ifeq ($(CONFIG_RTE_LIBRTE_BNXT_PMD_SYSTEM), y)
-CFLAGS += -DTF_USE_SYSTEM_MEM
-endif
 endif
 
 #
diff --git a/drivers/net/bnxt/bnxt_hwrm.c b/drivers/net/bnxt/bnxt_hwrm.c
index f5f0dfe73..8296d1d44 100644
--- a/drivers/net/bnxt/bnxt_hwrm.c
+++ b/drivers/net/bnxt/bnxt_hwrm.c
@@ -5534,30 +5534,3 @@  int bnxt_hwrm_cfa_vfr_free(struct bnxt *bp, uint16_t vf_idx)
 	PMD_DRV_LOG(DEBUG, "VFR %d freed\n", vf_idx);
 	return rc;
 }
-
-#ifdef RTE_LIBRTE_BNXT_PMD_SYSTEM
-int
-bnxt_hwrm_oem_cmd(struct bnxt *bp, uint32_t entry_num)
-{
-	struct hwrm_oem_cmd_input req = {0};
-	struct hwrm_oem_cmd_output *resp = bp->hwrm_cmd_resp_addr;
-	struct bnxt_dmabuf_info oem_data;
-	int rc = 0;
-
-	HWRM_PREP(&req, HWRM_OEM_CMD, BNXT_USE_CHIMP_MB);
-	req.IANA = 0x14e4;
-
-	memset(&oem_data, 0, sizeof(struct bnxt_dmabuf_info));
-	oem_data.entry_num = (entry_num);
-	memcpy(&req.oem_data[0], &oem_data, sizeof(struct bnxt_dmabuf_info));
-
-	rc = bnxt_hwrm_send_message(bp, &req, sizeof(req), BNXT_USE_CHIMP_MB);
-	HWRM_CHECK_RESULT();
-
-	bp->dmabuf.entry_num = entry_num;
-
-	HWRM_UNLOCK();
-
-	return rc;
-}
-#endif /* RTE_LIBRTE_BNXT_PMD_SYSTEM */
diff --git a/drivers/net/bnxt/meson.build b/drivers/net/bnxt/meson.build
index 190469e29..a89d1f97c 100644
--- a/drivers/net/bnxt/meson.build
+++ b/drivers/net/bnxt/meson.build
@@ -29,7 +29,6 @@  sources = files('bnxt_cpr.c',
 	'tf_core/rand.c',
 	'tf_core/stack.c',
         'tf_core/tf_em_common.c',
-        'tf_core/tf_em_host.c',
         'tf_core/tf_em_internal.c',
 	'tf_core/tf_rm.c',
 	'tf_core/tf_tbl.c',
diff --git a/drivers/net/bnxt/tf_core/Makefile b/drivers/net/bnxt/tf_core/Makefile
index cf6aaec6c..84840366a 100644
--- a/drivers/net/bnxt/tf_core/Makefile
+++ b/drivers/net/bnxt/tf_core/Makefile
@@ -16,11 +16,7 @@  SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_core/tf_msg.c
 SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_core/tf_tbl.c
 SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_core/tf_em_common.c
 SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_core/tf_em_internal.c
-ifeq ($(CONFIG_RTE_LIBRTE_BNXT_PMD_SYSTEM), n)
 SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_core/tf_em_host.c
-else
-SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD_SYSTEM) += tf_core/tf_em_system.c
-endif
 SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_core/tf_session.c
 SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_core/tf_device.c
 SRCS-$(CONFIG_RTE_LIBRTE_BNXT_PMD) += tf_core/tf_device_p4.c
diff --git a/drivers/net/bnxt/tf_core/tf_device.c b/drivers/net/bnxt/tf_core/tf_device.c
index f08f7eba7..112944095 100644
--- a/drivers/net/bnxt/tf_core/tf_device.c
+++ b/drivers/net/bnxt/tf_core/tf_device.c
@@ -92,11 +92,7 @@  tf_dev_bind_p4(struct tf *tfp,
 	em_cfg.num_elements = TF_EM_TBL_TYPE_MAX;
 	em_cfg.cfg = tf_em_ext_p4;
 	em_cfg.resources = resources;
-#ifdef TF_USE_SYSTEM_MEM
-	em_cfg.mem_type = TF_EEM_MEM_TYPE_SYSTEM;
-#else
 	em_cfg.mem_type = TF_EEM_MEM_TYPE_HOST;
-#endif
 	rc = tf_em_ext_common_bind(tfp, &em_cfg);
 	if (rc) {
 		TFP_DRV_LOG(ERR,
diff --git a/drivers/net/bnxt/tf_core/tf_em.h b/drivers/net/bnxt/tf_core/tf_em.h
index ae2e64dd7..2a67e4760 100644
--- a/drivers/net/bnxt/tf_core/tf_em.h
+++ b/drivers/net/bnxt/tf_core/tf_em.h
@@ -9,15 +9,6 @@ 
 #include "tf_core.h"
 #include "tf_session.h"
 
-#ifdef TF_USE_SYSTEM_MEM
-/**
- * Select EEM sysmem mmap export to be done at init
- * or on the first write to EEM.
- */
-#define TF_EM_SYSMEM_DELAY_EXPORT 1
-#else
-#define TF_EM_SYSMEM_DELAY_EXPORT 0
-#endif
 
 #define SUPPORT_CFA_HW_P4 1
 #define SUPPORT_CFA_HW_P58 0
@@ -85,13 +76,8 @@ 
 /*
  * System memory always uses 4K pages
  */
-#ifdef TF_USE_SYSTEM_MEM
-#define TF_EM_PAGE_SIZE (1 << TF_EM_PAGE_SIZE_4K)
-#define TF_EM_PAGE_ALIGNMENT (1 << TF_EM_PAGE_SIZE_4K)
-#else
 #define TF_EM_PAGE_SIZE	(1 << TF_EM_PAGE_SHIFT)
 #define TF_EM_PAGE_ALIGNMENT (1 << TF_EM_PAGE_SHIFT)
-#endif
 
 /*
  * Used to build GFID:
@@ -176,13 +162,9 @@  struct tf_em_cfg_parms {
  *
  * @ref tf_em_ext_common_unbind
  *
- * @ref tf_em_ext_host_alloc
- *
- * @ref tf_em_ext_host_free
- *
- * @ref tf_em_ext_system_alloc
+ * @ref tf_em_ext_alloc
  *
- * @ref tf_em_ext_system_free
+ * @ref tf_em_ext_free
  *
  * @ref tf_em_ext_common_free
  *
diff --git a/drivers/net/bnxt/tf_core/tf_em_common.c b/drivers/net/bnxt/tf_core/tf_em_common.c
index 0037fce42..0d8b90871 100644
--- a/drivers/net/bnxt/tf_core/tf_em_common.c
+++ b/drivers/net/bnxt/tf_core/tf_em_common.c
@@ -783,34 +783,6 @@  tf_insert_eem_entry(struct tf_tbl_scope_cb *tbl_scope_cb,
 	uint64_t big_hash;
 	int rc;
 
-#if (TF_EM_SYSMEM_DELAY_EXPORT == 1)
-	if (!tbl_scope_cb->valid) {
-		rc = offload_system_mmap(tbl_scope_cb);
-
-		if (rc) {
-			struct tf_rm_free_parms fparms = { 0 };
-			uint32_t rm_tbl_scope_id;
-
-			TFP_DRV_LOG(ERR,
-				    "System alloc mmap failed\n");
-
-			rm_tbl_scope_id =
-				tf_tbl_scope_adjust(parms->tbl_scope_id);
-
-			if (rm_tbl_scope_id == TF_TBL_SCOPE_INVALID)
-				return -EINVAL;
-
-			/* Free Table control block */
-			fparms.rm_db = eem_db[TF_DIR_RX];
-			fparms.db_index = TF_EM_TBL_TYPE_TBL_SCOPE;
-			fparms.index = parms->tbl_scope_id;
-			tf_rm_free(&fparms);
-			return -EINVAL;
-		}
-
-		tbl_scope_cb->valid = true;
-	}
-#endif
 	/* Get mask to use on hash */
 	mask = tf_em_get_key_mask(tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_KEY0_TABLE].num_entries);
 
@@ -1128,38 +1100,6 @@  int tf_tbl_ext_common_set(struct tf *tfp,
 		return -EINVAL;
 	}
 
-#if (TF_EM_SYSMEM_DELAY_EXPORT == 1)
-	if (!tbl_scope_cb->valid) {
-		rc = offload_system_mmap(tbl_scope_cb);
-
-		if (rc) {
-			struct tf_rm_free_parms fparms = { 0 };
-			uint32_t rm_tbl_scope_id;
-
-			/* TODO: support allocation of table scope from
-			 * min in HCAPI RM.  For now call adjust function
-			 * on value obtained from RM.
-			 */
-			rm_tbl_scope_id =
-				tf_tbl_scope_adjust(parms->tbl_scope_id);
-
-			if (rm_tbl_scope_id == TF_TBL_SCOPE_INVALID)
-				return -EINVAL;
-
-			TFP_DRV_LOG(ERR,
-				    "System alloc mmap failed\n");
-			/* Free Table control block */
-			fparms.rm_db = eem_db[TF_DIR_RX];
-			fparms.db_index = TF_EM_TBL_TYPE_TBL_SCOPE;
-			fparms.index = rm_tbl_scope_id;
-			tf_rm_free(&fparms);
-			return -EINVAL;
-		}
-
-		tbl_scope_cb->valid = true;
-	}
-#endif
-
 	op.opcode = HCAPI_CFA_HWOPS_PUT;
 	key_tbl.base0 =
 		(uint8_t *)&tbl_scope_cb->em_ctx_info[parms->dir].em_tables[TF_RECORD_TABLE];
diff --git a/drivers/net/bnxt/tf_core/tf_em_system.c b/drivers/net/bnxt/tf_core/tf_em_system.c
deleted file mode 100644
index 32ee1ee69..000000000
--- a/drivers/net/bnxt/tf_core/tf_em_system.c
+++ /dev/null
@@ -1,521 +0,0 @@ 
-/* SPDX-License-Identifier: BSD-3-Clause
- * Copyright(c) 2019-2020 Broadcom
- * All rights reserved.
- */
-
-#include <string.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <stdbool.h>
-#include <math.h>
-#include <sys/param.h>
-#include <sys/mman.h>
-#include <sys/ioctl.h>
-#include <unistd.h>
-
-#include <rte_common.h>
-#include <rte_errno.h>
-#include <rte_log.h>
-
-#include "tf_core.h"
-#include "tf_util.h"
-#include "tf_common.h"
-#include "tf_em.h"
-#include "tf_em_common.h"
-#include "tf_msg.h"
-#include "tfp.h"
-#include "lookup3.h"
-#include "tf_ext_flow_handle.h"
-
-#include "bnxt.h"
-
-enum tf_em_req_type {
-	TF_EM_BNXT_LFC_CFA_EEM_DMABUF_EXPORT_REQ = 5,
-};
-
-struct tf_em_bnxt_lfc_req_hdr {
-	uint32_t ver;
-	uint32_t bus;
-	uint32_t devfn;
-	enum tf_em_req_type req_type;
-};
-
-struct tf_em_bnxt_lfc_cfa_eem_std_hdr {
-	uint16_t version;
-	uint16_t size;
-	uint32_t flags;
-	#define TF_EM_BNXT_LFC_EEM_CFG_PRIMARY_FUNC     (1 << 0)
-};
-
-struct tf_em_bnxt_lfc_dmabuf_fd {
-	int fd[TF_DIR_MAX][TF_MAX_TABLE];
-};
-
-#ifndef __user
-#define __user
-#endif
-
-struct tf_em_bnxt_lfc_cfa_eem_dmabuf_export_req {
-	struct tf_em_bnxt_lfc_cfa_eem_std_hdr std;
-	uint8_t dir;
-	uint32_t flags;
-	void __user *dma_fd;
-};
-
-struct tf_em_bnxt_lfc_req {
-	struct tf_em_bnxt_lfc_req_hdr hdr;
-	union {
-		struct tf_em_bnxt_lfc_cfa_eem_dmabuf_export_req
-		       eem_dmabuf_export_req;
-		uint64_t hreq;
-	} req;
-};
-
-#define TF_EEM_BNXT_LFC_IOCTL_MAGIC     0x98
-#define BNXT_LFC_REQ    \
-	_IOW(TF_EEM_BNXT_LFC_IOCTL_MAGIC, 1, struct tf_em_bnxt_lfc_req)
-
-/**
- * EM DBs.
- */
-extern void *eem_db[TF_DIR_MAX];
-
-extern struct tf_tbl_scope_cb tbl_scopes[TF_NUM_TBL_SCOPE];
-
-static void
-tf_em_dmabuf_mem_unmap(struct hcapi_cfa_em_table *tbl)
-{
-	struct hcapi_cfa_em_page_tbl *tp;
-	int level;
-	uint32_t page_no, pg_count;
-
-	for (level = (tbl->num_lvl - 1); level < tbl->num_lvl; level++) {
-		tp = &tbl->pg_tbl[level];
-
-		pg_count = tbl->page_cnt[level];
-		for (page_no = 0; page_no < pg_count; page_no++) {
-			if (tp->pg_va_tbl != NULL &&
-			    tp->pg_va_tbl[page_no] != NULL &&
-			    tp->pg_size != 0) {
-				(void)munmap(tp->pg_va_tbl[page_no],
-					     tp->pg_size);
-			}
-		}
-
-		tfp_free((void *)tp->pg_va_tbl);
-		tfp_free((void *)tp->pg_pa_tbl);
-	}
-}
-
-/**
- * Unregisters EM Ctx in Firmware
- *
- * [in] tfp
- *   Pointer to a TruFlow handle
- *
- * [in] tbl_scope_cb
- *   Pointer to a table scope control block
- *
- * [in] dir
- *   Receive or transmit direction
- */
-static void
-tf_em_ctx_unreg(struct tf_tbl_scope_cb *tbl_scope_cb,
-		int dir)
-{
-	struct hcapi_cfa_em_ctx_mem_info *ctxp =
-		&tbl_scope_cb->em_ctx_info[dir];
-	struct hcapi_cfa_em_table *tbl;
-	int i;
-
-	for (i = TF_KEY0_TABLE; i < TF_MAX_TABLE; i++) {
-		tbl = &ctxp->em_tables[i];
-			tf_em_dmabuf_mem_unmap(tbl);
-	}
-}
-
-static int tf_export_tbl_scope(int lfc_fd,
-			       int *fd,
-			       int bus,
-			       int devfn)
-{
-	struct tf_em_bnxt_lfc_req tf_lfc_req;
-	struct tf_em_bnxt_lfc_dmabuf_fd *dma_fd;
-	struct tfp_calloc_parms  mparms;
-	int rc;
-
-	memset(&tf_lfc_req, 0, sizeof(struct tf_em_bnxt_lfc_req));
-	tf_lfc_req.hdr.ver = 1;
-	tf_lfc_req.hdr.bus = bus;
-	tf_lfc_req.hdr.devfn = devfn;
-	tf_lfc_req.hdr.req_type = TF_EM_BNXT_LFC_CFA_EEM_DMABUF_EXPORT_REQ;
-	tf_lfc_req.req.eem_dmabuf_export_req.flags = O_ACCMODE;
-	tf_lfc_req.req.eem_dmabuf_export_req.std.version = 1;
-
-	mparms.nitems = 1;
-	mparms.size = sizeof(struct tf_em_bnxt_lfc_dmabuf_fd);
-	mparms.alignment = 0;
-	tfp_calloc(&mparms);
-	dma_fd = (struct tf_em_bnxt_lfc_dmabuf_fd *)mparms.mem_va;
-	tf_lfc_req.req.eem_dmabuf_export_req.dma_fd = dma_fd;
-
-	rc = ioctl(lfc_fd, BNXT_LFC_REQ, &tf_lfc_req);
-	if (rc) {
-		TFP_DRV_LOG(ERR,
-			    "EXT EEM export chanel_fd %d, rc=%d\n",
-			    lfc_fd,
-			    rc);
-		tfp_free(dma_fd);
-		return rc;
-	}
-
-	memcpy(fd, dma_fd->fd, sizeof(dma_fd->fd));
-	tfp_free(dma_fd);
-
-	return rc;
-}
-
-static int
-tf_em_dmabuf_mem_map(struct hcapi_cfa_em_table *tbl,
-		     int dmabuf_fd)
-{
-	struct hcapi_cfa_em_page_tbl *tp;
-	int level;
-	uint32_t page_no;
-	uint32_t pg_count;
-	uint32_t offset;
-	struct tfp_calloc_parms parms;
-
-	for (level = (tbl->num_lvl - 1); level < tbl->num_lvl; level++) {
-		tp = &tbl->pg_tbl[level];
-
-		pg_count = tbl->page_cnt[level];
-		offset = 0;
-
-		parms.nitems = pg_count;
-		parms.size = sizeof(void *);
-		parms.alignment = 0;
-
-		if ((tfp_calloc(&parms)) != 0)
-			return -ENOMEM;
-
-		tp->pg_va_tbl = parms.mem_va;
-		parms.nitems = pg_count;
-		parms.size = sizeof(void *);
-		parms.alignment = 0;
-
-		if ((tfp_calloc(&parms)) != 0) {
-			tfp_free((void *)tp->pg_va_tbl);
-			return -ENOMEM;
-		}
-
-		tp->pg_pa_tbl = parms.mem_va;
-		tp->pg_count = 0;
-		tp->pg_size =  TF_EM_PAGE_SIZE;
-
-		for (page_no = 0; page_no < pg_count; page_no++) {
-			tp->pg_va_tbl[page_no] = mmap(NULL,
-						      TF_EM_PAGE_SIZE,
-						      PROT_READ | PROT_WRITE,
-						      MAP_SHARED,
-						      dmabuf_fd,
-						      offset);
-			if (tp->pg_va_tbl[page_no] == (void *)-1) {
-				TFP_DRV_LOG(ERR,
-		"MMap memory error. level:%d page:%d pg_count:%d - %s\n",
-					    level,
-				     page_no,
-					    pg_count,
-					    strerror(errno));
-				return -ENOMEM;
-			}
-			offset += tp->pg_size;
-			tp->pg_count++;
-		}
-	}
-
-	return 0;
-}
-
-static int tf_mmap_tbl_scope(struct tf_tbl_scope_cb *tbl_scope_cb,
-			     enum tf_dir dir,
-			     int tbl_type,
-			     int dmabuf_fd)
-{
-	struct hcapi_cfa_em_table *tbl;
-
-	if (tbl_type == TF_EFC_TABLE)
-		return 0;
-
-	tbl = &tbl_scope_cb->em_ctx_info[dir].em_tables[tbl_type];
-	return tf_em_dmabuf_mem_map(tbl, dmabuf_fd);
-}
-
-#define TF_LFC_DEVICE "/dev/bnxt_lfc"
-
-static int
-tf_prepare_dmabuf_bnxt_lfc_device(struct tf_tbl_scope_cb *tbl_scope_cb)
-{
-	int lfc_fd;
-
-	lfc_fd = open(TF_LFC_DEVICE, O_RDWR);
-	if (!lfc_fd) {
-		TFP_DRV_LOG(ERR,
-			    "EEM: open %s device error\n",
-			    TF_LFC_DEVICE);
-		return -ENOENT;
-	}
-
-	tbl_scope_cb->lfc_fd = lfc_fd;
-
-	return 0;
-}
-
-int offload_system_mmap(struct tf_tbl_scope_cb *tbl_scope_cb)
-{
-	int rc;
-	int dmabuf_fd;
-	enum tf_dir dir;
-	enum hcapi_cfa_em_table_type tbl_type;
-
-	rc = tf_prepare_dmabuf_bnxt_lfc_device(tbl_scope_cb);
-	if (rc) {
-		TFP_DRV_LOG(ERR, "EEM: Prepare bnxt_lfc channel failed\n");
-		return rc;
-	}
-
-	rc = tf_export_tbl_scope(tbl_scope_cb->lfc_fd,
-				 (int *)tbl_scope_cb->fd,
-				 tbl_scope_cb->bus,
-				 tbl_scope_cb->devfn);
-	if (rc) {
-		TFP_DRV_LOG(ERR,
-			    "export dmabuf fd failed\n");
-		return rc;
-	}
-
-	tbl_scope_cb->valid = true;
-
-	for (dir = 0; dir < TF_DIR_MAX; dir++) {
-		for (tbl_type = TF_KEY0_TABLE; tbl_type <
-			     TF_MAX_TABLE; tbl_type++) {
-			if (tbl_type == TF_EFC_TABLE)
-				continue;
-
-			dmabuf_fd = tbl_scope_cb->fd[(dir ? 0 : 1)][tbl_type];
-			rc = tf_mmap_tbl_scope(tbl_scope_cb,
-					       dir,
-					       tbl_type,
-					       dmabuf_fd);
-			if (rc) {
-				TFP_DRV_LOG(ERR,
-					    "dir:%d tbl:%d mmap failed rc %d\n",
-					    dir,
-					    tbl_type,
-					    rc);
-				break;
-			}
-		}
-	}
-	return 0;
-}
-
-static int
-tf_destroy_dmabuf_bnxt_lfc_device(struct tf_tbl_scope_cb *tbl_scope_cb)
-{
-	close(tbl_scope_cb->lfc_fd);
-
-	return 0;
-}
-
-static int
-tf_dmabuf_alloc(struct tf *tfp, struct tf_tbl_scope_cb *tbl_scope_cb)
-{
-	int rc;
-
-	rc = tfp_msg_hwrm_oem_cmd(tfp,
-		tbl_scope_cb->em_ctx_info[TF_DIR_RX].em_tables[TF_KEY0_TABLE].num_entries);
-	if (rc)
-		PMD_DRV_LOG(ERR, "EEM: Failed to prepare system memory rc:%d\n",
-			    rc);
-
-	return 0;
-}
-
-static int
-tf_dmabuf_free(struct tf *tfp, struct tf_tbl_scope_cb *tbl_scope_cb)
-{
-	int rc;
-
-	rc = tfp_msg_hwrm_oem_cmd(tfp, 0);
-	if (rc)
-		TFP_DRV_LOG(ERR, "EEM: Failed to cleanup system memory\n");
-
-	tf_destroy_dmabuf_bnxt_lfc_device(tbl_scope_cb);
-
-	return 0;
-}
-
-int
-tf_em_ext_alloc(struct tf *tfp,
-		struct tf_alloc_tbl_scope_parms *parms)
-{
-	int rc;
-	struct tf_session *tfs;
-	struct tf_tbl_scope_cb *tbl_scope_cb;
-	struct tf_free_tbl_scope_parms free_parms;
-	int dir;
-	int i;
-	struct hcapi_cfa_em_table *em_tables;
-
-	TF_CHECK_PARMS2(tfp, parms);
-
-	/* Retrieve the session information */
-	rc = tf_session_get_session(tfp, &tfs);
-	if (rc) {
-		TFP_DRV_LOG(ERR,
-			    "Failed to lookup session, rc:%s\n",
-			    strerror(-rc));
-		return rc;
-	}
-
-	rc = tf_tbl_scope_alloc(&parms->tbl_scope_id);
-	if (rc) {
-		TFP_DRV_LOG(ERR,
-			    "Failed to allocate table scope\n");
-		return rc;
-	}
-
-	tbl_scope_cb = &tbl_scopes[parms->tbl_scope_id];
-	tbl_scope_cb->index = parms->tbl_scope_id;
-	tbl_scope_cb->tbl_scope_id = parms->tbl_scope_id;
-	tbl_scope_cb->bus = tfs->session_id.internal.bus;
-	tbl_scope_cb->devfn = tfs->session_id.internal.device;
-
-	for (dir = 0; dir < TF_DIR_MAX; dir++) {
-		rc = tf_msg_em_qcaps(tfp,
-				     dir,
-				     &tbl_scope_cb->em_caps[dir]);
-		if (rc) {
-			TFP_DRV_LOG(ERR,
-				    "EEM: Unable to query for EEM capability,"
-				    " rc:%s\n",
-				    strerror(-rc));
-			goto cleanup;
-		}
-	}
-
-	/*
-	 * Validate and setup table sizes
-	 */
-	if (tf_em_validate_num_entries(tbl_scope_cb, parms))
-		goto cleanup;
-
-	rc = tf_dmabuf_alloc(tfp, tbl_scope_cb);
-	if (rc) {
-		TFP_DRV_LOG(ERR,
-			    "System DMA buff alloc failed\n");
-		return -EIO;
-	}
-
-	for (dir = 0; dir < TF_DIR_MAX; dir++) {
-		for (i = TF_KEY0_TABLE; i < TF_MAX_TABLE; i++) {
-			if (i == TF_EFC_TABLE)
-				continue;
-
-			em_tables =
-				&tbl_scope_cb->em_ctx_info[dir].em_tables[i];
-
-			rc = tf_em_size_table(em_tables, TF_EM_PAGE_SIZE);
-			if (rc) {
-				TFP_DRV_LOG(ERR, "Size table failed\n");
-				goto cleanup;
-			}
-		}
-
-		em_tables = tbl_scope_cb->em_ctx_info[dir].em_tables;
-		rc = tf_create_tbl_pool_external(dir,
-					tbl_scope_cb,
-					em_tables[TF_RECORD_TABLE].num_entries,
-					em_tables[TF_RECORD_TABLE].entry_size);
-
-		if (rc) {
-			TFP_DRV_LOG(ERR,
-				    "%s TBL: Unable to allocate idx pools %s\n",
-				    tf_dir_2_str(dir),
-				    strerror(-rc));
-			goto cleanup_full;
-		}
-	}
-
-#if (TF_EM_SYSMEM_DELAY_EXPORT == 0)
-	rc = offload_system_mmap(tbl_scope_cb);
-
-	if (rc) {
-		TFP_DRV_LOG(ERR,
-			    "System alloc mmap failed\n");
-		goto cleanup_full;
-	}
-#endif
-
-	return rc;
-
-cleanup_full:
-	free_parms.tbl_scope_id = parms->tbl_scope_id;
-	tf_em_ext_free(tfp, &free_parms);
-	return -EINVAL;
-
-cleanup:
-	/* Free Table control block */
-	tf_tbl_scope_free(parms->tbl_scope_id);
-	return -EINVAL;
-}
-
-int
-tf_em_ext_free(struct tf *tfp,
-	       struct tf_free_tbl_scope_parms *parms)
-{
-	int rc;
-	struct tf_session *tfs;
-	struct tf_tbl_scope_cb *tbl_scope_cb;
-	int dir;
-
-	TF_CHECK_PARMS2(tfp, parms);
-
-	/* Retrieve the session information */
-	rc = tf_session_get_session(tfp, &tfs);
-	if (rc) {
-		TFP_DRV_LOG(ERR,
-			    "Failed to lookup session, rc:%s\n",
-			    strerror(-rc));
-		return rc;
-	}
-
-	tbl_scope_cb = &tbl_scopes[parms->tbl_scope_id];
-
-	rc = tf_tbl_scope_free(parms->tbl_scope_id);
-	if (rc) {
-		TFP_DRV_LOG(ERR,
-			    "Failed to free table scope\n");
-	}
-
-	for (dir = 0; dir < TF_DIR_MAX; dir++) {
-		/* Free associated external pools
-		 */
-		tf_destroy_tbl_pool_external(dir,
-					     tbl_scope_cb);
-
-		/* Unmap memory */
-		tf_em_ctx_unreg(tbl_scope_cb, dir);
-
-		tf_msg_em_op(tfp,
-			     dir,
-			     HWRM_TF_EXT_EM_OP_INPUT_OP_EXT_EM_DISABLE);
-	}
-
-	tf_dmabuf_free(tfp, tbl_scope_cb);
-	tbl_scope_cb->valid = false;
-
-	return rc;
-}
diff --git a/drivers/net/bnxt/tf_core/tf_tbl.h b/drivers/net/bnxt/tf_core/tf_tbl.h
index 930fcc324..2a5d24c94 100644
--- a/drivers/net/bnxt/tf_core/tf_tbl.h
+++ b/drivers/net/bnxt/tf_core/tf_tbl.h
@@ -38,13 +38,6 @@  struct tf_em_caps {
  */
 struct tf_tbl_scope_cb {
 	uint32_t tbl_scope_id;
-#ifdef TF_USE_SYSTEM_MEM
-	int lfc_fd;
-	uint32_t bus;
-	uint32_t devfn;
-	int fd[TF_DIR_MAX][TF_MAX_TABLE];
-	bool valid;
-#endif
 	int index;
 	struct hcapi_cfa_em_ctx_mem_info em_ctx_info[TF_DIR_MAX];
 	struct tf_em_caps em_caps[TF_DIR_MAX];
diff --git a/drivers/net/bnxt/tf_core/tfp.c b/drivers/net/bnxt/tf_core/tfp.c
index 3eade3127..426a182a9 100644
--- a/drivers/net/bnxt/tf_core/tfp.c
+++ b/drivers/net/bnxt/tf_core/tfp.c
@@ -87,18 +87,6 @@  tfp_send_msg_tunneled(struct tf *tfp,
 	return rc;
 }
 
-#ifdef TF_USE_SYSTEM_MEM
-int
-tfp_msg_hwrm_oem_cmd(struct tf *tfp,
-		     uint32_t max_flows)
-{
-	return bnxt_hwrm_oem_cmd(container_of(tfp,
-					      struct bnxt,
-					      tfp),
-				 max_flows);
-}
-#endif /* TF_USE_SYSTEM_MEM */
-
 /**
  * Allocates zero'ed memory from the heap.
  *