Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/96406/?format=api
{ "id": 96406, "url": "http://patchwork.dpdk.org/api/patches/96406/?format=api", "web_url": "http://patchwork.dpdk.org/project/dpdk/patch/20210729134711.35870-3-heinrich.kuhn@netronome.com/", "project": { "id": 1, "url": "http://patchwork.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<20210729134711.35870-3-heinrich.kuhn@netronome.com>", "list_archive_url": "https://inbox.dpdk.org/dev/20210729134711.35870-3-heinrich.kuhn@netronome.com", "date": "2021-07-29T13:47:06", "name": "[v3,2/7] net/nfp: move rxtx functions to their own file", "commit_ref": null, "pull_url": null, "state": "accepted", "archived": true, "hash": "d19c1ded5565425a3a1ad4868defb8fab97a0e56", "submitter": { "id": 1523, "url": "http://patchwork.dpdk.org/api/people/1523/?format=api", "name": "Heinrich Kuhn", "email": "heinrich.kuhn@netronome.com" }, "delegate": { "id": 319, "url": "http://patchwork.dpdk.org/api/users/319/?format=api", "username": "fyigit", "first_name": "Ferruh", "last_name": "Yigit", "email": "ferruh.yigit@amd.com" }, "mbox": "http://patchwork.dpdk.org/project/dpdk/patch/20210729134711.35870-3-heinrich.kuhn@netronome.com/mbox/", "series": [ { "id": 18064, "url": "http://patchwork.dpdk.org/api/series/18064/?format=api", "web_url": "http://patchwork.dpdk.org/project/dpdk/list/?series=18064", "date": "2021-07-29T13:47:04", "name": "Refactor the NFP PMD", "version": 3, "mbox": "http://patchwork.dpdk.org/series/18064/mbox/" } ], "comments": "http://patchwork.dpdk.org/api/patches/96406/comments/", "check": "warning", "checks": "http://patchwork.dpdk.org/api/patches/96406/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@inbox.dpdk.org", "Delivered-To": "patchwork@inbox.dpdk.org", "Received": [ "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id 4C18EA034F;\n\tThu, 29 Jul 2021 15:47:39 +0200 (CEST)", "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id 209E5410F3;\n\tThu, 29 Jul 2021 15:47:36 +0200 (CEST)", "from mail-wr1-f41.google.com (mail-wr1-f41.google.com\n [209.85.221.41]) by mails.dpdk.org (Postfix) with ESMTP id 463254111B\n for <dev@dpdk.org>; Thu, 29 Jul 2021 15:47:34 +0200 (CEST)", "by mail-wr1-f41.google.com with SMTP id c16so7009455wrp.13\n for <dev@dpdk.org>; Thu, 29 Jul 2021 06:47:34 -0700 (PDT)", "from localhost.localdomain (dsl-197-245-41-228.voxdsl.co.za.\n [197.245.41.228])\n by smtp.gmail.com with ESMTPSA id v2sm3498246wro.48.2021.07.29.06.47.31\n (version=TLS1_2 cipher=ECDHE-ECDSA-AES128-GCM-SHA256 bits=128/128);\n Thu, 29 Jul 2021 06:47:32 -0700 (PDT)" ], "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n d=netronome-com.20150623.gappssmtp.com; s=20150623;\n h=from:to:cc:subject:date:message-id:in-reply-to:references\n :mime-version:content-transfer-encoding;\n bh=gvKoGlXRstgH6RjaAZiZZlYe/Whi7BBUAae0dW9iEkg=;\n b=KnBrTgboKzq0qzDX86U9lnit0i3/07NrSrgtHzwn6mzWXo8d0MPnO7SoCHhlFO68a/\n 4SNhbU1b5LoY0fmoL03uesIXyV9BdiNku0tryIygLhZuD9zDg47uMSYtC7b2bnXhyyce\n 2mAWe45mKBk+6KIlT70TWz5tVcf245IKqsuMcMeURWLwSjY2vL2uY4AtVgvy2qIYKrcF\n rAxR9JUxNCrk9Xg5jttXB42mXV4N+kP4ug7AiBJ7Hcg2RaeS4WMRjLE9ZrOfu0ilWFeX\n UgsIXMixwxirSAke/XO67q6mNJguYjqW8b3k9d4lkBs8F8a1tJH5A6+jm2qsAMvUZZ8F\n h+Hw==", "X-Google-DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed;\n d=1e100.net; s=20161025;\n h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to\n :references:mime-version:content-transfer-encoding;\n bh=gvKoGlXRstgH6RjaAZiZZlYe/Whi7BBUAae0dW9iEkg=;\n b=AfXxFK9QWq72TLzqFhoXA3xEsIc/sDB9mzaZQrzT/7dEBjbzqjSHduWTcgBOaUi65L\n QLsSlTylzCdSdCUawRSzMaQFUKbdn7DTzZY28D4r7uP/QoM0TAjjGSKm/ndPvxT2cRB/\n Z5IB17GJ4tduYF5U2ZWeoge2Mzsl9Bz9v5YtG8ViufZ+d9QZbH5Xbqr0JuI/7A+C15ML\n FMWyPN6hH+G6YuIAl98uKoGZZdQbHEgKBCcwDgiD9C52KyigjWcCDUw85rvxJ791B0nh\n 5L/3IKts3wJxitHa+ziQZds5VpKhtDtpKvhcdD34FznXPVWXm/gZKPOBz/DBtMEerPUN\n K91A==", "X-Gm-Message-State": "AOAM533pu9xZ8AZ02BcaxzyI9oPoVV680MsBwI7aebe2RkeK2MvFSqec\n MU95CN2AKppPrGPrv/HZTC8+/ukfzdzacUcS6/0XM5dhRTWUmT+sLj/nBcPhWgedN/y0Mz08T4x\n uZeMYazT6GQYFawIVwgk4gifvc+9Qc+qNkZWg94MqQvbciONcMe/SXESCAqr/psOW", "X-Google-Smtp-Source": "\n ABdhPJyq77M36VXmrhsXzXsUGZr4i/CIukZ/4tHThrKqwid3fZhzfkxSDBbrxE4cV6DaPwd6P8SS8g==", "X-Received": "by 2002:a5d:6184:: with SMTP id j4mr5059783wru.340.1627566453195;\n Thu, 29 Jul 2021 06:47:33 -0700 (PDT)", "From": "Heinrich Kuhn <heinrich.kuhn@netronome.com>", "To": "dev@dpdk.org", "Cc": "Heinrich Kuhn <heinrich.kuhn@netronome.com>,\n Simon Horman <simon.horman@corigine.com>", "Date": "Thu, 29 Jul 2021 15:47:06 +0200", "Message-Id": "<20210729134711.35870-3-heinrich.kuhn@netronome.com>", "X-Mailer": "git-send-email 2.30.1 (Apple Git-130)", "In-Reply-To": "<20210729134711.35870-1-heinrich.kuhn@netronome.com>", "References": "<20210716083545.34444-1-heinrich.kuhn@netronome.com>\n <20210729134711.35870-1-heinrich.kuhn@netronome.com>", "MIME-Version": "1.0", "Content-Transfer-Encoding": "8bit", "Subject": "[dpdk-dev] [PATCH v3 2/7] net/nfp: move rxtx functions to their own\n file", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.29", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Create a new rxtx file and move the Rx/Tx functions to this file. This\ncommit will also move the needed shared functions to the nfp_net_pmd.h\nfile as needed.\n\nSigned-off-by: Heinrich Kuhn <heinrich.kuhn@netronome.com>\nSigned-off-by: Simon Horman <simon.horman@corigine.com>\n---\n drivers/net/nfp/meson.build | 1 +\n drivers/net/nfp/nfp_net.c | 1090 ---------------------------------\n drivers/net/nfp/nfp_net_pmd.h | 184 ++++--\n drivers/net/nfp/nfp_rxtx.c | 1002 ++++++++++++++++++++++++++++++\n drivers/net/nfp/nfp_rxtx.h | 27 +\n 5 files changed, 1173 insertions(+), 1131 deletions(-)\n create mode 100644 drivers/net/nfp/nfp_rxtx.c", "diff": "diff --git a/drivers/net/nfp/meson.build b/drivers/net/nfp/meson.build\nindex b51e2e5f20..1b289e2354 100644\n--- a/drivers/net/nfp/meson.build\n+++ b/drivers/net/nfp/meson.build\n@@ -19,4 +19,5 @@ sources = files(\n 'nfpcore/nfp_nsp_eth.c',\n 'nfpcore/nfp_hwinfo.c',\n 'nfp_net.c',\n+ 'nfp_rxtx.c',\n )\ndiff --git a/drivers/net/nfp/nfp_net.c b/drivers/net/nfp/nfp_net.c\nindex 29eb8c84ad..de0de808f4 100644\n--- a/drivers/net/nfp/nfp_net.c\n+++ b/drivers/net/nfp/nfp_net.c\n@@ -68,29 +68,11 @@ static int nfp_init_phyports(struct nfp_pf_dev *pf_dev);\n static int nfp_net_link_update(struct rte_eth_dev *dev, int wait_to_complete);\n static int nfp_net_promisc_enable(struct rte_eth_dev *dev);\n static int nfp_net_promisc_disable(struct rte_eth_dev *dev);\n-static int nfp_net_rx_fill_freelist(struct nfp_net_rxq *rxq);\n-static uint32_t nfp_net_rx_queue_count(struct rte_eth_dev *dev,\n-\t\t\t\t uint16_t queue_idx);\n-static uint16_t nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,\n-\t\t\t\t uint16_t nb_pkts);\n-static void nfp_net_rx_queue_release(void *rxq);\n-static int nfp_net_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,\n-\t\t\t\t uint16_t nb_desc, unsigned int socket_id,\n-\t\t\t\t const struct rte_eth_rxconf *rx_conf,\n-\t\t\t\t struct rte_mempool *mp);\n-static int nfp_net_tx_free_bufs(struct nfp_net_txq *txq);\n-static void nfp_net_tx_queue_release(void *txq);\n-static int nfp_net_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,\n-\t\t\t\t uint16_t nb_desc, unsigned int socket_id,\n-\t\t\t\t const struct rte_eth_txconf *tx_conf);\n static int nfp_net_start(struct rte_eth_dev *dev);\n static int nfp_net_stats_get(struct rte_eth_dev *dev,\n \t\t\t struct rte_eth_stats *stats);\n static int nfp_net_stats_reset(struct rte_eth_dev *dev);\n static int nfp_net_stop(struct rte_eth_dev *dev);\n-static uint16_t nfp_net_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,\n-\t\t\t\t uint16_t nb_pkts);\n-\n static int nfp_net_rss_config_default(struct rte_eth_dev *dev);\n static int nfp_net_rss_hash_update(struct rte_eth_dev *dev,\n \t\t\t\t struct rte_eth_rss_conf *rss_conf);\n@@ -108,184 +90,6 @@ static int nfp_fw_setup(struct rte_pci_device *dev,\n \t\t\tstruct nfp_eth_table *nfp_eth_table,\n \t\t\tstruct nfp_hwinfo *hwinfo);\n \n-\n-/* The offset of the queue controller queues in the PCIe Target */\n-#define NFP_PCIE_QUEUE(_q) (0x80000 + (NFP_QCP_QUEUE_ADDR_SZ * ((_q) & 0xff)))\n-\n-/* Maximum value which can be added to a queue with one transaction */\n-#define NFP_QCP_MAX_ADD\t0x7f\n-\n-#define RTE_MBUF_DMA_ADDR_DEFAULT(mb) \\\n-\t(uint64_t)((mb)->buf_iova + RTE_PKTMBUF_HEADROOM)\n-\n-/* nfp_qcp_ptr - Read or Write Pointer of a queue */\n-enum nfp_qcp_ptr {\n-\tNFP_QCP_READ_PTR = 0,\n-\tNFP_QCP_WRITE_PTR\n-};\n-\n-/*\n- * nfp_qcp_ptr_add - Add the value to the selected pointer of a queue\n- * @q: Base address for queue structure\n- * @ptr: Add to the Read or Write pointer\n- * @val: Value to add to the queue pointer\n- *\n- * If @val is greater than @NFP_QCP_MAX_ADD multiple writes are performed.\n- */\n-static inline void\n-nfp_qcp_ptr_add(uint8_t *q, enum nfp_qcp_ptr ptr, uint32_t val)\n-{\n-\tuint32_t off;\n-\n-\tif (ptr == NFP_QCP_READ_PTR)\n-\t\toff = NFP_QCP_QUEUE_ADD_RPTR;\n-\telse\n-\t\toff = NFP_QCP_QUEUE_ADD_WPTR;\n-\n-\twhile (val > NFP_QCP_MAX_ADD) {\n-\t\tnn_writel(rte_cpu_to_le_32(NFP_QCP_MAX_ADD), q + off);\n-\t\tval -= NFP_QCP_MAX_ADD;\n-\t}\n-\n-\tnn_writel(rte_cpu_to_le_32(val), q + off);\n-}\n-\n-/*\n- * nfp_qcp_read - Read the current Read/Write pointer value for a queue\n- * @q: Base address for queue structure\n- * @ptr: Read or Write pointer\n- */\n-static inline uint32_t\n-nfp_qcp_read(uint8_t *q, enum nfp_qcp_ptr ptr)\n-{\n-\tuint32_t off;\n-\tuint32_t val;\n-\n-\tif (ptr == NFP_QCP_READ_PTR)\n-\t\toff = NFP_QCP_QUEUE_STS_LO;\n-\telse\n-\t\toff = NFP_QCP_QUEUE_STS_HI;\n-\n-\tval = rte_cpu_to_le_32(nn_readl(q + off));\n-\n-\tif (ptr == NFP_QCP_READ_PTR)\n-\t\treturn val & NFP_QCP_QUEUE_STS_LO_READPTR_mask;\n-\telse\n-\t\treturn val & NFP_QCP_QUEUE_STS_HI_WRITEPTR_mask;\n-}\n-\n-/*\n- * Functions to read/write from/to Config BAR\n- * Performs any endian conversion necessary.\n- */\n-static inline uint8_t\n-nn_cfg_readb(struct nfp_net_hw *hw, int off)\n-{\n-\treturn nn_readb(hw->ctrl_bar + off);\n-}\n-\n-static inline void\n-nn_cfg_writeb(struct nfp_net_hw *hw, int off, uint8_t val)\n-{\n-\tnn_writeb(val, hw->ctrl_bar + off);\n-}\n-\n-static inline uint32_t\n-nn_cfg_readl(struct nfp_net_hw *hw, int off)\n-{\n-\treturn rte_le_to_cpu_32(nn_readl(hw->ctrl_bar + off));\n-}\n-\n-static inline void\n-nn_cfg_writel(struct nfp_net_hw *hw, int off, uint32_t val)\n-{\n-\tnn_writel(rte_cpu_to_le_32(val), hw->ctrl_bar + off);\n-}\n-\n-static inline uint64_t\n-nn_cfg_readq(struct nfp_net_hw *hw, int off)\n-{\n-\treturn rte_le_to_cpu_64(nn_readq(hw->ctrl_bar + off));\n-}\n-\n-static inline void\n-nn_cfg_writeq(struct nfp_net_hw *hw, int off, uint64_t val)\n-{\n-\tnn_writeq(rte_cpu_to_le_64(val), hw->ctrl_bar + off);\n-}\n-\n-static void\n-nfp_net_rx_queue_release_mbufs(struct nfp_net_rxq *rxq)\n-{\n-\tunsigned i;\n-\n-\tif (rxq->rxbufs == NULL)\n-\t\treturn;\n-\n-\tfor (i = 0; i < rxq->rx_count; i++) {\n-\t\tif (rxq->rxbufs[i].mbuf) {\n-\t\t\trte_pktmbuf_free_seg(rxq->rxbufs[i].mbuf);\n-\t\t\trxq->rxbufs[i].mbuf = NULL;\n-\t\t}\n-\t}\n-}\n-\n-static void\n-nfp_net_rx_queue_release(void *rx_queue)\n-{\n-\tstruct nfp_net_rxq *rxq = rx_queue;\n-\n-\tif (rxq) {\n-\t\tnfp_net_rx_queue_release_mbufs(rxq);\n-\t\trte_free(rxq->rxbufs);\n-\t\trte_free(rxq);\n-\t}\n-}\n-\n-static void\n-nfp_net_reset_rx_queue(struct nfp_net_rxq *rxq)\n-{\n-\tnfp_net_rx_queue_release_mbufs(rxq);\n-\trxq->rd_p = 0;\n-\trxq->nb_rx_hold = 0;\n-}\n-\n-static void\n-nfp_net_tx_queue_release_mbufs(struct nfp_net_txq *txq)\n-{\n-\tunsigned i;\n-\n-\tif (txq->txbufs == NULL)\n-\t\treturn;\n-\n-\tfor (i = 0; i < txq->tx_count; i++) {\n-\t\tif (txq->txbufs[i].mbuf) {\n-\t\t\trte_pktmbuf_free_seg(txq->txbufs[i].mbuf);\n-\t\t\ttxq->txbufs[i].mbuf = NULL;\n-\t\t}\n-\t}\n-}\n-\n-static void\n-nfp_net_tx_queue_release(void *tx_queue)\n-{\n-\tstruct nfp_net_txq *txq = tx_queue;\n-\n-\tif (txq) {\n-\t\tnfp_net_tx_queue_release_mbufs(txq);\n-\t\trte_free(txq->txbufs);\n-\t\trte_free(txq);\n-\t}\n-}\n-\n-static void\n-nfp_net_reset_tx_queue(struct nfp_net_txq *txq)\n-{\n-\tnfp_net_tx_queue_release_mbufs(txq);\n-\ttxq->wr_p = 0;\n-\ttxq->rd_p = 0;\n-}\n-\n static int\n __nfp_net_reconfig(struct nfp_net_hw *hw, uint32_t update)\n {\n@@ -463,18 +267,6 @@ nfp_net_disable_queues(struct rte_eth_dev *dev)\n \thw->ctrl = new_ctrl;\n }\n \n-static int\n-nfp_net_rx_freelist_setup(struct rte_eth_dev *dev)\n-{\n-\tint i;\n-\n-\tfor (i = 0; i < dev->data->nb_rx_queues; i++) {\n-\t\tif (nfp_net_rx_fill_freelist(dev->data->rx_queues[i]) < 0)\n-\t\t\treturn -1;\n-\t}\n-\treturn 0;\n-}\n-\n static void\n nfp_net_params_setup(struct nfp_net_hw *hw)\n {\n@@ -1351,44 +1143,6 @@ nfp_net_supported_ptypes_get(struct rte_eth_dev *dev)\n \treturn NULL;\n }\n \n-static uint32_t\n-nfp_net_rx_queue_count(struct rte_eth_dev *dev, uint16_t queue_idx)\n-{\n-\tstruct nfp_net_rxq *rxq;\n-\tstruct nfp_net_rx_desc *rxds;\n-\tuint32_t idx;\n-\tuint32_t count;\n-\n-\trxq = (struct nfp_net_rxq *)dev->data->rx_queues[queue_idx];\n-\n-\tidx = rxq->rd_p;\n-\n-\tcount = 0;\n-\n-\t/*\n-\t * Other PMDs are just checking the DD bit in intervals of 4\n-\t * descriptors and counting all four if the first has the DD\n-\t * bit on. Of course, this is not accurate but can be good for\n-\t * performance. But ideally that should be done in descriptors\n-\t * chunks belonging to the same cache line\n-\t */\n-\n-\twhile (count < rxq->rx_count) {\n-\t\trxds = &rxq->rxds[idx];\n-\t\tif ((rxds->rxd.meta_len_dd & PCIE_DESC_RX_DD) == 0)\n-\t\t\tbreak;\n-\n-\t\tcount++;\n-\t\tidx++;\n-\n-\t\t/* Wrapping? */\n-\t\tif ((idx) == rxq->rx_count)\n-\t\t\tidx = 0;\n-\t}\n-\n-\treturn count;\n-}\n-\n static int\n nfp_rx_queue_intr_enable(struct rte_eth_dev *dev, uint16_t queue_id)\n {\n@@ -1570,850 +1324,6 @@ nfp_net_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu)\n \treturn 0;\n }\n \n-static int\n-nfp_net_rx_queue_setup(struct rte_eth_dev *dev,\n-\t\t uint16_t queue_idx, uint16_t nb_desc,\n-\t\t unsigned int socket_id,\n-\t\t const struct rte_eth_rxconf *rx_conf,\n-\t\t struct rte_mempool *mp)\n-{\n-\tconst struct rte_memzone *tz;\n-\tstruct nfp_net_rxq *rxq;\n-\tstruct nfp_net_hw *hw;\n-\tuint32_t rx_desc_sz;\n-\n-\thw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);\n-\n-\tPMD_INIT_FUNC_TRACE();\n-\n-\t/* Validating number of descriptors */\n-\trx_desc_sz = nb_desc * sizeof(struct nfp_net_rx_desc);\n-\tif (rx_desc_sz % NFP_ALIGN_RING_DESC != 0 ||\n-\t nb_desc > NFP_NET_MAX_RX_DESC ||\n-\t nb_desc < NFP_NET_MIN_RX_DESC) {\n-\t\tPMD_DRV_LOG(ERR, \"Wrong nb_desc value\");\n-\t\treturn -EINVAL;\n-\t}\n-\n-\t/*\n-\t * Free memory prior to re-allocation if needed. This is the case after\n-\t * calling nfp_net_stop\n-\t */\n-\tif (dev->data->rx_queues[queue_idx]) {\n-\t\tnfp_net_rx_queue_release(dev->data->rx_queues[queue_idx]);\n-\t\tdev->data->rx_queues[queue_idx] = NULL;\n-\t}\n-\n-\t/* Allocating rx queue data structure */\n-\trxq = rte_zmalloc_socket(\"ethdev RX queue\", sizeof(struct nfp_net_rxq),\n-\t\t\t\t RTE_CACHE_LINE_SIZE, socket_id);\n-\tif (rxq == NULL)\n-\t\treturn -ENOMEM;\n-\n-\t/* Hw queues mapping based on firmware configuration */\n-\trxq->qidx = queue_idx;\n-\trxq->fl_qcidx = queue_idx * hw->stride_rx;\n-\trxq->rx_qcidx = rxq->fl_qcidx + (hw->stride_rx - 1);\n-\trxq->qcp_fl = hw->rx_bar + NFP_QCP_QUEUE_OFF(rxq->fl_qcidx);\n-\trxq->qcp_rx = hw->rx_bar + NFP_QCP_QUEUE_OFF(rxq->rx_qcidx);\n-\n-\t/*\n-\t * Tracking mbuf size for detecting a potential mbuf overflow due to\n-\t * RX offset\n-\t */\n-\trxq->mem_pool = mp;\n-\trxq->mbuf_size = rxq->mem_pool->elt_size;\n-\trxq->mbuf_size -= (sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM);\n-\thw->flbufsz = rxq->mbuf_size;\n-\n-\trxq->rx_count = nb_desc;\n-\trxq->port_id = dev->data->port_id;\n-\trxq->rx_free_thresh = rx_conf->rx_free_thresh;\n-\trxq->drop_en = rx_conf->rx_drop_en;\n-\n-\t/*\n-\t * Allocate RX ring hardware descriptors. A memzone large enough to\n-\t * handle the maximum ring size is allocated in order to allow for\n-\t * resizing in later calls to the queue setup function.\n-\t */\n-\ttz = rte_eth_dma_zone_reserve(dev, \"rx_ring\", queue_idx,\n-\t\t\t\t sizeof(struct nfp_net_rx_desc) *\n-\t\t\t\t NFP_NET_MAX_RX_DESC, NFP_MEMZONE_ALIGN,\n-\t\t\t\t socket_id);\n-\n-\tif (tz == NULL) {\n-\t\tPMD_DRV_LOG(ERR, \"Error allocating rx dma\");\n-\t\tnfp_net_rx_queue_release(rxq);\n-\t\treturn -ENOMEM;\n-\t}\n-\n-\t/* Saving physical and virtual addresses for the RX ring */\n-\trxq->dma = (uint64_t)tz->iova;\n-\trxq->rxds = (struct nfp_net_rx_desc *)tz->addr;\n-\n-\t/* mbuf pointers array for referencing mbufs linked to RX descriptors */\n-\trxq->rxbufs = rte_zmalloc_socket(\"rxq->rxbufs\",\n-\t\t\t\t\t sizeof(*rxq->rxbufs) * nb_desc,\n-\t\t\t\t\t RTE_CACHE_LINE_SIZE, socket_id);\n-\tif (rxq->rxbufs == NULL) {\n-\t\tnfp_net_rx_queue_release(rxq);\n-\t\treturn -ENOMEM;\n-\t}\n-\n-\tPMD_RX_LOG(DEBUG, \"rxbufs=%p hw_ring=%p dma_addr=0x%\" PRIx64,\n-\t\t rxq->rxbufs, rxq->rxds, (unsigned long int)rxq->dma);\n-\n-\tnfp_net_reset_rx_queue(rxq);\n-\n-\tdev->data->rx_queues[queue_idx] = rxq;\n-\trxq->hw = hw;\n-\n-\t/*\n-\t * Telling the HW about the physical address of the RX ring and number\n-\t * of descriptors in log2 format\n-\t */\n-\tnn_cfg_writeq(hw, NFP_NET_CFG_RXR_ADDR(queue_idx), rxq->dma);\n-\tnn_cfg_writeb(hw, NFP_NET_CFG_RXR_SZ(queue_idx), rte_log2_u32(nb_desc));\n-\n-\treturn 0;\n-}\n-\n-static int\n-nfp_net_rx_fill_freelist(struct nfp_net_rxq *rxq)\n-{\n-\tstruct nfp_net_rx_buff *rxe = rxq->rxbufs;\n-\tuint64_t dma_addr;\n-\tunsigned i;\n-\n-\tPMD_RX_LOG(DEBUG, \"nfp_net_rx_fill_freelist for %u descriptors\",\n-\t\t rxq->rx_count);\n-\n-\tfor (i = 0; i < rxq->rx_count; i++) {\n-\t\tstruct nfp_net_rx_desc *rxd;\n-\t\tstruct rte_mbuf *mbuf = rte_pktmbuf_alloc(rxq->mem_pool);\n-\n-\t\tif (mbuf == NULL) {\n-\t\t\tPMD_DRV_LOG(ERR, \"RX mbuf alloc failed queue_id=%u\",\n-\t\t\t\t(unsigned)rxq->qidx);\n-\t\t\treturn -ENOMEM;\n-\t\t}\n-\n-\t\tdma_addr = rte_cpu_to_le_64(RTE_MBUF_DMA_ADDR_DEFAULT(mbuf));\n-\n-\t\trxd = &rxq->rxds[i];\n-\t\trxd->fld.dd = 0;\n-\t\trxd->fld.dma_addr_hi = (dma_addr >> 32) & 0xff;\n-\t\trxd->fld.dma_addr_lo = dma_addr & 0xffffffff;\n-\t\trxe[i].mbuf = mbuf;\n-\t\tPMD_RX_LOG(DEBUG, \"[%d]: %\" PRIx64, i, dma_addr);\n-\t}\n-\n-\t/* Make sure all writes are flushed before telling the hardware */\n-\trte_wmb();\n-\n-\t/* Not advertising the whole ring as the firmware gets confused if so */\n-\tPMD_RX_LOG(DEBUG, \"Increment FL write pointer in %u\",\n-\t\t rxq->rx_count - 1);\n-\n-\tnfp_qcp_ptr_add(rxq->qcp_fl, NFP_QCP_WRITE_PTR, rxq->rx_count - 1);\n-\n-\treturn 0;\n-}\n-\n-static int\n-nfp_net_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,\n-\t\t uint16_t nb_desc, unsigned int socket_id,\n-\t\t const struct rte_eth_txconf *tx_conf)\n-{\n-\tconst struct rte_memzone *tz;\n-\tstruct nfp_net_txq *txq;\n-\tuint16_t tx_free_thresh;\n-\tstruct nfp_net_hw *hw;\n-\tuint32_t tx_desc_sz;\n-\n-\thw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);\n-\n-\tPMD_INIT_FUNC_TRACE();\n-\n-\t/* Validating number of descriptors */\n-\ttx_desc_sz = nb_desc * sizeof(struct nfp_net_tx_desc);\n-\tif (tx_desc_sz % NFP_ALIGN_RING_DESC != 0 ||\n-\t nb_desc > NFP_NET_MAX_TX_DESC ||\n-\t nb_desc < NFP_NET_MIN_TX_DESC) {\n-\t\tPMD_DRV_LOG(ERR, \"Wrong nb_desc value\");\n-\t\treturn -EINVAL;\n-\t}\n-\n-\ttx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?\n-\t\t\t\t tx_conf->tx_free_thresh :\n-\t\t\t\t DEFAULT_TX_FREE_THRESH);\n-\n-\tif (tx_free_thresh > (nb_desc)) {\n-\t\tPMD_DRV_LOG(ERR,\n-\t\t\t\"tx_free_thresh must be less than the number of TX \"\n-\t\t\t\"descriptors. (tx_free_thresh=%u port=%d \"\n-\t\t\t\"queue=%d)\", (unsigned int)tx_free_thresh,\n-\t\t\tdev->data->port_id, (int)queue_idx);\n-\t\treturn -(EINVAL);\n-\t}\n-\n-\t/*\n-\t * Free memory prior to re-allocation if needed. This is the case after\n-\t * calling nfp_net_stop\n-\t */\n-\tif (dev->data->tx_queues[queue_idx]) {\n-\t\tPMD_TX_LOG(DEBUG, \"Freeing memory prior to re-allocation %d\",\n-\t\t\t queue_idx);\n-\t\tnfp_net_tx_queue_release(dev->data->tx_queues[queue_idx]);\n-\t\tdev->data->tx_queues[queue_idx] = NULL;\n-\t}\n-\n-\t/* Allocating tx queue data structure */\n-\ttxq = rte_zmalloc_socket(\"ethdev TX queue\", sizeof(struct nfp_net_txq),\n-\t\t\t\t RTE_CACHE_LINE_SIZE, socket_id);\n-\tif (txq == NULL) {\n-\t\tPMD_DRV_LOG(ERR, \"Error allocating tx dma\");\n-\t\treturn -ENOMEM;\n-\t}\n-\n-\t/*\n-\t * Allocate TX ring hardware descriptors. A memzone large enough to\n-\t * handle the maximum ring size is allocated in order to allow for\n-\t * resizing in later calls to the queue setup function.\n-\t */\n-\ttz = rte_eth_dma_zone_reserve(dev, \"tx_ring\", queue_idx,\n-\t\t\t\t sizeof(struct nfp_net_tx_desc) *\n-\t\t\t\t NFP_NET_MAX_TX_DESC, NFP_MEMZONE_ALIGN,\n-\t\t\t\t socket_id);\n-\tif (tz == NULL) {\n-\t\tPMD_DRV_LOG(ERR, \"Error allocating tx dma\");\n-\t\tnfp_net_tx_queue_release(txq);\n-\t\treturn -ENOMEM;\n-\t}\n-\n-\ttxq->tx_count = nb_desc;\n-\ttxq->tx_free_thresh = tx_free_thresh;\n-\ttxq->tx_pthresh = tx_conf->tx_thresh.pthresh;\n-\ttxq->tx_hthresh = tx_conf->tx_thresh.hthresh;\n-\ttxq->tx_wthresh = tx_conf->tx_thresh.wthresh;\n-\n-\t/* queue mapping based on firmware configuration */\n-\ttxq->qidx = queue_idx;\n-\ttxq->tx_qcidx = queue_idx * hw->stride_tx;\n-\ttxq->qcp_q = hw->tx_bar + NFP_QCP_QUEUE_OFF(txq->tx_qcidx);\n-\n-\ttxq->port_id = dev->data->port_id;\n-\n-\t/* Saving physical and virtual addresses for the TX ring */\n-\ttxq->dma = (uint64_t)tz->iova;\n-\ttxq->txds = (struct nfp_net_tx_desc *)tz->addr;\n-\n-\t/* mbuf pointers array for referencing mbufs linked to TX descriptors */\n-\ttxq->txbufs = rte_zmalloc_socket(\"txq->txbufs\",\n-\t\t\t\t\t sizeof(*txq->txbufs) * nb_desc,\n-\t\t\t\t\t RTE_CACHE_LINE_SIZE, socket_id);\n-\tif (txq->txbufs == NULL) {\n-\t\tnfp_net_tx_queue_release(txq);\n-\t\treturn -ENOMEM;\n-\t}\n-\tPMD_TX_LOG(DEBUG, \"txbufs=%p hw_ring=%p dma_addr=0x%\" PRIx64,\n-\t\t txq->txbufs, txq->txds, (unsigned long int)txq->dma);\n-\n-\tnfp_net_reset_tx_queue(txq);\n-\n-\tdev->data->tx_queues[queue_idx] = txq;\n-\ttxq->hw = hw;\n-\n-\t/*\n-\t * Telling the HW about the physical address of the TX ring and number\n-\t * of descriptors in log2 format\n-\t */\n-\tnn_cfg_writeq(hw, NFP_NET_CFG_TXR_ADDR(queue_idx), txq->dma);\n-\tnn_cfg_writeb(hw, NFP_NET_CFG_TXR_SZ(queue_idx), rte_log2_u32(nb_desc));\n-\n-\treturn 0;\n-}\n-\n-/* nfp_net_tx_tso - Set TX descriptor for TSO */\n-static inline void\n-nfp_net_tx_tso(struct nfp_net_txq *txq, struct nfp_net_tx_desc *txd,\n-\t struct rte_mbuf *mb)\n-{\n-\tuint64_t ol_flags;\n-\tstruct nfp_net_hw *hw = txq->hw;\n-\n-\tif (!(hw->cap & NFP_NET_CFG_CTRL_LSO_ANY))\n-\t\tgoto clean_txd;\n-\n-\tol_flags = mb->ol_flags;\n-\n-\tif (!(ol_flags & PKT_TX_TCP_SEG))\n-\t\tgoto clean_txd;\n-\n-\ttxd->l3_offset = mb->l2_len;\n-\ttxd->l4_offset = mb->l2_len + mb->l3_len;\n-\ttxd->lso_hdrlen = mb->l2_len + mb->l3_len + mb->l4_len;\n-\ttxd->mss = rte_cpu_to_le_16(mb->tso_segsz);\n-\ttxd->flags = PCIE_DESC_TX_LSO;\n-\treturn;\n-\n-clean_txd:\n-\ttxd->flags = 0;\n-\ttxd->l3_offset = 0;\n-\ttxd->l4_offset = 0;\n-\ttxd->lso_hdrlen = 0;\n-\ttxd->mss = 0;\n-}\n-\n-/* nfp_net_tx_cksum - Set TX CSUM offload flags in TX descriptor */\n-static inline void\n-nfp_net_tx_cksum(struct nfp_net_txq *txq, struct nfp_net_tx_desc *txd,\n-\t\t struct rte_mbuf *mb)\n-{\n-\tuint64_t ol_flags;\n-\tstruct nfp_net_hw *hw = txq->hw;\n-\n-\tif (!(hw->cap & NFP_NET_CFG_CTRL_TXCSUM))\n-\t\treturn;\n-\n-\tol_flags = mb->ol_flags;\n-\n-\t/* IPv6 does not need checksum */\n-\tif (ol_flags & PKT_TX_IP_CKSUM)\n-\t\ttxd->flags |= PCIE_DESC_TX_IP4_CSUM;\n-\n-\tswitch (ol_flags & PKT_TX_L4_MASK) {\n-\tcase PKT_TX_UDP_CKSUM:\n-\t\ttxd->flags |= PCIE_DESC_TX_UDP_CSUM;\n-\t\tbreak;\n-\tcase PKT_TX_TCP_CKSUM:\n-\t\ttxd->flags |= PCIE_DESC_TX_TCP_CSUM;\n-\t\tbreak;\n-\t}\n-\n-\tif (ol_flags & (PKT_TX_IP_CKSUM | PKT_TX_L4_MASK))\n-\t\ttxd->flags |= PCIE_DESC_TX_CSUM;\n-}\n-\n-/* nfp_net_rx_cksum - set mbuf checksum flags based on RX descriptor flags */\n-static inline void\n-nfp_net_rx_cksum(struct nfp_net_rxq *rxq, struct nfp_net_rx_desc *rxd,\n-\t\t struct rte_mbuf *mb)\n-{\n-\tstruct nfp_net_hw *hw = rxq->hw;\n-\n-\tif (!(hw->ctrl & NFP_NET_CFG_CTRL_RXCSUM))\n-\t\treturn;\n-\n-\t/* If IPv4 and IP checksum error, fail */\n-\tif (unlikely((rxd->rxd.flags & PCIE_DESC_RX_IP4_CSUM) &&\n-\t !(rxd->rxd.flags & PCIE_DESC_RX_IP4_CSUM_OK)))\n-\t\tmb->ol_flags |= PKT_RX_IP_CKSUM_BAD;\n-\telse\n-\t\tmb->ol_flags |= PKT_RX_IP_CKSUM_GOOD;\n-\n-\t/* If neither UDP nor TCP return */\n-\tif (!(rxd->rxd.flags & PCIE_DESC_RX_TCP_CSUM) &&\n-\t !(rxd->rxd.flags & PCIE_DESC_RX_UDP_CSUM))\n-\t\treturn;\n-\n-\tif (likely(rxd->rxd.flags & PCIE_DESC_RX_L4_CSUM_OK))\n-\t\tmb->ol_flags |= PKT_RX_L4_CKSUM_GOOD;\n-\telse\n-\t\tmb->ol_flags |= PKT_RX_L4_CKSUM_BAD;\n-}\n-\n-#define NFP_HASH_OFFSET ((uint8_t *)mbuf->buf_addr + mbuf->data_off - 4)\n-#define NFP_HASH_TYPE_OFFSET ((uint8_t *)mbuf->buf_addr + mbuf->data_off - 8)\n-\n-#define NFP_DESC_META_LEN(d) (d->rxd.meta_len_dd & PCIE_DESC_RX_META_LEN_MASK)\n-\n-/*\n- * nfp_net_set_hash - Set mbuf hash data\n- *\n- * The RSS hash and hash-type are pre-pended to the packet data.\n- * Extract and decode it and set the mbuf fields.\n- */\n-static inline void\n-nfp_net_set_hash(struct nfp_net_rxq *rxq, struct nfp_net_rx_desc *rxd,\n-\t\t struct rte_mbuf *mbuf)\n-{\n-\tstruct nfp_net_hw *hw = rxq->hw;\n-\tuint8_t *meta_offset;\n-\tuint32_t meta_info;\n-\tuint32_t hash = 0;\n-\tuint32_t hash_type = 0;\n-\n-\tif (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS))\n-\t\treturn;\n-\n-\t/* this is true for new firmwares */\n-\tif (likely(((hw->cap & NFP_NET_CFG_CTRL_RSS2) ||\n-\t (NFD_CFG_MAJOR_VERSION_of(hw->ver) == 4)) &&\n-\t NFP_DESC_META_LEN(rxd))) {\n-\t\t/*\n-\t\t * new metadata api:\n-\t\t * <---- 32 bit ----->\n-\t\t * m field type word\n-\t\t * e data field #2\n-\t\t * t data field #1\n-\t\t * a data field #0\n-\t\t * ====================\n-\t\t * packet data\n-\t\t *\n-\t\t * Field type word contains up to 8 4bit field types\n-\t\t * A 4bit field type refers to a data field word\n-\t\t * A data field word can have several 4bit field types\n-\t\t */\n-\t\tmeta_offset = rte_pktmbuf_mtod(mbuf, uint8_t *);\n-\t\tmeta_offset -= NFP_DESC_META_LEN(rxd);\n-\t\tmeta_info = rte_be_to_cpu_32(*(uint32_t *)meta_offset);\n-\t\tmeta_offset += 4;\n-\t\t/* NFP PMD just supports metadata for hashing */\n-\t\tswitch (meta_info & NFP_NET_META_FIELD_MASK) {\n-\t\tcase NFP_NET_META_HASH:\n-\t\t\t/* next field type is about the hash type */\n-\t\t\tmeta_info >>= NFP_NET_META_FIELD_SIZE;\n-\t\t\t/* hash value is in the data field */\n-\t\t\thash = rte_be_to_cpu_32(*(uint32_t *)meta_offset);\n-\t\t\thash_type = meta_info & NFP_NET_META_FIELD_MASK;\n-\t\t\tbreak;\n-\t\tdefault:\n-\t\t\t/* Unsupported metadata can be a performance issue */\n-\t\t\treturn;\n-\t\t}\n-\t} else {\n-\t\tif (!(rxd->rxd.flags & PCIE_DESC_RX_RSS))\n-\t\t\treturn;\n-\n-\t\thash = rte_be_to_cpu_32(*(uint32_t *)NFP_HASH_OFFSET);\n-\t\thash_type = rte_be_to_cpu_32(*(uint32_t *)NFP_HASH_TYPE_OFFSET);\n-\t}\n-\n-\tmbuf->hash.rss = hash;\n-\tmbuf->ol_flags |= PKT_RX_RSS_HASH;\n-\n-\tswitch (hash_type) {\n-\tcase NFP_NET_RSS_IPV4:\n-\t\tmbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV4;\n-\t\tbreak;\n-\tcase NFP_NET_RSS_IPV6:\n-\t\tmbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6;\n-\t\tbreak;\n-\tcase NFP_NET_RSS_IPV6_EX:\n-\t\tmbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT;\n-\t\tbreak;\n-\tcase NFP_NET_RSS_IPV4_TCP:\n-\t\tmbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT;\n-\t\tbreak;\n-\tcase NFP_NET_RSS_IPV6_TCP:\n-\t\tmbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT;\n-\t\tbreak;\n-\tcase NFP_NET_RSS_IPV4_UDP:\n-\t\tmbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT;\n-\t\tbreak;\n-\tcase NFP_NET_RSS_IPV6_UDP:\n-\t\tmbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT;\n-\t\tbreak;\n-\tdefault:\n-\t\tmbuf->packet_type |= RTE_PTYPE_INNER_L4_MASK;\n-\t}\n-}\n-\n-static inline void\n-nfp_net_mbuf_alloc_failed(struct nfp_net_rxq *rxq)\n-{\n-\trte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++;\n-}\n-\n-#define NFP_DESC_META_LEN(d) (d->rxd.meta_len_dd & PCIE_DESC_RX_META_LEN_MASK)\n-\n-/*\n- * RX path design:\n- *\n- * There are some decisions to take:\n- * 1) How to check DD RX descriptors bit\n- * 2) How and when to allocate new mbufs\n- *\n- * Current implementation checks just one single DD bit each loop. As each\n- * descriptor is 8 bytes, it is likely a good idea to check descriptors in\n- * a single cache line instead. Tests with this change have not shown any\n- * performance improvement but it requires further investigation. For example,\n- * depending on which descriptor is next, the number of descriptors could be\n- * less than 8 for just checking those in the same cache line. This implies\n- * extra work which could be counterproductive by itself. Indeed, last firmware\n- * changes are just doing this: writing several descriptors with the DD bit\n- * for saving PCIe bandwidth and DMA operations from the NFP.\n- *\n- * Mbuf allocation is done when a new packet is received. Then the descriptor\n- * is automatically linked with the new mbuf and the old one is given to the\n- * user. The main drawback with this design is mbuf allocation is heavier than\n- * using bulk allocations allowed by DPDK with rte_mempool_get_bulk. From the\n- * cache point of view it does not seem allocating the mbuf early on as we are\n- * doing now have any benefit at all. Again, tests with this change have not\n- * shown any improvement. Also, rte_mempool_get_bulk returns all or nothing\n- * so looking at the implications of this type of allocation should be studied\n- * deeply\n- */\n-\n-static uint16_t\n-nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)\n-{\n-\tstruct nfp_net_rxq *rxq;\n-\tstruct nfp_net_rx_desc *rxds;\n-\tstruct nfp_net_rx_buff *rxb;\n-\tstruct nfp_net_hw *hw;\n-\tstruct rte_mbuf *mb;\n-\tstruct rte_mbuf *new_mb;\n-\tuint16_t nb_hold;\n-\tuint64_t dma_addr;\n-\tint avail;\n-\n-\trxq = rx_queue;\n-\tif (unlikely(rxq == NULL)) {\n-\t\t/*\n-\t\t * DPDK just checks the queue is lower than max queues\n-\t\t * enabled. But the queue needs to be configured\n-\t\t */\n-\t\tRTE_LOG_DP(ERR, PMD, \"RX Bad queue\\n\");\n-\t\treturn -EINVAL;\n-\t}\n-\n-\thw = rxq->hw;\n-\tavail = 0;\n-\tnb_hold = 0;\n-\n-\twhile (avail < nb_pkts) {\n-\t\trxb = &rxq->rxbufs[rxq->rd_p];\n-\t\tif (unlikely(rxb == NULL)) {\n-\t\t\tRTE_LOG_DP(ERR, PMD, \"rxb does not exist!\\n\");\n-\t\t\tbreak;\n-\t\t}\n-\n-\t\trxds = &rxq->rxds[rxq->rd_p];\n-\t\tif ((rxds->rxd.meta_len_dd & PCIE_DESC_RX_DD) == 0)\n-\t\t\tbreak;\n-\n-\t\t/*\n-\t\t * Memory barrier to ensure that we won't do other\n-\t\t * reads before the DD bit.\n-\t\t */\n-\t\trte_rmb();\n-\n-\t\t/*\n-\t\t * We got a packet. Let's alloc a new mbuf for refilling the\n-\t\t * free descriptor ring as soon as possible\n-\t\t */\n-\t\tnew_mb = rte_pktmbuf_alloc(rxq->mem_pool);\n-\t\tif (unlikely(new_mb == NULL)) {\n-\t\t\tRTE_LOG_DP(DEBUG, PMD,\n-\t\t\t\"RX mbuf alloc failed port_id=%u queue_id=%u\\n\",\n-\t\t\t\trxq->port_id, (unsigned int)rxq->qidx);\n-\t\t\tnfp_net_mbuf_alloc_failed(rxq);\n-\t\t\tbreak;\n-\t\t}\n-\n-\t\tnb_hold++;\n-\n-\t\t/*\n-\t\t * Grab the mbuf and refill the descriptor with the\n-\t\t * previously allocated mbuf\n-\t\t */\n-\t\tmb = rxb->mbuf;\n-\t\trxb->mbuf = new_mb;\n-\n-\t\tPMD_RX_LOG(DEBUG, \"Packet len: %u, mbuf_size: %u\",\n-\t\t\t rxds->rxd.data_len, rxq->mbuf_size);\n-\n-\t\t/* Size of this segment */\n-\t\tmb->data_len = rxds->rxd.data_len - NFP_DESC_META_LEN(rxds);\n-\t\t/* Size of the whole packet. We just support 1 segment */\n-\t\tmb->pkt_len = rxds->rxd.data_len - NFP_DESC_META_LEN(rxds);\n-\n-\t\tif (unlikely((mb->data_len + hw->rx_offset) >\n-\t\t\t rxq->mbuf_size)) {\n-\t\t\t/*\n-\t\t\t * This should not happen and the user has the\n-\t\t\t * responsibility of avoiding it. But we have\n-\t\t\t * to give some info about the error\n-\t\t\t */\n-\t\t\tRTE_LOG_DP(ERR, PMD,\n-\t\t\t\t\"mbuf overflow likely due to the RX offset.\\n\"\n-\t\t\t\t\"\\t\\tYour mbuf size should have extra space for\"\n-\t\t\t\t\" RX offset=%u bytes.\\n\"\n-\t\t\t\t\"\\t\\tCurrently you just have %u bytes available\"\n-\t\t\t\t\" but the received packet is %u bytes long\",\n-\t\t\t\thw->rx_offset,\n-\t\t\t\trxq->mbuf_size - hw->rx_offset,\n-\t\t\t\tmb->data_len);\n-\t\t\treturn -EINVAL;\n-\t\t}\n-\n-\t\t/* Filling the received mbuf with packet info */\n-\t\tif (hw->rx_offset)\n-\t\t\tmb->data_off = RTE_PKTMBUF_HEADROOM + hw->rx_offset;\n-\t\telse\n-\t\t\tmb->data_off = RTE_PKTMBUF_HEADROOM +\n-\t\t\t\t NFP_DESC_META_LEN(rxds);\n-\n-\t\t/* No scatter mode supported */\n-\t\tmb->nb_segs = 1;\n-\t\tmb->next = NULL;\n-\n-\t\tmb->port = rxq->port_id;\n-\n-\t\t/* Checking the RSS flag */\n-\t\tnfp_net_set_hash(rxq, rxds, mb);\n-\n-\t\t/* Checking the checksum flag */\n-\t\tnfp_net_rx_cksum(rxq, rxds, mb);\n-\n-\t\tif ((rxds->rxd.flags & PCIE_DESC_RX_VLAN) &&\n-\t\t (hw->ctrl & NFP_NET_CFG_CTRL_RXVLAN)) {\n-\t\t\tmb->vlan_tci = rte_cpu_to_le_32(rxds->rxd.vlan);\n-\t\t\tmb->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;\n-\t\t}\n-\n-\t\t/* Adding the mbuf to the mbuf array passed by the app */\n-\t\trx_pkts[avail++] = mb;\n-\n-\t\t/* Now resetting and updating the descriptor */\n-\t\trxds->vals[0] = 0;\n-\t\trxds->vals[1] = 0;\n-\t\tdma_addr = rte_cpu_to_le_64(RTE_MBUF_DMA_ADDR_DEFAULT(new_mb));\n-\t\trxds->fld.dd = 0;\n-\t\trxds->fld.dma_addr_hi = (dma_addr >> 32) & 0xff;\n-\t\trxds->fld.dma_addr_lo = dma_addr & 0xffffffff;\n-\n-\t\trxq->rd_p++;\n-\t\tif (unlikely(rxq->rd_p == rxq->rx_count)) /* wrapping?*/\n-\t\t\trxq->rd_p = 0;\n-\t}\n-\n-\tif (nb_hold == 0)\n-\t\treturn nb_hold;\n-\n-\tPMD_RX_LOG(DEBUG, \"RX port_id=%u queue_id=%u, %d packets received\",\n-\t\t rxq->port_id, (unsigned int)rxq->qidx, nb_hold);\n-\n-\tnb_hold += rxq->nb_rx_hold;\n-\n-\t/*\n-\t * FL descriptors needs to be written before incrementing the\n-\t * FL queue WR pointer\n-\t */\n-\trte_wmb();\n-\tif (nb_hold > rxq->rx_free_thresh) {\n-\t\tPMD_RX_LOG(DEBUG, \"port=%u queue=%u nb_hold=%u avail=%u\",\n-\t\t\t rxq->port_id, (unsigned int)rxq->qidx,\n-\t\t\t (unsigned)nb_hold, (unsigned)avail);\n-\t\tnfp_qcp_ptr_add(rxq->qcp_fl, NFP_QCP_WRITE_PTR, nb_hold);\n-\t\tnb_hold = 0;\n-\t}\n-\trxq->nb_rx_hold = nb_hold;\n-\n-\treturn avail;\n-}\n-\n-/*\n- * nfp_net_tx_free_bufs - Check for descriptors with a complete\n- * status\n- * @txq: TX queue to work with\n- * Returns number of descriptors freed\n- */\n-int\n-nfp_net_tx_free_bufs(struct nfp_net_txq *txq)\n-{\n-\tuint32_t qcp_rd_p;\n-\tint todo;\n-\n-\tPMD_TX_LOG(DEBUG, \"queue %u. Check for descriptor with a complete\"\n-\t\t \" status\", txq->qidx);\n-\n-\t/* Work out how many packets have been sent */\n-\tqcp_rd_p = nfp_qcp_read(txq->qcp_q, NFP_QCP_READ_PTR);\n-\n-\tif (qcp_rd_p == txq->rd_p) {\n-\t\tPMD_TX_LOG(DEBUG, \"queue %u: It seems harrier is not sending \"\n-\t\t\t \"packets (%u, %u)\", txq->qidx,\n-\t\t\t qcp_rd_p, txq->rd_p);\n-\t\treturn 0;\n-\t}\n-\n-\tif (qcp_rd_p > txq->rd_p)\n-\t\ttodo = qcp_rd_p - txq->rd_p;\n-\telse\n-\t\ttodo = qcp_rd_p + txq->tx_count - txq->rd_p;\n-\n-\tPMD_TX_LOG(DEBUG, \"qcp_rd_p %u, txq->rd_p: %u, qcp->rd_p: %u\",\n-\t\t qcp_rd_p, txq->rd_p, txq->rd_p);\n-\n-\tif (todo == 0)\n-\t\treturn todo;\n-\n-\ttxq->rd_p += todo;\n-\tif (unlikely(txq->rd_p >= txq->tx_count))\n-\t\ttxq->rd_p -= txq->tx_count;\n-\n-\treturn todo;\n-}\n-\n-/* Leaving always free descriptors for avoiding wrapping confusion */\n-static inline\n-uint32_t nfp_free_tx_desc(struct nfp_net_txq *txq)\n-{\n-\tif (txq->wr_p >= txq->rd_p)\n-\t\treturn txq->tx_count - (txq->wr_p - txq->rd_p) - 8;\n-\telse\n-\t\treturn txq->rd_p - txq->wr_p - 8;\n-}\n-\n-/*\n- * nfp_net_txq_full - Check if the TX queue free descriptors\n- * is below tx_free_threshold\n- *\n- * @txq: TX queue to check\n- *\n- * This function uses the host copy* of read/write pointers\n- */\n-static inline\n-uint32_t nfp_net_txq_full(struct nfp_net_txq *txq)\n-{\n-\treturn (nfp_free_tx_desc(txq) < txq->tx_free_thresh);\n-}\n-\n-static uint16_t\n-nfp_net_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)\n-{\n-\tstruct nfp_net_txq *txq;\n-\tstruct nfp_net_hw *hw;\n-\tstruct nfp_net_tx_desc *txds, txd;\n-\tstruct rte_mbuf *pkt;\n-\tuint64_t dma_addr;\n-\tint pkt_size, dma_size;\n-\tuint16_t free_descs, issued_descs;\n-\tstruct rte_mbuf **lmbuf;\n-\tint i;\n-\n-\ttxq = tx_queue;\n-\thw = txq->hw;\n-\ttxds = &txq->txds[txq->wr_p];\n-\n-\tPMD_TX_LOG(DEBUG, \"working for queue %u at pos %d and %u packets\",\n-\t\t txq->qidx, txq->wr_p, nb_pkts);\n-\n-\tif ((nfp_free_tx_desc(txq) < nb_pkts) || (nfp_net_txq_full(txq)))\n-\t\tnfp_net_tx_free_bufs(txq);\n-\n-\tfree_descs = (uint16_t)nfp_free_tx_desc(txq);\n-\tif (unlikely(free_descs == 0))\n-\t\treturn 0;\n-\n-\tpkt = *tx_pkts;\n-\n-\ti = 0;\n-\tissued_descs = 0;\n-\tPMD_TX_LOG(DEBUG, \"queue: %u. Sending %u packets\",\n-\t\t txq->qidx, nb_pkts);\n-\t/* Sending packets */\n-\twhile ((i < nb_pkts) && free_descs) {\n-\t\t/* Grabbing the mbuf linked to the current descriptor */\n-\t\tlmbuf = &txq->txbufs[txq->wr_p].mbuf;\n-\t\t/* Warming the cache for releasing the mbuf later on */\n-\t\tRTE_MBUF_PREFETCH_TO_FREE(*lmbuf);\n-\n-\t\tpkt = *(tx_pkts + i);\n-\n-\t\tif (unlikely((pkt->nb_segs > 1) &&\n-\t\t\t !(hw->cap & NFP_NET_CFG_CTRL_GATHER))) {\n-\t\t\tPMD_INIT_LOG(INFO, \"NFP_NET_CFG_CTRL_GATHER not set\");\n-\t\t\trte_panic(\"Multisegment packet unsupported\\n\");\n-\t\t}\n-\n-\t\t/* Checking if we have enough descriptors */\n-\t\tif (unlikely(pkt->nb_segs > free_descs))\n-\t\t\tgoto xmit_end;\n-\n-\t\t/*\n-\t\t * Checksum and VLAN flags just in the first descriptor for a\n-\t\t * multisegment packet, but TSO info needs to be in all of them.\n-\t\t */\n-\t\ttxd.data_len = pkt->pkt_len;\n-\t\tnfp_net_tx_tso(txq, &txd, pkt);\n-\t\tnfp_net_tx_cksum(txq, &txd, pkt);\n-\n-\t\tif ((pkt->ol_flags & PKT_TX_VLAN_PKT) &&\n-\t\t (hw->cap & NFP_NET_CFG_CTRL_TXVLAN)) {\n-\t\t\ttxd.flags |= PCIE_DESC_TX_VLAN;\n-\t\t\ttxd.vlan = pkt->vlan_tci;\n-\t\t}\n-\n-\t\t/*\n-\t\t * mbuf data_len is the data in one segment and pkt_len data\n-\t\t * in the whole packet. When the packet is just one segment,\n-\t\t * then data_len = pkt_len\n-\t\t */\n-\t\tpkt_size = pkt->pkt_len;\n-\n-\t\twhile (pkt) {\n-\t\t\t/* Copying TSO, VLAN and cksum info */\n-\t\t\t*txds = txd;\n-\n-\t\t\t/* Releasing mbuf used by this descriptor previously*/\n-\t\t\tif (*lmbuf)\n-\t\t\t\trte_pktmbuf_free_seg(*lmbuf);\n-\n-\t\t\t/*\n-\t\t\t * Linking mbuf with descriptor for being released\n-\t\t\t * next time descriptor is used\n-\t\t\t */\n-\t\t\t*lmbuf = pkt;\n-\n-\t\t\tdma_size = pkt->data_len;\n-\t\t\tdma_addr = rte_mbuf_data_iova(pkt);\n-\t\t\tPMD_TX_LOG(DEBUG, \"Working with mbuf at dma address:\"\n-\t\t\t\t \"%\" PRIx64 \"\", dma_addr);\n-\n-\t\t\t/* Filling descriptors fields */\n-\t\t\ttxds->dma_len = dma_size;\n-\t\t\ttxds->data_len = txd.data_len;\n-\t\t\ttxds->dma_addr_hi = (dma_addr >> 32) & 0xff;\n-\t\t\ttxds->dma_addr_lo = (dma_addr & 0xffffffff);\n-\t\t\tASSERT(free_descs > 0);\n-\t\t\tfree_descs--;\n-\n-\t\t\ttxq->wr_p++;\n-\t\t\tif (unlikely(txq->wr_p == txq->tx_count)) /* wrapping?*/\n-\t\t\t\ttxq->wr_p = 0;\n-\n-\t\t\tpkt_size -= dma_size;\n-\n-\t\t\t/*\n-\t\t\t * Making the EOP, packets with just one segment\n-\t\t\t * the priority\n-\t\t\t */\n-\t\t\tif (likely(!pkt_size))\n-\t\t\t\ttxds->offset_eop = PCIE_DESC_TX_EOP;\n-\t\t\telse\n-\t\t\t\ttxds->offset_eop = 0;\n-\n-\t\t\tpkt = pkt->next;\n-\t\t\t/* Referencing next free TX descriptor */\n-\t\t\ttxds = &txq->txds[txq->wr_p];\n-\t\t\tlmbuf = &txq->txbufs[txq->wr_p].mbuf;\n-\t\t\tissued_descs++;\n-\t\t}\n-\t\ti++;\n-\t}\n-\n-xmit_end:\n-\t/* Increment write pointers. Force memory write before we let HW know */\n-\trte_wmb();\n-\tnfp_qcp_ptr_add(txq->qcp_q, NFP_QCP_WRITE_PTR, issued_descs);\n-\n-\treturn i;\n-}\n-\n static int\n nfp_net_vlan_offload_set(struct rte_eth_dev *dev, int mask)\n {\ndiff --git a/drivers/net/nfp/nfp_net_pmd.h b/drivers/net/nfp/nfp_net_pmd.h\nindex a3a3ba32d6..9265496bf0 100644\n--- a/drivers/net/nfp/nfp_net_pmd.h\n+++ b/drivers/net/nfp/nfp_net_pmd.h\n@@ -41,6 +41,12 @@ struct nfp_net_adapter;\n #define NFP_QCP_QUEUE_STS_HI 0x000c\n #define NFP_QCP_QUEUE_STS_HI_WRITEPTR_mask (0x3ffff)\n \n+/* The offset of the queue controller queues in the PCIe Target */\n+#define NFP_PCIE_QUEUE(_q) (0x80000 + (NFP_QCP_QUEUE_ADDR_SZ * ((_q) & 0xff)))\n+\n+/* Maximum value which can be added to a queue with one transaction */\n+#define NFP_QCP_MAX_ADD\t0x7f\n+\n /* Interrupt definitions */\n #define NFP_NET_IRQ_LSC_IDX 0\n \n@@ -95,47 +101,11 @@ struct nfp_net_adapter;\n #include <linux/types.h>\n #include <rte_io.h>\n \n-static inline uint8_t nn_readb(volatile const void *addr)\n-{\n-\treturn rte_read8(addr);\n-}\n-\n-static inline void nn_writeb(uint8_t val, volatile void *addr)\n-{\n-\trte_write8(val, addr);\n-}\n-\n-static inline uint32_t nn_readl(volatile const void *addr)\n-{\n-\treturn rte_read32(addr);\n-}\n-\n-static inline void nn_writel(uint32_t val, volatile void *addr)\n-{\n-\trte_write32(val, addr);\n-}\n-\n-static inline void nn_writew(uint16_t val, volatile void *addr)\n-{\n-\trte_write16(val, addr);\n-}\n-\n-static inline uint64_t nn_readq(volatile void *addr)\n-{\n-\tconst volatile uint32_t *p = addr;\n-\tuint32_t low, high;\n-\n-\thigh = nn_readl((volatile const void *)(p + 1));\n-\tlow = nn_readl((volatile const void *)p);\n-\n-\treturn low + ((uint64_t)high << 32);\n-}\n-\n-static inline void nn_writeq(uint64_t val, volatile void *addr)\n-{\n-\tnn_writel(val >> 32, (volatile char *)addr + 4);\n-\tnn_writel(val, addr);\n-}\n+/* nfp_qcp_ptr - Read or Write Pointer of a queue */\n+enum nfp_qcp_ptr {\n+\tNFP_QCP_READ_PTR = 0,\n+\tNFP_QCP_WRITE_PTR\n+};\n \n struct nfp_pf_dev {\n \t/* Backpointer to associated pci device */\n@@ -247,6 +217,138 @@ struct nfp_net_adapter {\n \tstruct nfp_net_hw hw;\n };\n \n+static inline uint8_t nn_readb(volatile const void *addr)\n+{\n+\treturn rte_read8(addr);\n+}\n+\n+static inline void nn_writeb(uint8_t val, volatile void *addr)\n+{\n+\trte_write8(val, addr);\n+}\n+\n+static inline uint32_t nn_readl(volatile const void *addr)\n+{\n+\treturn rte_read32(addr);\n+}\n+\n+static inline void nn_writel(uint32_t val, volatile void *addr)\n+{\n+\trte_write32(val, addr);\n+}\n+\n+static inline void nn_writew(uint16_t val, volatile void *addr)\n+{\n+\trte_write16(val, addr);\n+}\n+\n+static inline uint64_t nn_readq(volatile void *addr)\n+{\n+\tconst volatile uint32_t *p = addr;\n+\tuint32_t low, high;\n+\n+\thigh = nn_readl((volatile const void *)(p + 1));\n+\tlow = nn_readl((volatile const void *)p);\n+\n+\treturn low + ((uint64_t)high << 32);\n+}\n+\n+static inline void nn_writeq(uint64_t val, volatile void *addr)\n+{\n+\tnn_writel(val >> 32, (volatile char *)addr + 4);\n+\tnn_writel(val, addr);\n+}\n+\n+/*\n+ * Functions to read/write from/to Config BAR\n+ * Performs any endian conversion necessary.\n+ */\n+static inline uint8_t\n+nn_cfg_readb(struct nfp_net_hw *hw, int off)\n+{\n+\treturn nn_readb(hw->ctrl_bar + off);\n+}\n+\n+static inline void\n+nn_cfg_writeb(struct nfp_net_hw *hw, int off, uint8_t val)\n+{\n+\tnn_writeb(val, hw->ctrl_bar + off);\n+}\n+\n+static inline uint32_t\n+nn_cfg_readl(struct nfp_net_hw *hw, int off)\n+{\n+\treturn rte_le_to_cpu_32(nn_readl(hw->ctrl_bar + off));\n+}\n+\n+static inline void\n+nn_cfg_writel(struct nfp_net_hw *hw, int off, uint32_t val)\n+{\n+\tnn_writel(rte_cpu_to_le_32(val), hw->ctrl_bar + off);\n+}\n+\n+static inline uint64_t\n+nn_cfg_readq(struct nfp_net_hw *hw, int off)\n+{\n+\treturn rte_le_to_cpu_64(nn_readq(hw->ctrl_bar + off));\n+}\n+\n+static inline void\n+nn_cfg_writeq(struct nfp_net_hw *hw, int off, uint64_t val)\n+{\n+\tnn_writeq(rte_cpu_to_le_64(val), hw->ctrl_bar + off);\n+}\n+\n+/*\n+ * nfp_qcp_ptr_add - Add the value to the selected pointer of a queue\n+ * @q: Base address for queue structure\n+ * @ptr: Add to the Read or Write pointer\n+ * @val: Value to add to the queue pointer\n+ *\n+ * If @val is greater than @NFP_QCP_MAX_ADD multiple writes are performed.\n+ */\n+static inline void\n+nfp_qcp_ptr_add(uint8_t *q, enum nfp_qcp_ptr ptr, uint32_t val)\n+{\n+\tuint32_t off;\n+\n+\tif (ptr == NFP_QCP_READ_PTR)\n+\t\toff = NFP_QCP_QUEUE_ADD_RPTR;\n+\telse\n+\t\toff = NFP_QCP_QUEUE_ADD_WPTR;\n+\n+\twhile (val > NFP_QCP_MAX_ADD) {\n+\t\tnn_writel(rte_cpu_to_le_32(NFP_QCP_MAX_ADD), q + off);\n+\t\tval -= NFP_QCP_MAX_ADD;\n+}\n+\n+nn_writel(rte_cpu_to_le_32(val), q + off);\n+}\n+\n+/*\n+ * nfp_qcp_read - Read the current Read/Write pointer value for a queue\n+ * @q: Base address for queue structure\n+ * @ptr: Read or Write pointer\n+ */\n+static inline uint32_t\n+nfp_qcp_read(uint8_t *q, enum nfp_qcp_ptr ptr)\n+{\n+\tuint32_t off;\n+\tuint32_t val;\n+\n+\tif (ptr == NFP_QCP_READ_PTR)\n+\t\toff = NFP_QCP_QUEUE_STS_LO;\n+\telse\n+\t\toff = NFP_QCP_QUEUE_STS_HI;\n+\n+\tval = rte_cpu_to_le_32(nn_readl(q + off));\n+\n+\tif (ptr == NFP_QCP_READ_PTR)\n+\t\treturn val & NFP_QCP_QUEUE_STS_LO_READPTR_mask;\n+\telse\n+\t\treturn val & NFP_QCP_QUEUE_STS_HI_WRITEPTR_mask;\n+}\n+\n #define NFP_NET_DEV_PRIVATE_TO_HW(adapter)\\\n \t(&((struct nfp_net_adapter *)adapter)->hw)\n \ndiff --git a/drivers/net/nfp/nfp_rxtx.c b/drivers/net/nfp/nfp_rxtx.c\nnew file mode 100644\nindex 0000000000..9ee9e5c9a3\n--- /dev/null\n+++ b/drivers/net/nfp/nfp_rxtx.c\n@@ -0,0 +1,1002 @@\n+/* SPDX-License-Identifier: BSD-3-Clause\n+ * Copyright (c) 2014-2021 Netronome Systems, Inc.\n+ * All rights reserved.\n+ *\n+ * Small portions derived from code Copyright(c) 2010-2015 Intel Corporation.\n+ */\n+\n+/*\n+ * vim:shiftwidth=8:noexpandtab\n+ *\n+ * @file dpdk/pmd/nfp_rxtx.c\n+ *\n+ * Netronome vNIC DPDK Poll-Mode Driver: Rx/Tx functions\n+ */\n+\n+#include <ethdev_driver.h>\n+#include <ethdev_pci.h>\n+\n+#include \"nfp_net_pmd.h\"\n+#include \"nfp_rxtx.h\"\n+#include \"nfp_net_logs.h\"\n+#include \"nfp_net_ctrl.h\"\n+\n+/* Prototypes */\n+static int nfp_net_rx_fill_freelist(struct nfp_net_rxq *rxq);\n+static inline void nfp_net_mbuf_alloc_failed(struct nfp_net_rxq *rxq);\n+static inline void nfp_net_set_hash(struct nfp_net_rxq *rxq,\n+\t\t\t\t struct nfp_net_rx_desc *rxd,\n+\t\t\t\t struct rte_mbuf *mbuf);\n+static inline void nfp_net_rx_cksum(struct nfp_net_rxq *rxq,\n+\t\t\t\t struct nfp_net_rx_desc *rxd,\n+\t\t\t\t struct rte_mbuf *mb);\n+static void nfp_net_rx_queue_release_mbufs(struct nfp_net_rxq *rxq);\n+static int nfp_net_tx_free_bufs(struct nfp_net_txq *txq);\n+static void nfp_net_tx_queue_release_mbufs(struct nfp_net_txq *txq);\n+static inline uint32_t nfp_free_tx_desc(struct nfp_net_txq *txq);\n+static inline uint32_t nfp_net_txq_full(struct nfp_net_txq *txq);\n+static inline void nfp_net_tx_tso(struct nfp_net_txq *txq,\n+\t\t\t\t struct nfp_net_tx_desc *txd,\n+\t\t\t\t struct rte_mbuf *mb);\n+static inline void nfp_net_tx_cksum(struct nfp_net_txq *txq,\n+\t\t\t\t struct nfp_net_tx_desc *txd,\n+\t\t\t\t struct rte_mbuf *mb);\n+\n+static int\n+nfp_net_rx_fill_freelist(struct nfp_net_rxq *rxq)\n+{\n+\tstruct nfp_net_rx_buff *rxe = rxq->rxbufs;\n+\tuint64_t dma_addr;\n+\tunsigned int i;\n+\n+\tPMD_RX_LOG(DEBUG, \"Fill Rx Freelist for %u descriptors\",\n+\t\t rxq->rx_count);\n+\n+\tfor (i = 0; i < rxq->rx_count; i++) {\n+\t\tstruct nfp_net_rx_desc *rxd;\n+\t\tstruct rte_mbuf *mbuf = rte_pktmbuf_alloc(rxq->mem_pool);\n+\n+\t\tif (mbuf == NULL) {\n+\t\t\tPMD_DRV_LOG(ERR, \"RX mbuf alloc failed queue_id=%u\",\n+\t\t\t\t(unsigned int)rxq->qidx);\n+\t\t\treturn -ENOMEM;\n+\t\t}\n+\n+\t\tdma_addr = rte_cpu_to_le_64(RTE_MBUF_DMA_ADDR_DEFAULT(mbuf));\n+\n+\t\trxd = &rxq->rxds[i];\n+\t\trxd->fld.dd = 0;\n+\t\trxd->fld.dma_addr_hi = (dma_addr >> 32) & 0xff;\n+\t\trxd->fld.dma_addr_lo = dma_addr & 0xffffffff;\n+\t\trxe[i].mbuf = mbuf;\n+\t\tPMD_RX_LOG(DEBUG, \"[%d]: %\" PRIx64, i, dma_addr);\n+\t}\n+\n+\t/* Make sure all writes are flushed before telling the hardware */\n+\trte_wmb();\n+\n+\t/* Not advertising the whole ring as the firmware gets confused if so */\n+\tPMD_RX_LOG(DEBUG, \"Increment FL write pointer in %u\",\n+\t\t rxq->rx_count - 1);\n+\n+\tnfp_qcp_ptr_add(rxq->qcp_fl, NFP_QCP_WRITE_PTR, rxq->rx_count - 1);\n+\n+\treturn 0;\n+}\n+\n+int\n+nfp_net_rx_freelist_setup(struct rte_eth_dev *dev)\n+{\n+\tint i;\n+\n+\tfor (i = 0; i < dev->data->nb_rx_queues; i++) {\n+\t\tif (nfp_net_rx_fill_freelist(dev->data->rx_queues[i]) < 0)\n+\t\t\treturn -1;\n+\t}\n+\treturn 0;\n+}\n+\n+uint32_t\n+nfp_net_rx_queue_count(struct rte_eth_dev *dev, uint16_t queue_idx)\n+{\n+\tstruct nfp_net_rxq *rxq;\n+\tstruct nfp_net_rx_desc *rxds;\n+\tuint32_t idx;\n+\tuint32_t count;\n+\n+\trxq = (struct nfp_net_rxq *)dev->data->rx_queues[queue_idx];\n+\n+\tidx = rxq->rd_p;\n+\n+\tcount = 0;\n+\n+\t/*\n+\t * Other PMDs are just checking the DD bit in intervals of 4\n+\t * descriptors and counting all four if the first has the DD\n+\t * bit on. Of course, this is not accurate but can be good for\n+\t * performance. But ideally that should be done in descriptors\n+\t * chunks belonging to the same cache line\n+\t */\n+\n+\twhile (count < rxq->rx_count) {\n+\t\trxds = &rxq->rxds[idx];\n+\t\tif ((rxds->rxd.meta_len_dd & PCIE_DESC_RX_DD) == 0)\n+\t\t\tbreak;\n+\n+\t\tcount++;\n+\t\tidx++;\n+\n+\t\t/* Wrapping? */\n+\t\tif ((idx) == rxq->rx_count)\n+\t\t\tidx = 0;\n+\t}\n+\n+\treturn count;\n+}\n+\n+static inline void\n+nfp_net_mbuf_alloc_failed(struct nfp_net_rxq *rxq)\n+{\n+\trte_eth_devices[rxq->port_id].data->rx_mbuf_alloc_failed++;\n+}\n+\n+/*\n+ * nfp_net_set_hash - Set mbuf hash data\n+ *\n+ * The RSS hash and hash-type are pre-pended to the packet data.\n+ * Extract and decode it and set the mbuf fields.\n+ */\n+static inline void\n+nfp_net_set_hash(struct nfp_net_rxq *rxq, struct nfp_net_rx_desc *rxd,\n+\t\t struct rte_mbuf *mbuf)\n+{\n+\tstruct nfp_net_hw *hw = rxq->hw;\n+\tuint8_t *meta_offset;\n+\tuint32_t meta_info;\n+\tuint32_t hash = 0;\n+\tuint32_t hash_type = 0;\n+\n+\tif (!(hw->ctrl & NFP_NET_CFG_CTRL_RSS))\n+\t\treturn;\n+\n+\t/* this is true for new firmwares */\n+\tif (likely(((hw->cap & NFP_NET_CFG_CTRL_RSS2) ||\n+\t (NFD_CFG_MAJOR_VERSION_of(hw->ver) == 4)) &&\n+\t NFP_DESC_META_LEN(rxd))) {\n+\t\t/*\n+\t\t * new metadata api:\n+\t\t * <---- 32 bit ----->\n+\t\t * m field type word\n+\t\t * e data field #2\n+\t\t * t data field #1\n+\t\t * a data field #0\n+\t\t * ====================\n+\t\t * packet data\n+\t\t *\n+\t\t * Field type word contains up to 8 4bit field types\n+\t\t * A 4bit field type refers to a data field word\n+\t\t * A data field word can have several 4bit field types\n+\t\t */\n+\t\tmeta_offset = rte_pktmbuf_mtod(mbuf, uint8_t *);\n+\t\tmeta_offset -= NFP_DESC_META_LEN(rxd);\n+\t\tmeta_info = rte_be_to_cpu_32(*(uint32_t *)meta_offset);\n+\t\tmeta_offset += 4;\n+\t\t/* NFP PMD just supports metadata for hashing */\n+\t\tswitch (meta_info & NFP_NET_META_FIELD_MASK) {\n+\t\tcase NFP_NET_META_HASH:\n+\t\t\t/* next field type is about the hash type */\n+\t\t\tmeta_info >>= NFP_NET_META_FIELD_SIZE;\n+\t\t\t/* hash value is in the data field */\n+\t\t\thash = rte_be_to_cpu_32(*(uint32_t *)meta_offset);\n+\t\t\thash_type = meta_info & NFP_NET_META_FIELD_MASK;\n+\t\t\tbreak;\n+\t\tdefault:\n+\t\t\t/* Unsupported metadata can be a performance issue */\n+\t\t\treturn;\n+\t\t}\n+\t} else {\n+\t\tif (!(rxd->rxd.flags & PCIE_DESC_RX_RSS))\n+\t\t\treturn;\n+\n+\t\thash = rte_be_to_cpu_32(*(uint32_t *)NFP_HASH_OFFSET);\n+\t\thash_type = rte_be_to_cpu_32(*(uint32_t *)NFP_HASH_TYPE_OFFSET);\n+\t}\n+\n+\tmbuf->hash.rss = hash;\n+\tmbuf->ol_flags |= PKT_RX_RSS_HASH;\n+\n+\tswitch (hash_type) {\n+\tcase NFP_NET_RSS_IPV4:\n+\t\tmbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV4;\n+\t\tbreak;\n+\tcase NFP_NET_RSS_IPV6:\n+\t\tmbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6;\n+\t\tbreak;\n+\tcase NFP_NET_RSS_IPV6_EX:\n+\t\tmbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT;\n+\t\tbreak;\n+\tcase NFP_NET_RSS_IPV4_TCP:\n+\t\tmbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT;\n+\t\tbreak;\n+\tcase NFP_NET_RSS_IPV6_TCP:\n+\t\tmbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT;\n+\t\tbreak;\n+\tcase NFP_NET_RSS_IPV4_UDP:\n+\t\tmbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT;\n+\t\tbreak;\n+\tcase NFP_NET_RSS_IPV6_UDP:\n+\t\tmbuf->packet_type |= RTE_PTYPE_INNER_L3_IPV6_EXT;\n+\t\tbreak;\n+\tdefault:\n+\t\tmbuf->packet_type |= RTE_PTYPE_INNER_L4_MASK;\n+\t}\n+}\n+\n+/* nfp_net_rx_cksum - set mbuf checksum flags based on RX descriptor flags */\n+static inline void\n+nfp_net_rx_cksum(struct nfp_net_rxq *rxq, struct nfp_net_rx_desc *rxd,\n+\t\t struct rte_mbuf *mb)\n+{\n+\tstruct nfp_net_hw *hw = rxq->hw;\n+\n+\tif (!(hw->ctrl & NFP_NET_CFG_CTRL_RXCSUM))\n+\t\treturn;\n+\n+\t/* If IPv4 and IP checksum error, fail */\n+\tif (unlikely((rxd->rxd.flags & PCIE_DESC_RX_IP4_CSUM) &&\n+\t !(rxd->rxd.flags & PCIE_DESC_RX_IP4_CSUM_OK)))\n+\t\tmb->ol_flags |= PKT_RX_IP_CKSUM_BAD;\n+\telse\n+\t\tmb->ol_flags |= PKT_RX_IP_CKSUM_GOOD;\n+\n+\t/* If neither UDP nor TCP return */\n+\tif (!(rxd->rxd.flags & PCIE_DESC_RX_TCP_CSUM) &&\n+\t !(rxd->rxd.flags & PCIE_DESC_RX_UDP_CSUM))\n+\t\treturn;\n+\n+\tif (likely(rxd->rxd.flags & PCIE_DESC_RX_L4_CSUM_OK))\n+\t\tmb->ol_flags |= PKT_RX_L4_CKSUM_GOOD;\n+\telse\n+\t\tmb->ol_flags |= PKT_RX_L4_CKSUM_BAD;\n+}\n+\n+/*\n+ * RX path design:\n+ *\n+ * There are some decisions to take:\n+ * 1) How to check DD RX descriptors bit\n+ * 2) How and when to allocate new mbufs\n+ *\n+ * Current implementation checks just one single DD bit each loop. As each\n+ * descriptor is 8 bytes, it is likely a good idea to check descriptors in\n+ * a single cache line instead. Tests with this change have not shown any\n+ * performance improvement but it requires further investigation. For example,\n+ * depending on which descriptor is next, the number of descriptors could be\n+ * less than 8 for just checking those in the same cache line. This implies\n+ * extra work which could be counterproductive by itself. Indeed, last firmware\n+ * changes are just doing this: writing several descriptors with the DD bit\n+ * for saving PCIe bandwidth and DMA operations from the NFP.\n+ *\n+ * Mbuf allocation is done when a new packet is received. Then the descriptor\n+ * is automatically linked with the new mbuf and the old one is given to the\n+ * user. The main drawback with this design is mbuf allocation is heavier than\n+ * using bulk allocations allowed by DPDK with rte_mempool_get_bulk. From the\n+ * cache point of view it does not seem allocating the mbuf early on as we are\n+ * doing now have any benefit at all. Again, tests with this change have not\n+ * shown any improvement. Also, rte_mempool_get_bulk returns all or nothing\n+ * so looking at the implications of this type of allocation should be studied\n+ * deeply\n+ */\n+\n+uint16_t\n+nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, uint16_t nb_pkts)\n+{\n+\tstruct nfp_net_rxq *rxq;\n+\tstruct nfp_net_rx_desc *rxds;\n+\tstruct nfp_net_rx_buff *rxb;\n+\tstruct nfp_net_hw *hw;\n+\tstruct rte_mbuf *mb;\n+\tstruct rte_mbuf *new_mb;\n+\tuint16_t nb_hold;\n+\tuint64_t dma_addr;\n+\tint avail;\n+\n+\trxq = rx_queue;\n+\tif (unlikely(rxq == NULL)) {\n+\t\t/*\n+\t\t * DPDK just checks the queue is lower than max queues\n+\t\t * enabled. But the queue needs to be configured\n+\t\t */\n+\t\tRTE_LOG_DP(ERR, PMD, \"RX Bad queue\\n\");\n+\t\treturn -EINVAL;\n+\t}\n+\n+\thw = rxq->hw;\n+\tavail = 0;\n+\tnb_hold = 0;\n+\n+\twhile (avail < nb_pkts) {\n+\t\trxb = &rxq->rxbufs[rxq->rd_p];\n+\t\tif (unlikely(rxb == NULL)) {\n+\t\t\tRTE_LOG_DP(ERR, PMD, \"rxb does not exist!\\n\");\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\trxds = &rxq->rxds[rxq->rd_p];\n+\t\tif ((rxds->rxd.meta_len_dd & PCIE_DESC_RX_DD) == 0)\n+\t\t\tbreak;\n+\n+\t\t/*\n+\t\t * Memory barrier to ensure that we won't do other\n+\t\t * reads before the DD bit.\n+\t\t */\n+\t\trte_rmb();\n+\n+\t\t/*\n+\t\t * We got a packet. Let's alloc a new mbuf for refilling the\n+\t\t * free descriptor ring as soon as possible\n+\t\t */\n+\t\tnew_mb = rte_pktmbuf_alloc(rxq->mem_pool);\n+\t\tif (unlikely(new_mb == NULL)) {\n+\t\t\tRTE_LOG_DP(DEBUG, PMD,\n+\t\t\t\"RX mbuf alloc failed port_id=%u queue_id=%u\\n\",\n+\t\t\t\trxq->port_id, (unsigned int)rxq->qidx);\n+\t\t\tnfp_net_mbuf_alloc_failed(rxq);\n+\t\t\tbreak;\n+\t\t}\n+\n+\t\tnb_hold++;\n+\n+\t\t/*\n+\t\t * Grab the mbuf and refill the descriptor with the\n+\t\t * previously allocated mbuf\n+\t\t */\n+\t\tmb = rxb->mbuf;\n+\t\trxb->mbuf = new_mb;\n+\n+\t\tPMD_RX_LOG(DEBUG, \"Packet len: %u, mbuf_size: %u\",\n+\t\t\t rxds->rxd.data_len, rxq->mbuf_size);\n+\n+\t\t/* Size of this segment */\n+\t\tmb->data_len = rxds->rxd.data_len - NFP_DESC_META_LEN(rxds);\n+\t\t/* Size of the whole packet. We just support 1 segment */\n+\t\tmb->pkt_len = rxds->rxd.data_len - NFP_DESC_META_LEN(rxds);\n+\n+\t\tif (unlikely((mb->data_len + hw->rx_offset) >\n+\t\t\t rxq->mbuf_size)) {\n+\t\t\t/*\n+\t\t\t * This should not happen and the user has the\n+\t\t\t * responsibility of avoiding it. But we have\n+\t\t\t * to give some info about the error\n+\t\t\t */\n+\t\t\tRTE_LOG_DP(ERR, PMD,\n+\t\t\t\t\"mbuf overflow likely due to the RX offset.\\n\"\n+\t\t\t\t\"\\t\\tYour mbuf size should have extra space for\"\n+\t\t\t\t\" RX offset=%u bytes.\\n\"\n+\t\t\t\t\"\\t\\tCurrently you just have %u bytes available\"\n+\t\t\t\t\" but the received packet is %u bytes long\",\n+\t\t\t\thw->rx_offset,\n+\t\t\t\trxq->mbuf_size - hw->rx_offset,\n+\t\t\t\tmb->data_len);\n+\t\t\treturn -EINVAL;\n+\t\t}\n+\n+\t\t/* Filling the received mbuf with packet info */\n+\t\tif (hw->rx_offset)\n+\t\t\tmb->data_off = RTE_PKTMBUF_HEADROOM + hw->rx_offset;\n+\t\telse\n+\t\t\tmb->data_off = RTE_PKTMBUF_HEADROOM +\n+\t\t\t\t NFP_DESC_META_LEN(rxds);\n+\n+\t\t/* No scatter mode supported */\n+\t\tmb->nb_segs = 1;\n+\t\tmb->next = NULL;\n+\n+\t\tmb->port = rxq->port_id;\n+\n+\t\t/* Checking the RSS flag */\n+\t\tnfp_net_set_hash(rxq, rxds, mb);\n+\n+\t\t/* Checking the checksum flag */\n+\t\tnfp_net_rx_cksum(rxq, rxds, mb);\n+\n+\t\tif ((rxds->rxd.flags & PCIE_DESC_RX_VLAN) &&\n+\t\t (hw->ctrl & NFP_NET_CFG_CTRL_RXVLAN)) {\n+\t\t\tmb->vlan_tci = rte_cpu_to_le_32(rxds->rxd.vlan);\n+\t\t\tmb->ol_flags |= PKT_RX_VLAN | PKT_RX_VLAN_STRIPPED;\n+\t\t}\n+\n+\t\t/* Adding the mbuf to the mbuf array passed by the app */\n+\t\trx_pkts[avail++] = mb;\n+\n+\t\t/* Now resetting and updating the descriptor */\n+\t\trxds->vals[0] = 0;\n+\t\trxds->vals[1] = 0;\n+\t\tdma_addr = rte_cpu_to_le_64(RTE_MBUF_DMA_ADDR_DEFAULT(new_mb));\n+\t\trxds->fld.dd = 0;\n+\t\trxds->fld.dma_addr_hi = (dma_addr >> 32) & 0xff;\n+\t\trxds->fld.dma_addr_lo = dma_addr & 0xffffffff;\n+\n+\t\trxq->rd_p++;\n+\t\tif (unlikely(rxq->rd_p == rxq->rx_count)) /* wrapping?*/\n+\t\t\trxq->rd_p = 0;\n+\t}\n+\n+\tif (nb_hold == 0)\n+\t\treturn nb_hold;\n+\n+\tPMD_RX_LOG(DEBUG, \"RX port_id=%u queue_id=%u, %d packets received\",\n+\t\t rxq->port_id, (unsigned int)rxq->qidx, nb_hold);\n+\n+\tnb_hold += rxq->nb_rx_hold;\n+\n+\t/*\n+\t * FL descriptors needs to be written before incrementing the\n+\t * FL queue WR pointer\n+\t */\n+\trte_wmb();\n+\tif (nb_hold > rxq->rx_free_thresh) {\n+\t\tPMD_RX_LOG(DEBUG, \"port=%u queue=%u nb_hold=%u avail=%u\",\n+\t\t\t rxq->port_id, (unsigned int)rxq->qidx,\n+\t\t\t (unsigned int)nb_hold, (unsigned int)avail);\n+\t\tnfp_qcp_ptr_add(rxq->qcp_fl, NFP_QCP_WRITE_PTR, nb_hold);\n+\t\tnb_hold = 0;\n+\t}\n+\trxq->nb_rx_hold = nb_hold;\n+\n+\treturn avail;\n+}\n+\n+static void\n+nfp_net_rx_queue_release_mbufs(struct nfp_net_rxq *rxq)\n+{\n+\tunsigned int i;\n+\n+\tif (rxq->rxbufs == NULL)\n+\t\treturn;\n+\n+\tfor (i = 0; i < rxq->rx_count; i++) {\n+\t\tif (rxq->rxbufs[i].mbuf) {\n+\t\t\trte_pktmbuf_free_seg(rxq->rxbufs[i].mbuf);\n+\t\t\trxq->rxbufs[i].mbuf = NULL;\n+\t\t}\n+\t}\n+}\n+\n+void\n+nfp_net_rx_queue_release(void *rx_queue)\n+{\n+\tstruct nfp_net_rxq *rxq = rx_queue;\n+\n+\tif (rxq) {\n+\t\tnfp_net_rx_queue_release_mbufs(rxq);\n+\t\trte_free(rxq->rxbufs);\n+\t\trte_free(rxq);\n+\t}\n+}\n+\n+void\n+nfp_net_reset_rx_queue(struct nfp_net_rxq *rxq)\n+{\n+\tnfp_net_rx_queue_release_mbufs(rxq);\n+\trxq->rd_p = 0;\n+\trxq->nb_rx_hold = 0;\n+}\n+\n+int\n+nfp_net_rx_queue_setup(struct rte_eth_dev *dev,\n+\t\t uint16_t queue_idx, uint16_t nb_desc,\n+\t\t unsigned int socket_id,\n+\t\t const struct rte_eth_rxconf *rx_conf,\n+\t\t struct rte_mempool *mp)\n+{\n+\tconst struct rte_memzone *tz;\n+\tstruct nfp_net_rxq *rxq;\n+\tstruct nfp_net_hw *hw;\n+\tuint32_t rx_desc_sz;\n+\n+\thw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);\n+\n+\tPMD_INIT_FUNC_TRACE();\n+\n+\t/* Validating number of descriptors */\n+\trx_desc_sz = nb_desc * sizeof(struct nfp_net_rx_desc);\n+\tif (rx_desc_sz % NFP_ALIGN_RING_DESC != 0 ||\n+\t nb_desc > NFP_NET_MAX_RX_DESC ||\n+\t nb_desc < NFP_NET_MIN_RX_DESC) {\n+\t\tPMD_DRV_LOG(ERR, \"Wrong nb_desc value\");\n+\t\treturn -EINVAL;\n+\t}\n+\n+\t/*\n+\t * Free memory prior to re-allocation if needed. This is the case after\n+\t * calling nfp_net_stop\n+\t */\n+\tif (dev->data->rx_queues[queue_idx]) {\n+\t\tnfp_net_rx_queue_release(dev->data->rx_queues[queue_idx]);\n+\t\tdev->data->rx_queues[queue_idx] = NULL;\n+\t}\n+\n+\t/* Allocating rx queue data structure */\n+\trxq = rte_zmalloc_socket(\"ethdev RX queue\", sizeof(struct nfp_net_rxq),\n+\t\t\t\t RTE_CACHE_LINE_SIZE, socket_id);\n+\tif (rxq == NULL)\n+\t\treturn -ENOMEM;\n+\n+\t/* Hw queues mapping based on firmware configuration */\n+\trxq->qidx = queue_idx;\n+\trxq->fl_qcidx = queue_idx * hw->stride_rx;\n+\trxq->rx_qcidx = rxq->fl_qcidx + (hw->stride_rx - 1);\n+\trxq->qcp_fl = hw->rx_bar + NFP_QCP_QUEUE_OFF(rxq->fl_qcidx);\n+\trxq->qcp_rx = hw->rx_bar + NFP_QCP_QUEUE_OFF(rxq->rx_qcidx);\n+\n+\t/*\n+\t * Tracking mbuf size for detecting a potential mbuf overflow due to\n+\t * RX offset\n+\t */\n+\trxq->mem_pool = mp;\n+\trxq->mbuf_size = rxq->mem_pool->elt_size;\n+\trxq->mbuf_size -= (sizeof(struct rte_mbuf) + RTE_PKTMBUF_HEADROOM);\n+\thw->flbufsz = rxq->mbuf_size;\n+\n+\trxq->rx_count = nb_desc;\n+\trxq->port_id = dev->data->port_id;\n+\trxq->rx_free_thresh = rx_conf->rx_free_thresh;\n+\trxq->drop_en = rx_conf->rx_drop_en;\n+\n+\t/*\n+\t * Allocate RX ring hardware descriptors. A memzone large enough to\n+\t * handle the maximum ring size is allocated in order to allow for\n+\t * resizing in later calls to the queue setup function.\n+\t */\n+\ttz = rte_eth_dma_zone_reserve(dev, \"rx_ring\", queue_idx,\n+\t\t\t\t sizeof(struct nfp_net_rx_desc) *\n+\t\t\t\t NFP_NET_MAX_RX_DESC, NFP_MEMZONE_ALIGN,\n+\t\t\t\t socket_id);\n+\n+\tif (tz == NULL) {\n+\t\tPMD_DRV_LOG(ERR, \"Error allocating rx dma\");\n+\t\tnfp_net_rx_queue_release(rxq);\n+\t\treturn -ENOMEM;\n+\t}\n+\n+\t/* Saving physical and virtual addresses for the RX ring */\n+\trxq->dma = (uint64_t)tz->iova;\n+\trxq->rxds = (struct nfp_net_rx_desc *)tz->addr;\n+\n+\t/* mbuf pointers array for referencing mbufs linked to RX descriptors */\n+\trxq->rxbufs = rte_zmalloc_socket(\"rxq->rxbufs\",\n+\t\t\t\t\t sizeof(*rxq->rxbufs) * nb_desc,\n+\t\t\t\t\t RTE_CACHE_LINE_SIZE, socket_id);\n+\tif (rxq->rxbufs == NULL) {\n+\t\tnfp_net_rx_queue_release(rxq);\n+\t\treturn -ENOMEM;\n+\t}\n+\n+\tPMD_RX_LOG(DEBUG, \"rxbufs=%p hw_ring=%p dma_addr=0x%\" PRIx64,\n+\t\t rxq->rxbufs, rxq->rxds, (unsigned long)rxq->dma);\n+\n+\tnfp_net_reset_rx_queue(rxq);\n+\n+\tdev->data->rx_queues[queue_idx] = rxq;\n+\trxq->hw = hw;\n+\n+\t/*\n+\t * Telling the HW about the physical address of the RX ring and number\n+\t * of descriptors in log2 format\n+\t */\n+\tnn_cfg_writeq(hw, NFP_NET_CFG_RXR_ADDR(queue_idx), rxq->dma);\n+\tnn_cfg_writeb(hw, NFP_NET_CFG_RXR_SZ(queue_idx), rte_log2_u32(nb_desc));\n+\n+\treturn 0;\n+}\n+\n+/*\n+ * nfp_net_tx_free_bufs - Check for descriptors with a complete\n+ * status\n+ * @txq: TX queue to work with\n+ * Returns number of descriptors freed\n+ */\n+static int\n+nfp_net_tx_free_bufs(struct nfp_net_txq *txq)\n+{\n+\tuint32_t qcp_rd_p;\n+\tint todo;\n+\n+\tPMD_TX_LOG(DEBUG, \"queue %u. Check for descriptor with a complete\"\n+\t\t \" status\", txq->qidx);\n+\n+\t/* Work out how many packets have been sent */\n+\tqcp_rd_p = nfp_qcp_read(txq->qcp_q, NFP_QCP_READ_PTR);\n+\n+\tif (qcp_rd_p == txq->rd_p) {\n+\t\tPMD_TX_LOG(DEBUG, \"queue %u: It seems harrier is not sending \"\n+\t\t\t \"packets (%u, %u)\", txq->qidx,\n+\t\t\t qcp_rd_p, txq->rd_p);\n+\t\treturn 0;\n+\t}\n+\n+\tif (qcp_rd_p > txq->rd_p)\n+\t\ttodo = qcp_rd_p - txq->rd_p;\n+\telse\n+\t\ttodo = qcp_rd_p + txq->tx_count - txq->rd_p;\n+\n+\tPMD_TX_LOG(DEBUG, \"qcp_rd_p %u, txq->rd_p: %u, qcp->rd_p: %u\",\n+\t\t qcp_rd_p, txq->rd_p, txq->rd_p);\n+\n+\tif (todo == 0)\n+\t\treturn todo;\n+\n+\ttxq->rd_p += todo;\n+\tif (unlikely(txq->rd_p >= txq->tx_count))\n+\t\ttxq->rd_p -= txq->tx_count;\n+\n+\treturn todo;\n+}\n+\n+static void\n+nfp_net_tx_queue_release_mbufs(struct nfp_net_txq *txq)\n+{\n+\tunsigned int i;\n+\n+\tif (txq->txbufs == NULL)\n+\t\treturn;\n+\n+\tfor (i = 0; i < txq->tx_count; i++) {\n+\t\tif (txq->txbufs[i].mbuf) {\n+\t\t\trte_pktmbuf_free_seg(txq->txbufs[i].mbuf);\n+\t\t\ttxq->txbufs[i].mbuf = NULL;\n+\t\t}\n+\t}\n+}\n+\n+void\n+nfp_net_tx_queue_release(void *tx_queue)\n+{\n+\tstruct nfp_net_txq *txq = tx_queue;\n+\n+\tif (txq) {\n+\t\tnfp_net_tx_queue_release_mbufs(txq);\n+\t\trte_free(txq->txbufs);\n+\t\trte_free(txq);\n+\t}\n+}\n+\n+void\n+nfp_net_reset_tx_queue(struct nfp_net_txq *txq)\n+{\n+\tnfp_net_tx_queue_release_mbufs(txq);\n+\ttxq->wr_p = 0;\n+\ttxq->rd_p = 0;\n+}\n+\n+int\n+nfp_net_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,\n+\t\t uint16_t nb_desc, unsigned int socket_id,\n+\t\t const struct rte_eth_txconf *tx_conf)\n+{\n+\tconst struct rte_memzone *tz;\n+\tstruct nfp_net_txq *txq;\n+\tuint16_t tx_free_thresh;\n+\tstruct nfp_net_hw *hw;\n+\tuint32_t tx_desc_sz;\n+\n+\thw = NFP_NET_DEV_PRIVATE_TO_HW(dev->data->dev_private);\n+\n+\tPMD_INIT_FUNC_TRACE();\n+\n+\t/* Validating number of descriptors */\n+\ttx_desc_sz = nb_desc * sizeof(struct nfp_net_tx_desc);\n+\tif (tx_desc_sz % NFP_ALIGN_RING_DESC != 0 ||\n+\t nb_desc > NFP_NET_MAX_TX_DESC ||\n+\t nb_desc < NFP_NET_MIN_TX_DESC) {\n+\t\tPMD_DRV_LOG(ERR, \"Wrong nb_desc value\");\n+\t\treturn -EINVAL;\n+\t}\n+\n+\ttx_free_thresh = (uint16_t)((tx_conf->tx_free_thresh) ?\n+\t\t\t\t tx_conf->tx_free_thresh :\n+\t\t\t\t DEFAULT_TX_FREE_THRESH);\n+\n+\tif (tx_free_thresh > (nb_desc)) {\n+\t\tPMD_DRV_LOG(ERR,\n+\t\t\t\"tx_free_thresh must be less than the number of TX \"\n+\t\t\t\"descriptors. (tx_free_thresh=%u port=%d \"\n+\t\t\t\"queue=%d)\", (unsigned int)tx_free_thresh,\n+\t\t\tdev->data->port_id, (int)queue_idx);\n+\t\treturn -(EINVAL);\n+\t}\n+\n+\t/*\n+\t * Free memory prior to re-allocation if needed. This is the case after\n+\t * calling nfp_net_stop\n+\t */\n+\tif (dev->data->tx_queues[queue_idx]) {\n+\t\tPMD_TX_LOG(DEBUG, \"Freeing memory prior to re-allocation %d\",\n+\t\t\t queue_idx);\n+\t\tnfp_net_tx_queue_release(dev->data->tx_queues[queue_idx]);\n+\t\tdev->data->tx_queues[queue_idx] = NULL;\n+\t}\n+\n+\t/* Allocating tx queue data structure */\n+\ttxq = rte_zmalloc_socket(\"ethdev TX queue\", sizeof(struct nfp_net_txq),\n+\t\t\t\t RTE_CACHE_LINE_SIZE, socket_id);\n+\tif (txq == NULL) {\n+\t\tPMD_DRV_LOG(ERR, \"Error allocating tx dma\");\n+\t\treturn -ENOMEM;\n+\t}\n+\n+\t/*\n+\t * Allocate TX ring hardware descriptors. A memzone large enough to\n+\t * handle the maximum ring size is allocated in order to allow for\n+\t * resizing in later calls to the queue setup function.\n+\t */\n+\ttz = rte_eth_dma_zone_reserve(dev, \"tx_ring\", queue_idx,\n+\t\t\t\t sizeof(struct nfp_net_tx_desc) *\n+\t\t\t\t NFP_NET_MAX_TX_DESC, NFP_MEMZONE_ALIGN,\n+\t\t\t\t socket_id);\n+\tif (tz == NULL) {\n+\t\tPMD_DRV_LOG(ERR, \"Error allocating tx dma\");\n+\t\tnfp_net_tx_queue_release(txq);\n+\t\treturn -ENOMEM;\n+\t}\n+\n+\ttxq->tx_count = nb_desc;\n+\ttxq->tx_free_thresh = tx_free_thresh;\n+\ttxq->tx_pthresh = tx_conf->tx_thresh.pthresh;\n+\ttxq->tx_hthresh = tx_conf->tx_thresh.hthresh;\n+\ttxq->tx_wthresh = tx_conf->tx_thresh.wthresh;\n+\n+\t/* queue mapping based on firmware configuration */\n+\ttxq->qidx = queue_idx;\n+\ttxq->tx_qcidx = queue_idx * hw->stride_tx;\n+\ttxq->qcp_q = hw->tx_bar + NFP_QCP_QUEUE_OFF(txq->tx_qcidx);\n+\n+\ttxq->port_id = dev->data->port_id;\n+\n+\t/* Saving physical and virtual addresses for the TX ring */\n+\ttxq->dma = (uint64_t)tz->iova;\n+\ttxq->txds = (struct nfp_net_tx_desc *)tz->addr;\n+\n+\t/* mbuf pointers array for referencing mbufs linked to TX descriptors */\n+\ttxq->txbufs = rte_zmalloc_socket(\"txq->txbufs\",\n+\t\t\t\t\t sizeof(*txq->txbufs) * nb_desc,\n+\t\t\t\t\t RTE_CACHE_LINE_SIZE, socket_id);\n+\tif (txq->txbufs == NULL) {\n+\t\tnfp_net_tx_queue_release(txq);\n+\t\treturn -ENOMEM;\n+\t}\n+\tPMD_TX_LOG(DEBUG, \"txbufs=%p hw_ring=%p dma_addr=0x%\" PRIx64,\n+\t\t txq->txbufs, txq->txds, (unsigned long)txq->dma);\n+\n+\tnfp_net_reset_tx_queue(txq);\n+\n+\tdev->data->tx_queues[queue_idx] = txq;\n+\ttxq->hw = hw;\n+\n+\t/*\n+\t * Telling the HW about the physical address of the TX ring and number\n+\t * of descriptors in log2 format\n+\t */\n+\tnn_cfg_writeq(hw, NFP_NET_CFG_TXR_ADDR(queue_idx), txq->dma);\n+\tnn_cfg_writeb(hw, NFP_NET_CFG_TXR_SZ(queue_idx), rte_log2_u32(nb_desc));\n+\n+\treturn 0;\n+}\n+\n+/* Leaving always free descriptors for avoiding wrapping confusion */\n+static inline\n+uint32_t nfp_free_tx_desc(struct nfp_net_txq *txq)\n+{\n+\tif (txq->wr_p >= txq->rd_p)\n+\t\treturn txq->tx_count - (txq->wr_p - txq->rd_p) - 8;\n+\telse\n+\t\treturn txq->rd_p - txq->wr_p - 8;\n+}\n+\n+/*\n+ * nfp_net_txq_full - Check if the TX queue free descriptors\n+ * is below tx_free_threshold\n+ *\n+ * @txq: TX queue to check\n+ *\n+ * This function uses the host copy* of read/write pointers\n+ */\n+static inline\n+uint32_t nfp_net_txq_full(struct nfp_net_txq *txq)\n+{\n+\treturn (nfp_free_tx_desc(txq) < txq->tx_free_thresh);\n+}\n+\n+/* nfp_net_tx_tso - Set TX descriptor for TSO */\n+static inline void\n+nfp_net_tx_tso(struct nfp_net_txq *txq, struct nfp_net_tx_desc *txd,\n+\t struct rte_mbuf *mb)\n+{\n+\tuint64_t ol_flags;\n+\tstruct nfp_net_hw *hw = txq->hw;\n+\n+\tif (!(hw->cap & NFP_NET_CFG_CTRL_LSO_ANY))\n+\t\tgoto clean_txd;\n+\n+\tol_flags = mb->ol_flags;\n+\n+\tif (!(ol_flags & PKT_TX_TCP_SEG))\n+\t\tgoto clean_txd;\n+\n+\ttxd->l3_offset = mb->l2_len;\n+\ttxd->l4_offset = mb->l2_len + mb->l3_len;\n+\ttxd->lso_hdrlen = mb->l2_len + mb->l3_len + mb->l4_len;\n+\ttxd->mss = rte_cpu_to_le_16(mb->tso_segsz);\n+\ttxd->flags = PCIE_DESC_TX_LSO;\n+\treturn;\n+\n+clean_txd:\n+\ttxd->flags = 0;\n+\ttxd->l3_offset = 0;\n+\ttxd->l4_offset = 0;\n+\ttxd->lso_hdrlen = 0;\n+\ttxd->mss = 0;\n+}\n+\n+/* nfp_net_tx_cksum - Set TX CSUM offload flags in TX descriptor */\n+static inline void\n+nfp_net_tx_cksum(struct nfp_net_txq *txq, struct nfp_net_tx_desc *txd,\n+\t\t struct rte_mbuf *mb)\n+{\n+\tuint64_t ol_flags;\n+\tstruct nfp_net_hw *hw = txq->hw;\n+\n+\tif (!(hw->cap & NFP_NET_CFG_CTRL_TXCSUM))\n+\t\treturn;\n+\n+\tol_flags = mb->ol_flags;\n+\n+\t/* IPv6 does not need checksum */\n+\tif (ol_flags & PKT_TX_IP_CKSUM)\n+\t\ttxd->flags |= PCIE_DESC_TX_IP4_CSUM;\n+\n+\tswitch (ol_flags & PKT_TX_L4_MASK) {\n+\tcase PKT_TX_UDP_CKSUM:\n+\t\ttxd->flags |= PCIE_DESC_TX_UDP_CSUM;\n+\t\tbreak;\n+\tcase PKT_TX_TCP_CKSUM:\n+\t\ttxd->flags |= PCIE_DESC_TX_TCP_CSUM;\n+\t\tbreak;\n+\t}\n+\n+\tif (ol_flags & (PKT_TX_IP_CKSUM | PKT_TX_L4_MASK))\n+\t\ttxd->flags |= PCIE_DESC_TX_CSUM;\n+}\n+\n+uint16_t\n+nfp_net_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)\n+{\n+\tstruct nfp_net_txq *txq;\n+\tstruct nfp_net_hw *hw;\n+\tstruct nfp_net_tx_desc *txds, txd;\n+\tstruct rte_mbuf *pkt;\n+\tuint64_t dma_addr;\n+\tint pkt_size, dma_size;\n+\tuint16_t free_descs, issued_descs;\n+\tstruct rte_mbuf **lmbuf;\n+\tint i;\n+\n+\ttxq = tx_queue;\n+\thw = txq->hw;\n+\ttxds = &txq->txds[txq->wr_p];\n+\n+\tPMD_TX_LOG(DEBUG, \"working for queue %u at pos %d and %u packets\",\n+\t\t txq->qidx, txq->wr_p, nb_pkts);\n+\n+\tif ((nfp_free_tx_desc(txq) < nb_pkts) || (nfp_net_txq_full(txq)))\n+\t\tnfp_net_tx_free_bufs(txq);\n+\n+\tfree_descs = (uint16_t)nfp_free_tx_desc(txq);\n+\tif (unlikely(free_descs == 0))\n+\t\treturn 0;\n+\n+\tpkt = *tx_pkts;\n+\n+\ti = 0;\n+\tissued_descs = 0;\n+\tPMD_TX_LOG(DEBUG, \"queue: %u. Sending %u packets\",\n+\t\t txq->qidx, nb_pkts);\n+\t/* Sending packets */\n+\twhile ((i < nb_pkts) && free_descs) {\n+\t\t/* Grabbing the mbuf linked to the current descriptor */\n+\t\tlmbuf = &txq->txbufs[txq->wr_p].mbuf;\n+\t\t/* Warming the cache for releasing the mbuf later on */\n+\t\tRTE_MBUF_PREFETCH_TO_FREE(*lmbuf);\n+\n+\t\tpkt = *(tx_pkts + i);\n+\n+\t\tif (unlikely(pkt->nb_segs > 1 &&\n+\t\t\t !(hw->cap & NFP_NET_CFG_CTRL_GATHER))) {\n+\t\t\tPMD_INIT_LOG(INFO, \"NFP_NET_CFG_CTRL_GATHER not set\");\n+\t\t\trte_panic(\"Multisegment packet unsupported\\n\");\n+\t\t}\n+\n+\t\t/* Checking if we have enough descriptors */\n+\t\tif (unlikely(pkt->nb_segs > free_descs))\n+\t\t\tgoto xmit_end;\n+\n+\t\t/*\n+\t\t * Checksum and VLAN flags just in the first descriptor for a\n+\t\t * multisegment packet, but TSO info needs to be in all of them.\n+\t\t */\n+\t\ttxd.data_len = pkt->pkt_len;\n+\t\tnfp_net_tx_tso(txq, &txd, pkt);\n+\t\tnfp_net_tx_cksum(txq, &txd, pkt);\n+\n+\t\tif ((pkt->ol_flags & PKT_TX_VLAN_PKT) &&\n+\t\t (hw->cap & NFP_NET_CFG_CTRL_TXVLAN)) {\n+\t\t\ttxd.flags |= PCIE_DESC_TX_VLAN;\n+\t\t\ttxd.vlan = pkt->vlan_tci;\n+\t\t}\n+\n+\t\t/*\n+\t\t * mbuf data_len is the data in one segment and pkt_len data\n+\t\t * in the whole packet. When the packet is just one segment,\n+\t\t * then data_len = pkt_len\n+\t\t */\n+\t\tpkt_size = pkt->pkt_len;\n+\n+\t\twhile (pkt) {\n+\t\t\t/* Copying TSO, VLAN and cksum info */\n+\t\t\t*txds = txd;\n+\n+\t\t\t/* Releasing mbuf used by this descriptor previously*/\n+\t\t\tif (*lmbuf)\n+\t\t\t\trte_pktmbuf_free_seg(*lmbuf);\n+\n+\t\t\t/*\n+\t\t\t * Linking mbuf with descriptor for being released\n+\t\t\t * next time descriptor is used\n+\t\t\t */\n+\t\t\t*lmbuf = pkt;\n+\n+\t\t\tdma_size = pkt->data_len;\n+\t\t\tdma_addr = rte_mbuf_data_iova(pkt);\n+\t\t\tPMD_TX_LOG(DEBUG, \"Working with mbuf at dma address:\"\n+\t\t\t\t \"%\" PRIx64 \"\", dma_addr);\n+\n+\t\t\t/* Filling descriptors fields */\n+\t\t\ttxds->dma_len = dma_size;\n+\t\t\ttxds->data_len = txd.data_len;\n+\t\t\ttxds->dma_addr_hi = (dma_addr >> 32) & 0xff;\n+\t\t\ttxds->dma_addr_lo = (dma_addr & 0xffffffff);\n+\t\t\tASSERT(free_descs > 0);\n+\t\t\tfree_descs--;\n+\n+\t\t\ttxq->wr_p++;\n+\t\t\tif (unlikely(txq->wr_p == txq->tx_count)) /* wrapping?*/\n+\t\t\t\ttxq->wr_p = 0;\n+\n+\t\t\tpkt_size -= dma_size;\n+\n+\t\t\t/*\n+\t\t\t * Making the EOP, packets with just one segment\n+\t\t\t * the priority\n+\t\t\t */\n+\t\t\tif (likely(!pkt_size))\n+\t\t\t\ttxds->offset_eop = PCIE_DESC_TX_EOP;\n+\t\t\telse\n+\t\t\t\ttxds->offset_eop = 0;\n+\n+\t\t\tpkt = pkt->next;\n+\t\t\t/* Referencing next free TX descriptor */\n+\t\t\ttxds = &txq->txds[txq->wr_p];\n+\t\t\tlmbuf = &txq->txbufs[txq->wr_p].mbuf;\n+\t\t\tissued_descs++;\n+\t\t}\n+\t\ti++;\n+\t}\n+\n+xmit_end:\n+\t/* Increment write pointers. Force memory write before we let HW know */\n+\trte_wmb();\n+\tnfp_qcp_ptr_add(txq->qcp_q, NFP_QCP_WRITE_PTR, issued_descs);\n+\n+\treturn i;\n+}\ndiff --git a/drivers/net/nfp/nfp_rxtx.h b/drivers/net/nfp/nfp_rxtx.h\nindex 41a3a4b4e7..d2d0f3f175 100644\n--- a/drivers/net/nfp/nfp_rxtx.h\n+++ b/drivers/net/nfp/nfp_rxtx.h\n@@ -17,6 +17,14 @@\n #include <linux/types.h>\n #include <rte_io.h>\n \n+#define NFP_DESC_META_LEN(d) ((d)->rxd.meta_len_dd & PCIE_DESC_RX_META_LEN_MASK)\n+\n+#define NFP_HASH_OFFSET ((uint8_t *)mbuf->buf_addr + mbuf->data_off - 4)\n+#define NFP_HASH_TYPE_OFFSET ((uint8_t *)mbuf->buf_addr + mbuf->data_off - 8)\n+\n+#define RTE_MBUF_DMA_ADDR_DEFAULT(mb) \\\n+\t((uint64_t)((mb)->buf_iova + RTE_PKTMBUF_HEADROOM))\n+\n /*\n * The maximum number of descriptors is limited by design as\n * DPDK uses uint16_t variables for these values\n@@ -266,6 +274,25 @@ struct nfp_net_rxq {\n \tint rx_qcidx;\n } __rte_aligned(64);\n \n+int nfp_net_rx_freelist_setup(struct rte_eth_dev *dev);\n+uint32_t nfp_net_rx_queue_count(struct rte_eth_dev *dev,\n+\t\t\t\t uint16_t queue_idx);\n+uint16_t nfp_net_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,\n+\t\t\t\t uint16_t nb_pkts);\n+void nfp_net_rx_queue_release(void *rxq);\n+void nfp_net_reset_rx_queue(struct nfp_net_rxq *rxq);\n+int nfp_net_rx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,\n+\t\t\t\t uint16_t nb_desc, unsigned int socket_id,\n+\t\t\t\t const struct rte_eth_rxconf *rx_conf,\n+\t\t\t\t struct rte_mempool *mp);\n+void nfp_net_tx_queue_release(void *txq);\n+void nfp_net_reset_tx_queue(struct nfp_net_txq *txq);\n+int nfp_net_tx_queue_setup(struct rte_eth_dev *dev, uint16_t queue_idx,\n+\t\t\t\t uint16_t nb_desc, unsigned int socket_id,\n+\t\t\t\t const struct rte_eth_txconf *tx_conf);\n+uint16_t nfp_net_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts,\n+\t\t\t\t uint16_t nb_pkts);\n+\n #endif /* _NFP_RXTX_H_ */\n /*\n * Local variables:\n", "prefixes": [ "v3", "2/7" ] }