@@ -28,6 +28,7 @@ sources = files(
'roc_nix_debug.c',
'roc_nix_fc.c',
'roc_nix_irq.c',
+ 'roc_nix_inl_dev_irq.c',
'roc_nix_mac.c',
'roc_nix_mcast.c',
'roc_nix_npc.c',
@@ -129,4 +129,7 @@
/* HASH computation */
#include "roc_hash.h"
+/* NIX Inline dev */
+#include "roc_nix_inl.h"
+
#endif /* _ROC_API_H_ */
@@ -138,9 +138,10 @@ dev_irq_register(struct plt_intr_handle *intr_handle, plt_intr_callback_fn cb,
irq_init(intr_handle);
}
- if (vec > intr_handle->max_intr) {
- plt_err("Vector=%d greater than max_intr=%d", vec,
- intr_handle->max_intr);
+ if (vec > intr_handle->max_intr || vec >= PLT_DIM(intr_handle->efds)) {
+ plt_err("Vector=%d greater than max_intr=%d or "
+ "max_efd=%" PRIu64,
+ vec, intr_handle->max_intr, PLT_DIM(intr_handle->efds));
return -EINVAL;
}
new file mode 100644
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+#ifndef _ROC_NIX_INL_H_
+#define _ROC_NIX_INL_H_
+
+/* Inline device SSO Work callback */
+typedef void (*roc_nix_inl_sso_work_cb_t)(uint64_t *gw, void *args);
+
+#endif /* _ROC_NIX_INL_H_ */
new file mode 100644
@@ -0,0 +1,359 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+
+#include "roc_api.h"
+#include "roc_priv.h"
+
+static void
+nix_inl_sso_work_cb(struct nix_inl_dev *inl_dev)
+{
+ uintptr_t getwrk_op = inl_dev->ssow_base + SSOW_LF_GWS_OP_GET_WORK0;
+ uintptr_t tag_wqe_op = inl_dev->ssow_base + SSOW_LF_GWS_WQE0;
+ uint32_t wdata = BIT(16) | 1;
+ union {
+ __uint128_t get_work;
+ uint64_t u64[2];
+ } gw;
+ uint64_t work;
+
+again:
+ /* Try to do get work */
+ gw.get_work = wdata;
+ plt_write64(gw.u64[0], getwrk_op);
+ do {
+ roc_load_pair(gw.u64[0], gw.u64[1], tag_wqe_op);
+ } while (gw.u64[0] & BIT_ULL(63));
+
+ work = gw.u64[1];
+ /* Do we have any work? */
+ if (work) {
+ if (inl_dev->work_cb)
+ inl_dev->work_cb(gw.u64, inl_dev->cb_args);
+ else
+ plt_warn("Undelivered inl dev work gw0: %p gw1: %p",
+ (void *)gw.u64[0], (void *)gw.u64[1]);
+ goto again;
+ }
+
+ plt_atomic_thread_fence(__ATOMIC_ACQ_REL);
+}
+
+static int
+nix_inl_nix_reg_dump(struct nix_inl_dev *inl_dev)
+{
+ uintptr_t nix_base = inl_dev->nix_base;
+
+ /* General registers */
+ nix_lf_gen_reg_dump(nix_base, NULL);
+
+ /* Rx, Tx stat registers */
+ nix_lf_stat_reg_dump(nix_base, NULL, inl_dev->lf_tx_stats,
+ inl_dev->lf_rx_stats);
+
+ /* Intr registers */
+ nix_lf_int_reg_dump(nix_base, NULL, inl_dev->qints, inl_dev->cints);
+
+ return 0;
+}
+
+static void
+nix_inl_sso_hwgrp_irq(void *param)
+{
+ struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param;
+ uintptr_t sso_base = inl_dev->sso_base;
+ uint64_t intr;
+
+ intr = plt_read64(sso_base + SSO_LF_GGRP_INT);
+ if (intr == 0)
+ return;
+
+ /* Check for work executable interrupt */
+ if (intr & BIT(1))
+ nix_inl_sso_work_cb(inl_dev);
+
+ if (!(intr & BIT(1)))
+ plt_err("GGRP 0 GGRP_INT=0x%" PRIx64 "", intr);
+
+ /* Clear interrupt */
+ plt_write64(intr, sso_base + SSO_LF_GGRP_INT);
+}
+
+static void
+nix_inl_sso_hws_irq(void *param)
+{
+ struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param;
+ uintptr_t ssow_base = inl_dev->ssow_base;
+ uint64_t intr;
+
+ intr = plt_read64(ssow_base + SSOW_LF_GWS_INT);
+ if (intr == 0)
+ return;
+
+ plt_err("GWS 0 GWS_INT=0x%" PRIx64 "", intr);
+
+ /* Clear interrupt */
+ plt_write64(intr, ssow_base + SSOW_LF_GWS_INT);
+}
+
+int
+nix_inl_sso_register_irqs(struct nix_inl_dev *inl_dev)
+{
+ struct plt_intr_handle *handle = &inl_dev->pci_dev->intr_handle;
+ uintptr_t ssow_base = inl_dev->ssow_base;
+ uintptr_t sso_base = inl_dev->sso_base;
+ uint16_t sso_msixoff, ssow_msixoff;
+ int rc;
+
+ ssow_msixoff = inl_dev->ssow_msixoff;
+ sso_msixoff = inl_dev->sso_msixoff;
+ if (sso_msixoff == MSIX_VECTOR_INVALID ||
+ ssow_msixoff == MSIX_VECTOR_INVALID) {
+ plt_err("Invalid SSO/SSOW MSIX offsets (0x%x, 0x%x)",
+ sso_msixoff, ssow_msixoff);
+ return -EINVAL;
+ }
+
+ /*
+ * Setup SSOW interrupt
+ */
+
+ /* Clear SSOW interrupt enable */
+ plt_write64(~0ull, ssow_base + SSOW_LF_GWS_INT_ENA_W1C);
+ /* Register interrupt with vfio */
+ rc = dev_irq_register(handle, nix_inl_sso_hws_irq, inl_dev,
+ ssow_msixoff + SSOW_LF_INT_VEC_IOP);
+ /* Set SSOW interrupt enable */
+ plt_write64(~0ull, ssow_base + SSOW_LF_GWS_INT_ENA_W1S);
+
+ /*
+ * Setup SSO/HWGRP interrupt
+ */
+
+ /* Clear SSO interrupt enable */
+ plt_write64(~0ull, sso_base + SSO_LF_GGRP_INT_ENA_W1C);
+ /* Register IRQ */
+ rc |= dev_irq_register(handle, nix_inl_sso_hwgrp_irq, (void *)inl_dev,
+ sso_msixoff + SSO_LF_INT_VEC_GRP);
+ /* Enable hw interrupt */
+ plt_write64(~0ull, sso_base + SSO_LF_GGRP_INT_ENA_W1S);
+
+ /* Setup threshold for work exec interrupt to 1 wqe in IAQ */
+ plt_write64(0x1ull, sso_base + SSO_LF_GGRP_INT_THR);
+
+ return rc;
+}
+
+void
+nix_inl_sso_unregister_irqs(struct nix_inl_dev *inl_dev)
+{
+ struct plt_intr_handle *handle = &inl_dev->pci_dev->intr_handle;
+ uintptr_t ssow_base = inl_dev->ssow_base;
+ uintptr_t sso_base = inl_dev->sso_base;
+ uint16_t sso_msixoff, ssow_msixoff;
+
+ ssow_msixoff = inl_dev->ssow_msixoff;
+ sso_msixoff = inl_dev->sso_msixoff;
+
+ /* Clear SSOW interrupt enable */
+ plt_write64(~0ull, ssow_base + SSOW_LF_GWS_INT_ENA_W1C);
+ /* Clear SSO/HWGRP interrupt enable */
+ plt_write64(~0ull, sso_base + SSO_LF_GGRP_INT_ENA_W1C);
+ /* Clear SSO threshold */
+ plt_write64(0, sso_base + SSO_LF_GGRP_INT_THR);
+
+ /* Unregister IRQ */
+ dev_irq_unregister(handle, nix_inl_sso_hws_irq, (void *)inl_dev,
+ ssow_msixoff + SSOW_LF_INT_VEC_IOP);
+ dev_irq_unregister(handle, nix_inl_sso_hwgrp_irq, (void *)inl_dev,
+ sso_msixoff + SSO_LF_INT_VEC_GRP);
+}
+
+static void
+nix_inl_nix_q_irq(void *param)
+{
+ struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param;
+ uintptr_t nix_base = inl_dev->nix_base;
+ struct dev *dev = &inl_dev->dev;
+ volatile void *ctx;
+ uint64_t reg, intr;
+ uint8_t irq;
+ int rc;
+
+ intr = plt_read64(nix_base + NIX_LF_QINTX_INT(0));
+ if (intr == 0)
+ return;
+
+ plt_err("Queue_intr=0x%" PRIx64 " qintx 0 pf=%d, vf=%d", intr, dev->pf,
+ dev->vf);
+
+ /* Get and clear RQ0 interrupt */
+ reg = roc_atomic64_add_nosync(0,
+ (int64_t *)(nix_base + NIX_LF_RQ_OP_INT));
+ if (reg & BIT_ULL(42) /* OP_ERR */) {
+ plt_err("Failed to get rq_int");
+ return;
+ }
+ irq = reg & 0xff;
+ plt_write64(0 | irq, nix_base + NIX_LF_RQ_OP_INT);
+
+ if (irq & BIT_ULL(NIX_RQINT_DROP))
+ plt_err("RQ=0 NIX_RQINT_DROP");
+
+ if (irq & BIT_ULL(NIX_RQINT_RED))
+ plt_err("RQ=0 NIX_RQINT_RED");
+
+ /* Clear interrupt */
+ plt_write64(intr, nix_base + NIX_LF_QINTX_INT(0));
+
+ /* Dump registers to std out */
+ nix_inl_nix_reg_dump(inl_dev);
+
+ /* Dump RQ 0 */
+ rc = nix_q_ctx_get(dev, NIX_AQ_CTYPE_RQ, 0, &ctx);
+ if (rc) {
+ plt_err("Failed to get rq context");
+ return;
+ }
+ nix_lf_rq_dump(ctx);
+}
+
+static void
+nix_inl_nix_ras_irq(void *param)
+{
+ struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param;
+ uintptr_t nix_base = inl_dev->nix_base;
+ struct dev *dev = &inl_dev->dev;
+ volatile void *ctx;
+ uint64_t intr;
+ int rc;
+
+ intr = plt_read64(nix_base + NIX_LF_RAS);
+ if (intr == 0)
+ return;
+
+ plt_err("Ras_intr=0x%" PRIx64 " pf=%d, vf=%d", intr, dev->pf, dev->vf);
+ /* Clear interrupt */
+ plt_write64(intr, nix_base + NIX_LF_RAS);
+
+ /* Dump registers to std out */
+ nix_inl_nix_reg_dump(inl_dev);
+
+ /* Dump RQ 0 */
+ rc = nix_q_ctx_get(dev, NIX_AQ_CTYPE_RQ, 0, &ctx);
+ if (rc) {
+ plt_err("Failed to get rq context");
+ return;
+ }
+ nix_lf_rq_dump(ctx);
+}
+
+static void
+nix_inl_nix_err_irq(void *param)
+{
+ struct nix_inl_dev *inl_dev = (struct nix_inl_dev *)param;
+ uintptr_t nix_base = inl_dev->nix_base;
+ struct dev *dev = &inl_dev->dev;
+ volatile void *ctx;
+ uint64_t intr;
+ int rc;
+
+ intr = plt_read64(nix_base + NIX_LF_ERR_INT);
+ if (intr == 0)
+ return;
+
+ plt_err("Err_irq=0x%" PRIx64 " pf=%d, vf=%d", intr, dev->pf, dev->vf);
+
+ /* Clear interrupt */
+ plt_write64(intr, nix_base + NIX_LF_ERR_INT);
+
+ /* Dump registers to std out */
+ nix_inl_nix_reg_dump(inl_dev);
+
+ /* Dump RQ 0 */
+ rc = nix_q_ctx_get(dev, NIX_AQ_CTYPE_RQ, 0, &ctx);
+ if (rc) {
+ plt_err("Failed to get rq context");
+ return;
+ }
+ nix_lf_rq_dump(ctx);
+}
+
+int
+nix_inl_nix_register_irqs(struct nix_inl_dev *inl_dev)
+{
+ struct plt_intr_handle *handle = &inl_dev->pci_dev->intr_handle;
+ uintptr_t nix_base = inl_dev->nix_base;
+ uint16_t msixoff;
+ int rc;
+
+ msixoff = inl_dev->nix_msixoff;
+ if (msixoff == MSIX_VECTOR_INVALID) {
+ plt_err("Invalid NIXLF MSIX vector offset: 0x%x", msixoff);
+ return -EINVAL;
+ }
+
+ /* Disable err interrupts */
+ plt_write64(~0ull, nix_base + NIX_LF_ERR_INT_ENA_W1C);
+ /* DIsable RAS interrupts */
+ plt_write64(~0ull, nix_base + NIX_LF_RAS_ENA_W1C);
+
+ /* Register err irq */
+ rc = dev_irq_register(handle, nix_inl_nix_err_irq, inl_dev,
+ msixoff + NIX_LF_INT_VEC_ERR_INT);
+ rc |= dev_irq_register(handle, nix_inl_nix_ras_irq, inl_dev,
+ msixoff + NIX_LF_INT_VEC_POISON);
+
+ /* Enable all nix lf error irqs except RQ_DISABLED and CQ_DISABLED */
+ plt_write64(~(BIT_ULL(11) | BIT_ULL(24)),
+ nix_base + NIX_LF_ERR_INT_ENA_W1S);
+ /* Enable RAS interrupts */
+ plt_write64(~0ull, nix_base + NIX_LF_RAS_ENA_W1S);
+
+ /* Setup queue irq for RQ 0 */
+
+ /* Clear QINT CNT, interrupt */
+ plt_write64(0, nix_base + NIX_LF_QINTX_CNT(0));
+ plt_write64(~0ull, nix_base + NIX_LF_QINTX_ENA_W1C(0));
+
+ /* Register queue irq vector */
+ rc |= dev_irq_register(handle, nix_inl_nix_q_irq, inl_dev,
+ msixoff + NIX_LF_INT_VEC_QINT_START);
+
+ plt_write64(0, nix_base + NIX_LF_QINTX_CNT(0));
+ plt_write64(0, nix_base + NIX_LF_QINTX_INT(0));
+ /* Enable QINT interrupt */
+ plt_write64(~0ull, nix_base + NIX_LF_QINTX_ENA_W1S(0));
+
+ return rc;
+}
+
+void
+nix_inl_nix_unregister_irqs(struct nix_inl_dev *inl_dev)
+{
+ struct plt_intr_handle *handle = &inl_dev->pci_dev->intr_handle;
+ uintptr_t nix_base = inl_dev->nix_base;
+ uint16_t msixoff;
+
+ msixoff = inl_dev->nix_msixoff;
+ /* Disable err interrupts */
+ plt_write64(~0ull, nix_base + NIX_LF_ERR_INT_ENA_W1C);
+ /* DIsable RAS interrupts */
+ plt_write64(~0ull, nix_base + NIX_LF_RAS_ENA_W1C);
+
+ dev_irq_unregister(handle, nix_inl_nix_err_irq, inl_dev,
+ msixoff + NIX_LF_INT_VEC_ERR_INT);
+ dev_irq_unregister(handle, nix_inl_nix_ras_irq, inl_dev,
+ msixoff + NIX_LF_INT_VEC_POISON);
+
+ /* Clear QINT CNT */
+ plt_write64(0, nix_base + NIX_LF_QINTX_CNT(0));
+ plt_write64(0, nix_base + NIX_LF_QINTX_INT(0));
+
+ /* Disable QINT interrupt */
+ plt_write64(~0ull, nix_base + NIX_LF_QINTX_ENA_W1C(0));
+
+ /* Unregister queue irq vector */
+ dev_irq_unregister(handle, nix_inl_nix_q_irq, inl_dev,
+ msixoff + NIX_LF_INT_VEC_QINT_START);
+}
new file mode 100644
@@ -0,0 +1,57 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright(C) 2021 Marvell.
+ */
+#ifndef _ROC_NIX_INL_PRIV_H_
+#define _ROC_NIX_INL_PRIV_H_
+
+struct nix_inl_dev {
+ /* Base device object */
+ struct dev dev;
+
+ /* PCI device */
+ struct plt_pci_device *pci_dev;
+
+ /* LF specific BAR2 regions */
+ uintptr_t nix_base;
+ uintptr_t ssow_base;
+ uintptr_t sso_base;
+
+ /* MSIX vector offsets */
+ uint16_t nix_msixoff;
+ uint16_t ssow_msixoff;
+ uint16_t sso_msixoff;
+
+ /* SSO data */
+ uint32_t xaq_buf_size;
+ uint32_t xae_waes;
+ uint32_t iue;
+ uint64_t xaq_aura;
+ void *xaq_mem;
+ roc_nix_inl_sso_work_cb_t work_cb;
+ void *cb_args;
+
+ /* NIX data */
+ uint8_t lf_tx_stats;
+ uint8_t lf_rx_stats;
+ uint16_t cints;
+ uint16_t qints;
+ struct roc_nix_rq rq;
+ uint16_t rq_refs;
+ bool is_nix1;
+
+ /* NIX/CPT data */
+ void *inb_sa_base;
+ uint16_t inb_sa_sz;
+
+ /* Device arguments */
+ uint8_t selftest;
+ uint16_t ipsec_in_max_spi;
+};
+
+int nix_inl_sso_register_irqs(struct nix_inl_dev *inl_dev);
+void nix_inl_sso_unregister_irqs(struct nix_inl_dev *inl_dev);
+
+int nix_inl_nix_register_irqs(struct nix_inl_dev *inl_dev);
+void nix_inl_nix_unregister_irqs(struct nix_inl_dev *inl_dev);
+
+#endif /* _ROC_NIX_INL_PRIV_H_ */
@@ -113,10 +113,11 @@
#define plt_write64(val, addr) \
rte_write64_relaxed((val), (volatile void *)(addr))
-#define plt_wmb() rte_wmb()
-#define plt_rmb() rte_rmb()
-#define plt_io_wmb() rte_io_wmb()
-#define plt_io_rmb() rte_io_rmb()
+#define plt_wmb() rte_wmb()
+#define plt_rmb() rte_rmb()
+#define plt_io_wmb() rte_io_wmb()
+#define plt_io_rmb() rte_io_rmb()
+#define plt_atomic_thread_fence rte_atomic_thread_fence
#define plt_mmap mmap
#define PLT_PROT_READ PROT_READ
@@ -38,4 +38,7 @@
/* CPT */
#include "roc_cpt_priv.h"
+/* NIX Inline dev */
+#include "roc_nix_inl_priv.h"
+
#endif /* _ROC_PRIV_H_ */