@@ -782,6 +782,7 @@ struct bnxt {
#define BNXT_MULTIROOT_EN(bp) \
((bp)->flags2 & BNXT_FLAGS2_MULTIROOT_EN)
+#define BNXT_FLAGS2_COMPRESSED_RX_CQE BIT(5)
uint32_t fw_cap;
#define BNXT_FW_CAP_HOT_RESET BIT(0)
#define BNXT_FW_CAP_IF_CHANGE BIT(1)
@@ -814,6 +815,7 @@ struct bnxt {
#define BNXT_VNIC_CAP_VLAN_RX_STRIP BIT(3)
#define BNXT_RX_VLAN_STRIP_EN(bp) ((bp)->vnic_cap_flags & BNXT_VNIC_CAP_VLAN_RX_STRIP)
#define BNXT_VNIC_CAP_OUTER_RSS_TRUSTED_VF BIT(4)
+#define BNXT_VNIC_CAP_L2_CQE_MODE BIT(8)
unsigned int rx_nr_rings;
unsigned int rx_cp_nr_rings;
unsigned int rx_num_qs_per_vnic;
@@ -1013,6 +1015,21 @@ inline uint16_t bnxt_max_rings(struct bnxt *bp)
return max_rings;
}
+static inline bool
+bnxt_compressed_rx_cqe_mode_enabled(struct bnxt *bp)
+{
+ uint64_t rx_offloads = bp->eth_dev->data->dev_conf.rxmode.offloads;
+
+ if (bp->vnic_cap_flags & BNXT_VNIC_CAP_L2_CQE_MODE &&
+ bp->flags2 & BNXT_FLAGS2_COMPRESSED_RX_CQE &&
+ !(rx_offloads & RTE_ETH_RX_OFFLOAD_TCP_LRO) &&
+ !(rx_offloads & RTE_ETH_RX_OFFLOAD_BUFFER_SPLIT) &&
+ !bp->num_reps && !bp->ieee_1588)
+ return true;
+
+ return false;
+}
+
#define BNXT_FC_TIMER 1 /* Timer freq in Sec Flow Counters */
/**
@@ -103,6 +103,7 @@ static const struct rte_pci_id bnxt_pci_id_map[] = {
#define BNXT_DEVARG_REP_FC_F2R "rep-fc-f2r"
#define BNXT_DEVARG_APP_ID "app-id"
#define BNXT_DEVARG_IEEE_1588 "ieee-1588"
+#define BNXT_DEVARG_CQE_MODE "cqe-mode"
static const char *const bnxt_dev_args[] = {
BNXT_DEVARG_REPRESENTOR,
@@ -116,9 +117,15 @@ static const char *const bnxt_dev_args[] = {
BNXT_DEVARG_REP_FC_F2R,
BNXT_DEVARG_APP_ID,
BNXT_DEVARG_IEEE_1588,
+ BNXT_DEVARG_CQE_MODE,
NULL
};
+/*
+ * cqe-mode = an non-negative 8-bit number
+ */
+#define BNXT_DEVARG_CQE_MODE_INVALID(val) ((val) > 1)
+
/*
* app-id = an non-negative 8-bit number
*/
@@ -5706,6 +5713,43 @@ bnxt_parse_devarg_max_num_kflows(__rte_unused const char *key,
return 0;
}
+static int
+bnxt_parse_devarg_cqe_mode(__rte_unused const char *key,
+ const char *value, void *opaque_arg)
+{
+ struct bnxt *bp = opaque_arg;
+ unsigned long cqe_mode;
+ char *end = NULL;
+
+ if (!value || !opaque_arg) {
+ PMD_DRV_LOG(ERR,
+ "Invalid parameter passed to cqe-mode "
+ "devargs.\n");
+ return -EINVAL;
+ }
+
+ cqe_mode = strtoul(value, &end, 10);
+ if (end == NULL || *end != '\0' ||
+ (cqe_mode == ULONG_MAX && errno == ERANGE)) {
+ PMD_DRV_LOG(ERR,
+ "Invalid parameter passed to cqe-mode "
+ "devargs.\n");
+ return -EINVAL;
+ }
+
+ if (BNXT_DEVARG_CQE_MODE_INVALID(cqe_mode)) {
+ PMD_DRV_LOG(ERR, "Invalid cqe-mode(%d) devargs.\n",
+ (uint16_t)cqe_mode);
+ return -EINVAL;
+ }
+
+ if (cqe_mode == 1)
+ bp->flags2 |= BNXT_FLAGS2_COMPRESSED_RX_CQE;
+ PMD_DRV_LOG(INFO, "cqe-mode=%d feature enabled.\n", (uint8_t)cqe_mode);
+
+ return 0;
+}
+
static int
bnxt_parse_devarg_app_id(__rte_unused const char *key,
const char *value, void *opaque_arg)
@@ -6047,6 +6091,13 @@ bnxt_parse_dev_args(struct bnxt *bp, struct rte_devargs *devargs)
rte_kvargs_process(kvlist, BNXT_DEVARG_IEEE_1588,
bnxt_parse_devarg_ieee_1588, bp);
+ /*
+ * Handler for "cqe-mode" devarg.
+ * Invoked as for ex: "-a 000:00:0d.0,cqe-mode=1"
+ */
+ rte_kvargs_process(kvlist, BNXT_DEVARG_CQE_MODE,
+ bnxt_parse_devarg_cqe_mode, bp);
+
rte_kvargs_free(kvlist);
return ret;
}
@@ -2228,6 +2228,12 @@ int bnxt_hwrm_vnic_cfg(struct bnxt *bp, struct bnxt_vnic_info *vnic)
req.lb_rule = rte_cpu_to_le_16(vnic->lb_rule);
config_mru:
+ if (bnxt_compressed_rx_cqe_mode_enabled(bp)) {
+ req.l2_cqe_mode = HWRM_VNIC_CFG_INPUT_L2_CQE_MODE_COMPRESSED;
+ enables |= HWRM_VNIC_CFG_INPUT_ENABLES_L2_CQE_MODE;
+ PMD_DRV_LOG(DEBUG, "Enabling compressed Rx CQE\n");
+ }
+
req.enables = rte_cpu_to_le_32(enables);
req.vnic_id = rte_cpu_to_le_16(vnic->fw_vnic_id);
req.mru = rte_cpu_to_le_16(vnic->mru);
@@ -2604,6 +2610,16 @@ int bnxt_hwrm_vnic_tpa_cfg(struct bnxt *bp,
struct hwrm_vnic_tpa_cfg_input req = {.req_type = 0 };
struct hwrm_vnic_tpa_cfg_output *resp = bp->hwrm_cmd_resp_addr;
+ if (bnxt_compressed_rx_cqe_mode_enabled(bp)) {
+ /* Don't worry if disabling TPA */
+ if (!enable)
+ return 0;
+
+ /* Return an error if enabling TPA w/ compressed Rx CQE. */
+ PMD_DRV_LOG(ERR, "No HW support for LRO with compressed Rx\n");
+ return -ENOTSUP;
+ }
+
if ((BNXT_CHIP_P5(bp) || BNXT_CHIP_P7(bp)) && !bp->max_tpa_v2) {
if (enable)
PMD_DRV_LOG(ERR, "No HW support for LRO\n");
@@ -573,6 +573,7 @@ static int bnxt_alloc_rx_agg_ring(struct bnxt *bp, int queue_index)
return rc;
rxr->ag_raw_prod = 0;
+ rxr->ag_cons = 0;
if (BNXT_HAS_RING_GRPS(bp))
bp->grp_info[queue_index].ag_fw_ring_id = ring->fw_ring_id;
bnxt_set_db(bp, &rxr->ag_db, ring_type, map_idx, ring->fw_ring_id,
@@ -595,7 +596,17 @@ int bnxt_alloc_hwrm_rx_ring(struct bnxt *bp, int queue_index)
* Storage for the cp ring is allocated based on worst-case
* usage, the actual size to be used by hw is computed here.
*/
- cp_ring->ring_size = rxr->rx_ring_struct->ring_size * 2;
+ if (bnxt_compressed_rx_cqe_mode_enabled(bp)) {
+ if (bnxt_need_agg_ring(bp->eth_dev))
+ /* Worst case scenario, needed to accommodate Rx flush
+ * completion during RING_FREE.
+ */
+ cp_ring->ring_size = rxr->rx_ring_struct->ring_size * 2;
+ else
+ cp_ring->ring_size = rxr->rx_ring_struct->ring_size;
+ } else {
+ cp_ring->ring_size = rxr->rx_ring_struct->ring_size * 2;
+ }
if (bnxt_need_agg_ring(bp->eth_dev))
cp_ring->ring_size *= AGG_RING_SIZE_FACTOR;
@@ -907,6 +907,203 @@ void bnxt_set_mark_in_mbuf(struct bnxt *bp,
mbuf->ol_flags |= RTE_MBUF_F_RX_FDIR | RTE_MBUF_F_RX_FDIR_ID;
}
+static void
+bnxt_set_ol_flags_crx(struct bnxt_rx_ring_info *rxr,
+ struct rx_pkt_compress_cmpl *rxcmp,
+ struct rte_mbuf *mbuf)
+{
+ uint16_t flags_type, errors, flags;
+ uint16_t cserr, tmp;
+ uint64_t ol_flags;
+
+ flags_type = rte_le_to_cpu_16(rxcmp->flags_type);
+
+ cserr = rte_le_to_cpu_16(rxcmp->metadata1_cs_error_calc_v1) &
+ (RX_PKT_COMPRESS_CMPL_CS_ERROR_CALC_MASK |
+ BNXT_RXC_METADATA1_VLAN_VALID);
+
+ flags = cserr & BNXT_CRX_CQE_CSUM_CALC_MASK;
+ tmp = flags;
+
+ /* Set tunnel frame indicator.
+ * This is to correctly index into the flags_err table.
+ */
+ flags |= (flags & BNXT_CRX_TUN_CS_CALC) ? BNXT_PKT_CMPL_T_IP_CS_CALC << 3 : 0;
+
+ flags = flags >> BNXT_CRX_CQE_CSUM_CALC_SFT;
+
+ errors = cserr & BNXT_CRX_CQE_CSUM_ERROR_MASK;
+ errors = (errors >> RX_PKT_COMPRESS_CMPL_CS_ERROR_CALC_SFT) & flags;
+
+ ol_flags = rxr->ol_flags_table[flags & ~errors];
+
+ if (unlikely(errors)) {
+ /* Set tunnel frame indicator.
+ * This is to correctly index into the flags_err table.
+ */
+ errors |= (tmp & BNXT_CRX_TUN_CS_CALC) ? BNXT_PKT_CMPL_T_IP_CS_CALC << 2 : 0;
+ ol_flags |= rxr->ol_flags_err_table[errors];
+ }
+
+ if (flags_type & RX_PKT_COMPRESS_CMPL_FLAGS_RSS_VALID) {
+ mbuf->hash.rss = rte_le_to_cpu_32(rxcmp->rss_hash);
+ ol_flags |= RTE_MBUF_F_RX_RSS_HASH;
+ }
+
+#ifdef RTE_LIBRTE_IEEE1588
+ /* TODO: TIMESTAMP flags need to be parsed and set. */
+#endif
+
+ mbuf->ol_flags = ol_flags;
+}
+
+static uint32_t
+bnxt_parse_pkt_type_crx(struct rx_pkt_compress_cmpl *rxcmp)
+{
+ uint16_t flags_type, meta_cs;
+ uint8_t index;
+
+ flags_type = rte_le_to_cpu_16(rxcmp->flags_type);
+ meta_cs = rte_le_to_cpu_16(rxcmp->metadata1_cs_error_calc_v1);
+
+ /* Validate ptype table indexing at build time. */
+ /* TODO */
+ /* bnxt_check_ptype_constants(); */
+
+ /*
+ * Index format:
+ * bit 0: Set if IP tunnel encapsulated packet.
+ * bit 1: Set if IPv6 packet, clear if IPv4.
+ * bit 2: Set if VLAN tag present.
+ * bits 3-6: Four-bit hardware packet type field.
+ */
+ index = BNXT_CMPL_ITYPE_TO_IDX(flags_type) |
+ BNXT_CMPL_VLAN_TUN_TO_IDX_CRX(meta_cs) |
+ BNXT_CMPL_IP_VER_TO_IDX(flags_type);
+
+ return bnxt_ptype_table[index];
+}
+
+static int bnxt_rx_pages_crx(struct bnxt_rx_queue *rxq, struct rte_mbuf *mbuf,
+ uint32_t *tmp_raw_cons, uint8_t agg_buf)
+{
+ struct bnxt_cp_ring_info *cpr = rxq->cp_ring;
+ struct bnxt_rx_ring_info *rxr = rxq->rx_ring;
+ int i;
+ uint16_t cp_cons, ag_cons;
+ struct rx_pkt_compress_cmpl *rxcmp;
+ struct rte_mbuf *last = mbuf;
+
+ for (i = 0; i < agg_buf; i++) {
+ struct rte_mbuf **ag_buf;
+ struct rte_mbuf *ag_mbuf;
+
+ *tmp_raw_cons = NEXT_RAW_CMP(*tmp_raw_cons);
+ cp_cons = RING_CMP(cpr->cp_ring_struct, *tmp_raw_cons);
+ rxcmp = (struct rx_pkt_compress_cmpl *)&cpr->cp_desc_ring[cp_cons];
+
+#ifdef BNXT_DEBUG
+ bnxt_dump_cmpl(cp_cons, rxcmp);
+#endif
+
+ /*
+ * The consumer index aka the opaque field for the agg buffers
+ * is not * available in errors_agg_bufs_opaque. So maintain it
+ * in driver itself.
+ */
+ ag_cons = rxr->ag_cons;
+ ag_buf = &rxr->ag_buf_ring[ag_cons];
+ ag_mbuf = *ag_buf;
+
+ ag_mbuf->data_len = rte_le_to_cpu_16(rxcmp->len);
+
+ mbuf->nb_segs++;
+ mbuf->pkt_len += ag_mbuf->data_len;
+
+ last->next = ag_mbuf;
+ last = ag_mbuf;
+
+ *ag_buf = NULL;
+ /*
+ * As aggregation buffer consumed out of order in TPA module,
+ * use bitmap to track freed slots to be allocated and notified
+ * to NIC. TODO: Is this needed. Most likely not.
+ */
+ rte_bitmap_set(rxr->ag_bitmap, ag_cons);
+ rxr->ag_cons = RING_IDX(rxr->ag_ring_struct, RING_NEXT(ag_cons));
+ }
+ last->next = NULL;
+ bnxt_prod_ag_mbuf(rxq);
+ return 0;
+}
+
+static int bnxt_crx_pkt(struct rte_mbuf **rx_pkt,
+ struct bnxt_rx_queue *rxq,
+ struct rx_pkt_compress_cmpl *rxcmp,
+ uint32_t *raw_cons)
+{
+ struct bnxt_cp_ring_info *cpr = rxq->cp_ring;
+ struct bnxt_rx_ring_info *rxr = rxq->rx_ring;
+ uint32_t tmp_raw_cons = *raw_cons;
+ uint16_t cons, raw_prod;
+ struct rte_mbuf *mbuf;
+ int rc = 0;
+ uint8_t agg_buf = 0;
+
+ agg_buf = BNXT_CRX_CQE_AGG_BUFS(rxcmp);
+ /*
+ * Since size of rx_pkt_cmpl is same as rx_pkt_compress_cmpl,
+ * we should be able to use bnxt_agg_bufs_valid to check if AGG
+ * bufs are valid when using compressed CQEs.
+ * All we want to check here is if the CQE is valid and the
+ * location of valid bit is same irrespective of the CQE type.
+ */
+ if (agg_buf && !bnxt_agg_bufs_valid(cpr, agg_buf, tmp_raw_cons))
+ return -EBUSY;
+
+ raw_prod = rxr->rx_raw_prod;
+
+ cons = rxcmp->errors_agg_bufs_opaque & BNXT_CRX_CQE_OPAQUE_MASK;
+ mbuf = bnxt_consume_rx_buf(rxr, cons);
+ if (mbuf == NULL)
+ return -EBUSY;
+
+ mbuf->data_off = RTE_PKTMBUF_HEADROOM;
+ mbuf->nb_segs = 1;
+ mbuf->next = NULL;
+ mbuf->pkt_len = rxcmp->len;
+ mbuf->data_len = mbuf->pkt_len;
+ mbuf->port = rxq->port_id;
+
+#ifdef RTE_LIBRTE_IEEE1588
+ /* TODO: Add timestamp support. */
+#endif
+
+ bnxt_set_ol_flags_crx(rxr, rxcmp, mbuf);
+ mbuf->packet_type = bnxt_parse_pkt_type_crx(rxcmp);
+ bnxt_set_vlan_crx(rxcmp, mbuf);
+
+ if (bnxt_alloc_rx_data(rxq, rxr, raw_prod)) {
+ PMD_DRV_LOG(ERR, "mbuf alloc failed with prod=0x%x\n",
+ raw_prod);
+ rc = -ENOMEM;
+ goto rx;
+ }
+ raw_prod = RING_NEXT(raw_prod);
+ rxr->rx_raw_prod = raw_prod;
+
+ if (agg_buf)
+ bnxt_rx_pages_crx(rxq, mbuf, &tmp_raw_cons, agg_buf);
+
+rx:
+ rxr->rx_next_cons = RING_IDX(rxr->rx_ring_struct, RING_NEXT(cons));
+ *rx_pkt = mbuf;
+
+ *raw_cons = tmp_raw_cons;
+
+ return rc;
+}
+
static int bnxt_rx_pkt(struct rte_mbuf **rx_pkt,
struct bnxt_rx_queue *rxq, uint32_t *raw_cons)
{
@@ -1148,6 +1345,10 @@ uint16_t bnxt_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,
break;
if (CMP_TYPE(rxcmp) == CMPL_BASE_TYPE_HWRM_DONE) {
PMD_DRV_LOG(ERR, "Rx flush done\n");
+ } else if (CMP_TYPE(rxcmp) == CMPL_BASE_TYPE_RX_L2_COMPRESS) {
+ rc = bnxt_crx_pkt(&rx_pkts[nb_rx_pkts], rxq,
+ (struct rx_pkt_compress_cmpl *)rxcmp,
+ &raw_cons);
} else if ((CMP_TYPE(rxcmp) >= CMPL_BASE_TYPE_RX_TPA_START_V2) &&
(CMP_TYPE(rxcmp) <= CMPL_BASE_TYPE_RX_TPA_START_V3)) {
rc = bnxt_rx_pkt(&rx_pkts[nb_rx_pkts], rxq, &raw_cons);
@@ -52,6 +52,52 @@ static inline uint16_t bnxt_tpa_start_agg_id(struct bnxt *bp,
#define BNXT_OL_FLAGS_TBL_DIM 64
#define BNXT_OL_FLAGS_ERR_TBL_DIM 32
+#define BNXT_CRX_CQE_OPAQUE_MASK \
+ RX_PKT_COMPRESS_CMPL_ERRORS_AGG_BUFS_OPAQUE_OPAQUE_MASK
+#define BNXT_CRX_CQE_AGG_BUF_MASK \
+ RX_PKT_COMPRESS_CMPL_ERRORS_AGG_BUFS_OPAQUE_AGG_BUFS_MASK
+#define BNXT_CRX_CQE_AGG_BUF_SFT \
+ RX_PKT_COMPRESS_CMPL_ERRORS_AGG_BUFS_OPAQUE_AGG_BUFS_SFT
+#define BNXT_CRX_CQE_AGG_BUFS(cmp) \
+ (((cmp)->errors_agg_bufs_opaque & BNXT_CRX_CQE_AGG_BUF_MASK) >> \
+ BNXT_CRX_CQE_AGG_BUF_SFT)
+#define BNXT_CRX_CQE_CSUM_CALC_MASK \
+ (RX_PKT_COMPRESS_CMPL_CS_ERROR_CALC_IP_CS_CALC | \
+ RX_PKT_COMPRESS_CMPL_CS_ERROR_CALC_L4_CS_CALC | \
+ RX_PKT_COMPRESS_CMPL_CS_ERROR_CALC_T_IP_CS_CALC | \
+ RX_PKT_COMPRESS_CMPL_CS_ERROR_CALC_T_L4_CS_CALC)
+#define BNXT_CRX_CQE_CSUM_CALC_SFT 8
+#define BNXT_PKT_CMPL_T_IP_CS_CALC 0x4
+
+#define BNXT_CRX_TUN_CS_CALC \
+ (!!(RX_PKT_COMPRESS_CMPL_CS_ERROR_CALC_T_IP_CS_CALC | \
+ RX_PKT_COMPRESS_CMPL_CS_ERROR_CALC_T_L4_CS_CALC))
+
+# define BNXT_CRX_CQE_CSUM_ERROR_MASK \
+ (RX_PKT_COMPRESS_CMPL_CS_ERROR_CALC_IP_CS_ERROR | \
+ RX_PKT_COMPRESS_CMPL_CS_ERROR_CALC_L4_CS_ERROR | \
+ RX_PKT_COMPRESS_CMPL_CS_ERROR_CALC_T_IP_CS_ERROR | \
+ RX_PKT_COMPRESS_CMPL_CS_ERROR_CALC_T_L4_CS_ERROR)
+
+/* meta_format != 0 and bit3 is valid, the value in meta is VLAN.
+ * Use the bit as VLAN valid bit
+ */
+#define BNXT_RXC_METADATA1_VLAN_VALID \
+ RX_PKT_COMPRESS_CMPL_METADATA1_VALID
+
+static inline void bnxt_set_vlan_crx(struct rx_pkt_compress_cmpl *rxcmp,
+ struct rte_mbuf *mbuf)
+{
+ uint16_t metadata = rte_le_to_cpu_16(rxcmp->metadata1_cs_error_calc_v1);
+ uint16_t vlan_tci = rte_le_to_cpu_16(rxcmp->vlanc_tcid);
+
+ if (metadata & RX_PKT_COMPRESS_CMPL_METADATA1_VALID)
+ mbuf->vlan_tci =
+ vlan_tci & (RX_PKT_COMPRESS_CMPL_VLANC_TCID_VID_MASK |
+ RX_PKT_COMPRESS_CMPL_VLANC_TCID_DE |
+ RX_PKT_COMPRESS_CMPL_VLANC_TCID_PRI_MASK);
+}
+
struct bnxt_tpa_info {
struct rte_mbuf *mbuf;
uint16_t len;
@@ -70,6 +116,7 @@ struct bnxt_tpa_info {
struct bnxt_rx_ring_info {
uint16_t rx_raw_prod;
uint16_t ag_raw_prod;
+ uint16_t ag_cons; /* Needed with compressed CQE */
uint16_t rx_cons; /* Needed for representor */
uint16_t rx_next_cons;
struct bnxt_db_info rx_db;
@@ -160,6 +207,10 @@ bnxt_cfa_code_dynfield(struct rte_mbuf *mbuf)
#define CMPL_FLAGS2_VLAN_TUN_MSK \
(RX_PKT_CMPL_FLAGS2_META_FORMAT_VLAN | RX_PKT_CMPL_FLAGS2_T_IP_CS_CALC)
+#define CMPL_FLAGS2_VLAN_TUN_MSK_CRX \
+ (RX_PKT_COMPRESS_CMPL_METADATA1_VALID | \
+ RX_PKT_COMPRESS_CMPL_CS_ERROR_CALC_T_IP_CS_CALC)
+
#define BNXT_CMPL_ITYPE_TO_IDX(ft) \
(((ft) & RX_PKT_CMPL_FLAGS_ITYPE_MASK) >> \
(RX_PKT_CMPL_FLAGS_ITYPE_SFT - BNXT_PTYPE_TBL_TYPE_SFT))
@@ -168,6 +219,10 @@ bnxt_cfa_code_dynfield(struct rte_mbuf *mbuf)
(((f2) & CMPL_FLAGS2_VLAN_TUN_MSK) >> \
(RX_PKT_CMPL_FLAGS2_META_FORMAT_SFT - BNXT_PTYPE_TBL_VLAN_SFT))
+#define BNXT_CMPL_VLAN_TUN_TO_IDX_CRX(md) \
+ (((md) & CMPL_FLAGS2_VLAN_TUN_MSK_CRX) >> \
+ (RX_PKT_COMPRESS_CMPL_METADATA1_SFT - BNXT_PTYPE_TBL_VLAN_SFT))
+
#define BNXT_CMPL_IP_VER_TO_IDX(f2) \
(((f2) & RX_PKT_CMPL_FLAGS2_IP_TYPE) >> \
(RX_PKT_CMPL_FLAGS2_IP_TYPE_SFT - BNXT_PTYPE_TBL_IP_VER_SFT))