@@ -81,7 +81,7 @@ swap_udp(struct rte_udp_hdr *udp_hdr)
* 2,3,4. Swaps source and destination for MAC, IPv4/IPv6, UDP/TCP.
* Parses each layer and swaps it. When the next layer doesn't match it stops.
*/
-static void
+static bool
pkt_burst_5tuple_swap(struct fwd_stream *fs)
{
struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
@@ -105,10 +105,6 @@ pkt_burst_5tuple_swap(struct fwd_stream *fs)
uint8_t *byte;
} h;
- uint64_t start_tsc = 0;
-
- get_start_cycles(&start_tsc);
-
/*
* Receive a burst of packets and forward them.
*/
@@ -116,7 +112,7 @@ pkt_burst_5tuple_swap(struct fwd_stream *fs)
nb_pkt_per_burst);
inc_rx_burst_stats(fs, nb_rx);
if (unlikely(nb_rx == 0))
- return;
+ return false;
fs->rx_packets += nb_rx;
txp = &ports[fs->tx_port];
@@ -180,7 +176,8 @@ pkt_burst_5tuple_swap(struct fwd_stream *fs)
fs->fwd_dropped += (nb_rx - nb_tx);
rte_pktmbuf_free_bulk(&pkts_burst[nb_tx], nb_rx - nb_tx);
}
- get_end_cycles(fs, start_tsc);
+
+ return true;
}
static void
@@ -828,7 +828,7 @@ pkts_ip_csum_recalc(struct rte_mbuf **pkts_burst, const uint16_t nb_pkts, uint64
* IP, UDP, TCP and SCTP flags always concern the inner layer. The
* OUTER_IP is only useful for tunnel packets.
*/
-static void
+static bool
pkt_burst_checksum_forward(struct fwd_stream *fs)
{
struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
@@ -859,16 +859,12 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)
uint32_t rx_bad_outer_ip_csum;
struct testpmd_offload_info info;
- uint64_t start_tsc = 0;
-
- get_start_cycles(&start_tsc);
-
/* receive a burst of packet */
nb_rx = rte_eth_rx_burst(fs->rx_port, fs->rx_queue, pkts_burst,
nb_pkt_per_burst);
inc_rx_burst_stats(fs, nb_rx);
if (unlikely(nb_rx == 0))
- return;
+ return false;
fs->rx_packets += nb_rx;
rx_bad_ip_csum = 0;
@@ -1202,7 +1198,7 @@ pkt_burst_checksum_forward(struct fwd_stream *fs)
rte_pktmbuf_free_bulk(&tx_pkts_burst[nb_tx], nb_prep - nb_tx);
}
- get_end_cycles(fs, start_tsc);
+ return true;
}
static void
@@ -58,7 +58,7 @@ RTE_DEFINE_PER_LCORE(int, _next_flow);
* terminate receive traffic. Received traffic is simply discarded, but we
* still do so in order to maintain traffic statistics.
*/
-static void
+static bool
pkt_burst_flow_gen(struct fwd_stream *fs)
{
unsigned pkt_size = tx_pkt_length - 4; /* Adjust FCS */
@@ -77,11 +77,8 @@ pkt_burst_flow_gen(struct fwd_stream *fs)
uint16_t nb_clones = nb_pkt_flowgen_clones;
uint32_t retry;
uint64_t tx_offloads;
- uint64_t start_tsc = 0;
int next_flow = RTE_PER_LCORE(_next_flow);
- get_start_cycles(&start_tsc);
-
/* Receive a burst of packets and discard them. */
nb_rx = rte_eth_rx_burst(fs->rx_port, fs->rx_queue, pkts_burst,
nb_pkt_per_burst);
@@ -192,7 +189,7 @@ pkt_burst_flow_gen(struct fwd_stream *fs)
RTE_PER_LCORE(_next_flow) = next_flow;
- get_end_cycles(fs, start_tsc);
+ return true;
}
static int
@@ -269,7 +269,7 @@ ipv4_hdr_cksum(struct rte_ipv4_hdr *ip_h)
* Receive a burst of packets, lookup for ICMP echo requests, and, if any,
* send back ICMP echo replies.
*/
-static void
+static bool
reply_to_icmp_echo_rqsts(struct fwd_stream *fs)
{
struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
@@ -292,9 +292,6 @@ reply_to_icmp_echo_rqsts(struct fwd_stream *fs)
uint32_t cksum;
uint8_t i;
int l2_len;
- uint64_t start_tsc = 0;
-
- get_start_cycles(&start_tsc);
/*
* First, receive a burst of packets.
@@ -303,7 +300,7 @@ reply_to_icmp_echo_rqsts(struct fwd_stream *fs)
nb_pkt_per_burst);
inc_rx_burst_stats(fs, nb_rx);
if (unlikely(nb_rx == 0))
- return;
+ return false;
fs->rx_packets += nb_rx;
nb_replies = 0;
@@ -507,7 +504,7 @@ reply_to_icmp_echo_rqsts(struct fwd_stream *fs)
}
}
- get_end_cycles(fs, start_tsc);
+ return true;
}
static void
@@ -89,7 +89,7 @@ port_ieee1588_tx_timestamp_check(portid_t pi)
(wait_us == 1) ? "" : "s");
}
-static void
+static bool
ieee1588_packet_fwd(struct fwd_stream *fs)
{
struct rte_mbuf *mb;
@@ -103,7 +103,7 @@ ieee1588_packet_fwd(struct fwd_stream *fs)
* Receive 1 packet at a time.
*/
if (rte_eth_rx_burst(fs->rx_port, fs->rx_queue, &mb, 1) == 0)
- return;
+ return false;
fs->rx_packets += 1;
@@ -126,14 +126,14 @@ ieee1588_packet_fwd(struct fwd_stream *fs)
(unsigned) mb->pkt_len);
}
rte_pktmbuf_free(mb);
- return;
+ return false;
}
if (eth_type != RTE_ETHER_TYPE_1588) {
printf("Port %u Received NON PTP packet incorrectly"
" detected by hardware\n",
fs->rx_port);
rte_pktmbuf_free(mb);
- return;
+ return false;
}
/*
@@ -147,14 +147,14 @@ ieee1588_packet_fwd(struct fwd_stream *fs)
" protocol version 0x%x (should be 0x02)\n",
fs->rx_port, ptp_hdr->version);
rte_pktmbuf_free(mb);
- return;
+ return false;
}
if (ptp_hdr->msg_id != PTP_SYNC_MESSAGE) {
printf("Port %u Received PTP V2 Ethernet frame with unexpected"
" message ID 0x%x (expected 0x0 - PTP_SYNC_MESSAGE)\n",
fs->rx_port, ptp_hdr->msg_id);
rte_pktmbuf_free(mb);
- return;
+ return false;
}
printf("Port %u IEEE1588 PTP V2 SYNC Message filtered by hardware\n",
fs->rx_port);
@@ -168,7 +168,7 @@ ieee1588_packet_fwd(struct fwd_stream *fs)
" by hardware\n",
fs->rx_port);
rte_pktmbuf_free(mb);
- return;
+ return false;
}
/* For i40e we need the timesync register index. It is ignored for the
@@ -188,7 +188,7 @@ ieee1588_packet_fwd(struct fwd_stream *fs)
printf("Port %u sent PTP packet dropped\n", fs->tx_port);
fs->fwd_dropped += 1;
rte_pktmbuf_free(mb);
- return;
+ return false;
}
fs->tx_packets += 1;
@@ -196,6 +196,7 @@ ieee1588_packet_fwd(struct fwd_stream *fs)
* Check the TX timestamp.
*/
port_ieee1588_tx_timestamp_check(fs->tx_port);
+ return true;
}
static int
@@ -41,16 +41,13 @@
* This is the fastest possible forwarding operation, as it does not access
* to packets data.
*/
-static void
+static bool
pkt_burst_io_forward(struct fwd_stream *fs)
{
struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
uint16_t nb_rx;
uint16_t nb_tx;
uint32_t retry;
- uint64_t start_tsc = 0;
-
- get_start_cycles(&start_tsc);
/*
* Receive a burst of packets and forward them.
@@ -59,7 +56,7 @@ pkt_burst_io_forward(struct fwd_stream *fs)
pkts_burst, nb_pkt_per_burst);
inc_rx_burst_stats(fs, nb_rx);
if (unlikely(nb_rx == 0))
- return;
+ return false;
fs->rx_packets += nb_rx;
nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue,
@@ -82,7 +79,7 @@ pkt_burst_io_forward(struct fwd_stream *fs)
rte_pktmbuf_free_bulk(&pkts_burst[nb_tx], nb_rx - nb_tx);
}
- get_end_cycles(fs, start_tsc);
+ return true;
}
static void
@@ -41,7 +41,7 @@
* Change the source and the destination Ethernet addressed of packets
* before forwarding them.
*/
-static void
+static bool
pkt_burst_mac_forward(struct fwd_stream *fs)
{
struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
@@ -54,9 +54,6 @@ pkt_burst_mac_forward(struct fwd_stream *fs)
uint16_t i;
uint64_t ol_flags = 0;
uint64_t tx_offloads;
- uint64_t start_tsc = 0;
-
- get_start_cycles(&start_tsc);
/*
* Receive a burst of packets and forward them.
@@ -65,7 +62,7 @@ pkt_burst_mac_forward(struct fwd_stream *fs)
nb_pkt_per_burst);
inc_rx_burst_stats(fs, nb_rx);
if (unlikely(nb_rx == 0))
- return;
+ return false;
fs->rx_packets += nb_rx;
txp = &ports[fs->tx_port];
@@ -113,7 +110,7 @@ pkt_burst_mac_forward(struct fwd_stream *fs)
rte_pktmbuf_free_bulk(&pkts_burst[nb_tx], nb_rx - nb_tx);
}
- get_end_cycles(fs, start_tsc);
+ return true;
}
static void
@@ -47,7 +47,7 @@
* MAC swap forwarding mode: Swap the source and the destination Ethernet
* addresses of packets before forwarding them.
*/
-static void
+static bool
pkt_burst_mac_swap(struct fwd_stream *fs)
{
struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
@@ -55,9 +55,6 @@ pkt_burst_mac_swap(struct fwd_stream *fs)
uint16_t nb_rx;
uint16_t nb_tx;
uint32_t retry;
- uint64_t start_tsc = 0;
-
- get_start_cycles(&start_tsc);
/*
* Receive a burst of packets and forward them.
@@ -66,7 +63,7 @@ pkt_burst_mac_swap(struct fwd_stream *fs)
nb_pkt_per_burst);
inc_rx_burst_stats(fs, nb_rx);
if (unlikely(nb_rx == 0))
- return;
+ return false;
fs->rx_packets += nb_rx;
txp = &ports[fs->tx_port];
@@ -91,7 +88,8 @@ pkt_burst_mac_swap(struct fwd_stream *fs)
fs->fwd_dropped += (nb_rx - nb_tx);
rte_pktmbuf_free_bulk(&pkts_burst[nb_tx], nb_rx - nb_tx);
}
- get_end_cycles(fs, start_tsc);
+
+ return true;
}
static void
@@ -134,14 +134,13 @@ drop_pkts(struct rte_mbuf **pkts, uint16_t nb_rx, uint16_t nb_tx)
* out of the FIFO
* 4. Cases 2 and 3 combined
*/
-static void
+static bool
pkt_burst_noisy_vnf(struct fwd_stream *fs)
{
const uint64_t freq_khz = rte_get_timer_hz() / 1000;
struct noisy_config *ncf = noisy_cfg[fs->rx_port];
struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
struct rte_mbuf *tmp_pkts[MAX_PKT_BURST];
- uint64_t start_tsc = 0;
uint16_t nb_deqd = 0;
uint16_t nb_rx = 0;
uint16_t nb_tx = 0;
@@ -151,8 +150,6 @@ pkt_burst_noisy_vnf(struct fwd_stream *fs)
bool needs_flush = false;
uint64_t now;
- get_start_cycles(&start_tsc);
-
nb_rx = rte_eth_rx_burst(fs->rx_port, fs->rx_queue,
pkts_burst, nb_pkt_per_burst);
inc_rx_burst_stats(fs, nb_rx);
@@ -221,8 +218,7 @@ pkt_burst_noisy_vnf(struct fwd_stream *fs)
ncf->prev_time = rte_get_timer_cycles();
}
end:
- if (nb_rx > 0 || nb_tx > 0)
- get_end_cycles(fs, start_tsc);
+ return nb_rx > 0 || nb_tx > 0;
}
#define NOISY_STRSIZE 256
@@ -41,14 +41,11 @@
/*
* Received a burst of packets.
*/
-static void
+static bool
pkt_burst_receive(struct fwd_stream *fs)
{
struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
uint16_t nb_rx;
- uint64_t start_tsc = 0;
-
- get_start_cycles(&start_tsc);
/*
* Receive a burst of packets.
@@ -57,12 +54,12 @@ pkt_burst_receive(struct fwd_stream *fs)
nb_pkt_per_burst);
inc_rx_burst_stats(fs, nb_rx);
if (unlikely(nb_rx == 0))
- return;
+ return false;
fs->rx_packets += nb_rx;
rte_pktmbuf_free_bulk(pkts_burst, nb_rx);
- get_end_cycles(fs, start_tsc);
+ return true;
}
static void
@@ -89,21 +89,20 @@ forward_shared_rxq(struct fwd_stream *fs, uint16_t nb_rx,
&pkts_burst[nb_rx - nb_sub_burst]);
}
-static void
+static bool
shared_rxq_fwd(struct fwd_stream *fs)
{
struct rte_mbuf *pkts_burst[nb_pkt_per_burst];
uint16_t nb_rx;
- uint64_t start_tsc = 0;
- get_start_cycles(&start_tsc);
nb_rx = rte_eth_rx_burst(fs->rx_port, fs->rx_queue, pkts_burst,
nb_pkt_per_burst);
inc_rx_burst_stats(fs, nb_rx);
if (unlikely(nb_rx == 0))
- return;
+ return false;
forward_shared_rxq(fs, nb_rx, pkts_burst);
- get_end_cycles(fs, start_tsc);
+
+ return true;
}
static void
@@ -2271,9 +2271,19 @@ run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd)
nb_fs = fc->stream_nb;
prev_tsc = rte_rdtsc();
do {
- for (sm_id = 0; sm_id < nb_fs; sm_id++)
- if (!fsm[sm_id]->disabled)
- (*pkt_fwd)(fsm[sm_id]);
+ for (sm_id = 0; sm_id < nb_fs; sm_id++) {
+ struct fwd_stream *fs = fsm[sm_id];
+ uint64_t start_fs_tsc = 0;
+ bool busy;
+
+ if (fs->disabled)
+ continue;
+ if (record_core_cycles)
+ start_fs_tsc = rte_rdtsc();
+ busy = (*pkt_fwd)(fs);
+ if (record_core_cycles && busy)
+ fs->busy_cycles += rte_rdtsc() - start_fs_tsc;
+ }
#ifdef RTE_LIB_BITRATESTATS
if (bitrate_enabled != 0 &&
bitrate_lcore_id == rte_lcore_id()) {
@@ -382,7 +382,7 @@ struct fwd_lcore {
typedef int (*port_fwd_begin_t)(portid_t pi);
typedef void (*port_fwd_end_t)(portid_t pi);
typedef void (*stream_init_t)(struct fwd_stream *fs);
-typedef void (*packet_fwd_t)(struct fwd_stream *fs);
+typedef bool (*packet_fwd_t)(struct fwd_stream *fs);
struct fwd_engine {
const char *fwd_mode_name; /**< Forwarding mode name. */
@@ -837,20 +837,6 @@ mbuf_pool_find(unsigned int sock_id, uint16_t idx)
return rte_mempool_lookup((const char *)pool_name);
}
-static inline void
-get_start_cycles(uint64_t *start_tsc)
-{
- if (record_core_cycles)
- *start_tsc = rte_rdtsc();
-}
-
-static inline void
-get_end_cycles(struct fwd_stream *fs, uint64_t start_tsc)
-{
- if (record_core_cycles)
- fs->busy_cycles += rte_rdtsc() - start_tsc;
-}
-
static inline void
inc_rx_burst_stats(struct fwd_stream *fs, uint16_t nb_rx)
{
@@ -323,7 +323,7 @@ pkt_burst_prepare(struct rte_mbuf *pkt, struct rte_mempool *mbp,
/*
* Transmit a burst of multi-segments packets.
*/
-static void
+static bool
pkt_burst_transmit(struct fwd_stream *fs)
{
struct rte_mbuf *pkts_burst[MAX_PKT_BURST];
@@ -337,9 +337,6 @@ pkt_burst_transmit(struct fwd_stream *fs)
uint32_t retry;
uint64_t ol_flags = 0;
uint64_t tx_offloads;
- uint64_t start_tsc = 0;
-
- get_start_cycles(&start_tsc);
mbp = current_fwd_lcore()->mbp;
txp = &ports[fs->tx_port];
@@ -392,7 +389,7 @@ pkt_burst_transmit(struct fwd_stream *fs)
}
if (nb_pkt == 0)
- return;
+ return false;
nb_tx = rte_eth_tx_burst(fs->tx_port, fs->tx_queue, pkts_burst, nb_pkt);
@@ -424,7 +421,7 @@ pkt_burst_transmit(struct fwd_stream *fs)
rte_pktmbuf_free_bulk(&pkts_burst[nb_tx], nb_pkt - nb_tx);
}
- get_end_cycles(fs, start_tsc);
+ return true;
}
static int