@@ -13,6 +13,7 @@
#include <string.h>
#include "test.h"
+#include "test_ring.h"
/*
* Ring
@@ -41,6 +42,35 @@ struct lcore_pair {
static volatile unsigned lcore_count = 0;
+static void
+test_ring_print_test_string(unsigned int api_type, int esize,
+ unsigned int bsz, double value)
+{
+ if (esize == -1)
+ printf("legacy APIs");
+ else
+ printf("elem APIs: element size %dB", esize);
+
+ if (api_type == TEST_RING_IGNORE_API_TYPE)
+ return;
+
+ if ((api_type & TEST_RING_THREAD_DEF) == TEST_RING_THREAD_DEF)
+ printf(": default enqueue/dequeue: ");
+ else if ((api_type & TEST_RING_THREAD_SPSC) == TEST_RING_THREAD_SPSC)
+ printf(": SP/SC: ");
+ else if ((api_type & TEST_RING_THREAD_MPMC) == TEST_RING_THREAD_MPMC)
+ printf(": MP/MC: ");
+
+ if ((api_type & TEST_RING_ELEM_SINGLE) == TEST_RING_ELEM_SINGLE)
+ printf("single: ");
+ else if ((api_type & TEST_RING_ELEM_BULK) == TEST_RING_ELEM_BULK)
+ printf("bulk (size: %u): ", bsz);
+ else if ((api_type & TEST_RING_ELEM_BURST) == TEST_RING_ELEM_BURST)
+ printf("burst (size: %u): ", bsz);
+
+ printf("%.2F\n", value);
+}
+
/**** Functions to analyse our core mask to get cores for different tests ***/
static int
@@ -117,27 +147,21 @@ get_two_sockets(struct lcore_pair *lcp)
/* Get cycle counts for dequeuing from an empty ring. Should be 2 or 3 cycles */
static void
-test_empty_dequeue(struct rte_ring *r)
+test_empty_dequeue(struct rte_ring *r, const int esize,
+ const unsigned int api_type)
{
- const unsigned iter_shift = 26;
- const unsigned iterations = 1<<iter_shift;
- unsigned i = 0;
+ const unsigned int iter_shift = 26;
+ const unsigned int iterations = 1 << iter_shift;
+ unsigned int i = 0;
void *burst[MAX_BURST];
- const uint64_t sc_start = rte_rdtsc();
+ const uint64_t start = rte_rdtsc();
for (i = 0; i < iterations; i++)
- rte_ring_sc_dequeue_bulk(r, burst, bulk_sizes[0], NULL);
- const uint64_t sc_end = rte_rdtsc();
+ test_ring_dequeue(r, burst, esize, bulk_sizes[0], api_type);
+ const uint64_t end = rte_rdtsc();
- const uint64_t mc_start = rte_rdtsc();
- for (i = 0; i < iterations; i++)
- rte_ring_mc_dequeue_bulk(r, burst, bulk_sizes[0], NULL);
- const uint64_t mc_end = rte_rdtsc();
-
- printf("SC empty dequeue: %.2F\n",
- (double)(sc_end-sc_start) / iterations);
- printf("MC empty dequeue: %.2F\n",
- (double)(mc_end-mc_start) / iterations);
+ test_ring_print_test_string(api_type, esize, bulk_sizes[0],
+ ((double)(end - start)) / iterations);
}
/*
@@ -151,19 +175,21 @@ struct thread_params {
};
/*
- * Function that uses rdtsc to measure timing for ring enqueue. Needs pair
- * thread running dequeue_bulk function
+ * Helper function to call bulk SP/MP enqueue functions.
+ * flag == 0 -> enqueue
+ * flag == 1 -> dequeue
*/
-static int
-enqueue_bulk(void *p)
+static __rte_always_inline int
+enqueue_dequeue_bulk_helper(const unsigned int flag, const int esize,
+ struct thread_params *p)
{
- const unsigned iter_shift = 23;
- const unsigned iterations = 1<<iter_shift;
- struct thread_params *params = p;
- struct rte_ring *r = params->r;
- const unsigned size = params->size;
- unsigned i;
- void *burst[MAX_BURST] = {0};
+ int ret;
+ const unsigned int iter_shift = 23;
+ const unsigned int iterations = 1 << iter_shift;
+ struct rte_ring *r = p->r;
+ unsigned int bsize = p->size;
+ unsigned int i;
+ void *burst = NULL;
#ifdef RTE_USE_C11_MEM_MODEL
if (__atomic_add_fetch(&lcore_count, 1, __ATOMIC_RELAXED) != 2)
@@ -173,23 +199,67 @@ enqueue_bulk(void *p)
while(lcore_count != 2)
rte_pause();
+ burst = test_ring_calloc(MAX_BURST, esize);
+ if (burst == NULL)
+ return -1;
+
const uint64_t sp_start = rte_rdtsc();
for (i = 0; i < iterations; i++)
- while (rte_ring_sp_enqueue_bulk(r, burst, size, NULL) == 0)
- rte_pause();
+ do {
+ if (flag == 0)
+ ret = test_ring_enqueue(r, burst, esize, bsize,
+ TEST_RING_THREAD_SPSC |
+ TEST_RING_ELEM_BULK);
+ else if (flag == 1)
+ ret = test_ring_dequeue(r, burst, esize, bsize,
+ TEST_RING_THREAD_SPSC |
+ TEST_RING_ELEM_BULK);
+ if (ret == 0)
+ rte_pause();
+ } while (!ret);
const uint64_t sp_end = rte_rdtsc();
const uint64_t mp_start = rte_rdtsc();
for (i = 0; i < iterations; i++)
- while (rte_ring_mp_enqueue_bulk(r, burst, size, NULL) == 0)
- rte_pause();
+ do {
+ if (flag == 0)
+ ret = test_ring_enqueue(r, burst, esize, bsize,
+ TEST_RING_THREAD_MPMC |
+ TEST_RING_ELEM_BULK);
+ else if (flag == 1)
+ ret = test_ring_dequeue(r, burst, esize, bsize,
+ TEST_RING_THREAD_MPMC |
+ TEST_RING_ELEM_BULK);
+ if (ret == 0)
+ rte_pause();
+ } while (!ret);
const uint64_t mp_end = rte_rdtsc();
- params->spsc = ((double)(sp_end - sp_start))/(iterations*size);
- params->mpmc = ((double)(mp_end - mp_start))/(iterations*size);
+ p->spsc = ((double)(sp_end - sp_start))/(iterations * bsize);
+ p->mpmc = ((double)(mp_end - mp_start))/(iterations * bsize);
return 0;
}
+/*
+ * Function that uses rdtsc to measure timing for ring enqueue. Needs pair
+ * thread running dequeue_bulk function
+ */
+static int
+enqueue_bulk(void *p)
+{
+ struct thread_params *params = p;
+
+ return enqueue_dequeue_bulk_helper(0, -1, params);
+}
+
+static int
+enqueue_bulk_16B(void *p)
+{
+ struct thread_params *params = p;
+
+ return enqueue_dequeue_bulk_helper(0, 16, params);
+}
+
/*
* Function that uses rdtsc to measure timing for ring dequeue. Needs pair
* thread running enqueue_bulk function
@@ -197,49 +267,38 @@ enqueue_bulk(void *p)
static int
dequeue_bulk(void *p)
{
- const unsigned iter_shift = 23;
- const unsigned iterations = 1<<iter_shift;
struct thread_params *params = p;
- struct rte_ring *r = params->r;
- const unsigned size = params->size;
- unsigned i;
- void *burst[MAX_BURST] = {0};
-
-#ifdef RTE_USE_C11_MEM_MODEL
- if (__atomic_add_fetch(&lcore_count, 1, __ATOMIC_RELAXED) != 2)
-#else
- if (__sync_add_and_fetch(&lcore_count, 1) != 2)
-#endif
- while(lcore_count != 2)
- rte_pause();
- const uint64_t sc_start = rte_rdtsc();
- for (i = 0; i < iterations; i++)
- while (rte_ring_sc_dequeue_bulk(r, burst, size, NULL) == 0)
- rte_pause();
- const uint64_t sc_end = rte_rdtsc();
+ return enqueue_dequeue_bulk_helper(1, -1, params);
+}
- const uint64_t mc_start = rte_rdtsc();
- for (i = 0; i < iterations; i++)
- while (rte_ring_mc_dequeue_bulk(r, burst, size, NULL) == 0)
- rte_pause();
- const uint64_t mc_end = rte_rdtsc();
+static int
+dequeue_bulk_16B(void *p)
+{
+ struct thread_params *params = p;
- params->spsc = ((double)(sc_end - sc_start))/(iterations*size);
- params->mpmc = ((double)(mc_end - mc_start))/(iterations*size);
- return 0;
+ return enqueue_dequeue_bulk_helper(1, 16, params);
}
/*
* Function that calls the enqueue and dequeue bulk functions on pairs of cores.
* used to measure ring perf between hyperthreads, cores and sockets.
*/
-static void
-run_on_core_pair(struct lcore_pair *cores, struct rte_ring *r,
- lcore_function_t f1, lcore_function_t f2)
+static int
+run_on_core_pair(struct lcore_pair *cores, struct rte_ring *r, const int esize)
{
+ lcore_function_t *f1, *f2;
struct thread_params param1 = {0}, param2 = {0};
unsigned i;
+
+ if (esize == -1) {
+ f1 = enqueue_bulk;
+ f2 = dequeue_bulk;
+ } else {
+ f1 = enqueue_bulk_16B;
+ f2 = dequeue_bulk_16B;
+ }
+
for (i = 0; i < sizeof(bulk_sizes)/sizeof(bulk_sizes[0]); i++) {
lcore_count = 0;
param1.size = param2.size = bulk_sizes[i];
@@ -251,14 +310,20 @@ run_on_core_pair(struct lcore_pair *cores, struct rte_ring *r,
} else {
rte_eal_remote_launch(f1, ¶m1, cores->c1);
rte_eal_remote_launch(f2, ¶m2, cores->c2);
- rte_eal_wait_lcore(cores->c1);
- rte_eal_wait_lcore(cores->c2);
+ if (rte_eal_wait_lcore(cores->c1) < 0)
+ return -1;
+ if (rte_eal_wait_lcore(cores->c2) < 0)
+ return -1;
}
- printf("SP/SC bulk enq/dequeue (size: %u): %.2F\n", bulk_sizes[i],
- param1.spsc + param2.spsc);
- printf("MP/MC bulk enq/dequeue (size: %u): %.2F\n", bulk_sizes[i],
- param1.mpmc + param2.mpmc);
+ test_ring_print_test_string(
+ TEST_RING_THREAD_SPSC | TEST_RING_ELEM_BULK,
+ esize, bulk_sizes[i], param1.spsc + param2.spsc);
+ test_ring_print_test_string(
+ TEST_RING_THREAD_MPMC | TEST_RING_ELEM_BULK,
+ esize, bulk_sizes[i], param1.mpmc + param2.mpmc);
}
+
+ return 0;
}
static rte_atomic32_t synchro;
@@ -267,7 +332,7 @@ static uint64_t queue_count[RTE_MAX_LCORE];
#define TIME_MS 100
static int
-load_loop_fn(void *p)
+load_loop_fn_helper(struct thread_params *p, const int esize)
{
uint64_t time_diff = 0;
uint64_t begin = 0;
@@ -275,7 +340,11 @@ load_loop_fn(void *p)
uint64_t lcount = 0;
const unsigned int lcore = rte_lcore_id();
struct thread_params *params = p;
- void *burst[MAX_BURST] = {0};
+ void *burst = NULL;
+
+ burst = test_ring_calloc(MAX_BURST, esize);
+ if (burst == NULL)
+ return -1;
/* wait synchro for slaves */
if (lcore != rte_get_master_lcore())
@@ -284,22 +353,49 @@ load_loop_fn(void *p)
begin = rte_get_timer_cycles();
while (time_diff < hz * TIME_MS / 1000) {
- rte_ring_mp_enqueue_bulk(params->r, burst, params->size, NULL);
- rte_ring_mc_dequeue_bulk(params->r, burst, params->size, NULL);
+ test_ring_enqueue(params->r, burst, esize, params->size,
+ TEST_RING_THREAD_MPMC | TEST_RING_ELEM_BULK);
+ test_ring_dequeue(params->r, burst, esize, params->size,
+ TEST_RING_THREAD_MPMC | TEST_RING_ELEM_BULK);
lcount++;
time_diff = rte_get_timer_cycles() - begin;
}
queue_count[lcore] = lcount;
+
+ rte_free(burst);
+
return 0;
}
static int
-run_on_all_cores(struct rte_ring *r)
+load_loop_fn(void *p)
+{
+ struct thread_params *params = p;
+
+ return load_loop_fn_helper(params, -1);
+}
+
+static int
+load_loop_fn_16B(void *p)
+{
+ struct thread_params *params = p;
+
+ return load_loop_fn_helper(params, 16);
+}
+
+static int
+run_on_all_cores(struct rte_ring *r, const int esize)
{
uint64_t total = 0;
struct thread_params param;
+ lcore_function_t *lcore_f;
unsigned int i, c;
+ if (esize == -1)
+ lcore_f = load_loop_fn;
+ else
+ lcore_f = load_loop_fn_16B;
+
memset(¶m, 0, sizeof(struct thread_params));
for (i = 0; i < RTE_DIM(bulk_sizes); i++) {
printf("\nBulk enq/dequeue count on size %u\n", bulk_sizes[i]);
@@ -308,13 +404,12 @@ run_on_all_cores(struct rte_ring *r)
/* clear synchro and start slaves */
rte_atomic32_set(&synchro, 0);
- if (rte_eal_mp_remote_launch(load_loop_fn, ¶m,
- SKIP_MASTER) < 0)
+ if (rte_eal_mp_remote_launch(lcore_f, ¶m, SKIP_MASTER) < 0)
return -1;
/* start synchro and launch test on master */
rte_atomic32_set(&synchro, 1);
- load_loop_fn(¶m);
+ lcore_f(¶m);
rte_eal_mp_wait_lcore();
@@ -335,155 +430,152 @@ run_on_all_cores(struct rte_ring *r)
* Test function that determines how long an enqueue + dequeue of a single item
* takes on a single lcore. Result is for comparison with the bulk enq+deq.
*/
-static void
-test_single_enqueue_dequeue(struct rte_ring *r)
+static int
+test_single_enqueue_dequeue(struct rte_ring *r, const int esize,
+ const unsigned int api_type)
{
- const unsigned iter_shift = 24;
- const unsigned iterations = 1<<iter_shift;
- unsigned i = 0;
+ const unsigned int iter_shift = 24;
+ const unsigned int iterations = 1 << iter_shift;
+ unsigned int i = 0;
void *burst = NULL;
- const uint64_t sc_start = rte_rdtsc();
- for (i = 0; i < iterations; i++) {
- rte_ring_sp_enqueue(r, burst);
- rte_ring_sc_dequeue(r, &burst);
- }
- const uint64_t sc_end = rte_rdtsc();
+ /* alloc dummy object pointers */
+ burst = test_ring_calloc(1, esize);
+ if (burst == NULL)
+ return -1;
- const uint64_t mc_start = rte_rdtsc();
+ const uint64_t start = rte_rdtsc();
for (i = 0; i < iterations; i++) {
- rte_ring_mp_enqueue(r, burst);
- rte_ring_mc_dequeue(r, &burst);
+ test_ring_enqueue(r, burst, esize, 1, api_type);
+ test_ring_dequeue(r, burst, esize, 1, api_type);
}
- const uint64_t mc_end = rte_rdtsc();
+ const uint64_t end = rte_rdtsc();
+
+ test_ring_print_test_string(api_type, esize, 1,
+ ((double)(end - start)) / iterations);
- printf("SP/SC single enq/dequeue: %.2F\n",
- ((double)(sc_end-sc_start)) / iterations);
- printf("MP/MC single enq/dequeue: %.2F\n",
- ((double)(mc_end-mc_start)) / iterations);
+ rte_free(burst);
+
+ return 0;
}
/*
- * Test that does both enqueue and dequeue on a core using the burst() API calls
- * instead of the bulk() calls used in other tests. Results should be the same
- * as for the bulk function called on a single lcore.
+ * Test that does both enqueue and dequeue on a core using the burst/bulk API
+ * calls Results should be the same as for the bulk function called on a
+ * single lcore.
*/
-static void
-test_burst_enqueue_dequeue(struct rte_ring *r)
+static int
+test_burst_bulk_enqueue_dequeue(struct rte_ring *r, const int esize,
+ const unsigned int api_type)
{
- const unsigned iter_shift = 23;
- const unsigned iterations = 1<<iter_shift;
- unsigned sz, i = 0;
- void *burst[MAX_BURST] = {0};
+ const unsigned int iter_shift = 23;
+ const unsigned int iterations = 1 << iter_shift;
+ unsigned int sz, i = 0;
+ void **burst = NULL;
- for (sz = 0; sz < sizeof(bulk_sizes)/sizeof(bulk_sizes[0]); sz++) {
- const uint64_t sc_start = rte_rdtsc();
- for (i = 0; i < iterations; i++) {
- rte_ring_sp_enqueue_burst(r, burst,
- bulk_sizes[sz], NULL);
- rte_ring_sc_dequeue_burst(r, burst,
- bulk_sizes[sz], NULL);
- }
- const uint64_t sc_end = rte_rdtsc();
+ burst = test_ring_calloc(MAX_BURST, esize);
+ if (burst == NULL)
+ return -1;
- const uint64_t mc_start = rte_rdtsc();
+ for (sz = 0; sz < RTE_DIM(bulk_sizes); sz++) {
+ const uint64_t start = rte_rdtsc();
for (i = 0; i < iterations; i++) {
- rte_ring_mp_enqueue_burst(r, burst,
- bulk_sizes[sz], NULL);
- rte_ring_mc_dequeue_burst(r, burst,
- bulk_sizes[sz], NULL);
+ test_ring_enqueue(r, burst, esize, bulk_sizes[sz],
+ api_type);
+ test_ring_dequeue(r, burst, esize, bulk_sizes[sz],
+ api_type);
}
- const uint64_t mc_end = rte_rdtsc();
-
- double mc_avg = ((double)(mc_end-mc_start) / iterations) /
- bulk_sizes[sz];
- double sc_avg = ((double)(sc_end-sc_start) / iterations) /
- bulk_sizes[sz];
+ const uint64_t end = rte_rdtsc();
- printf("SP/SC burst enq/dequeue (size: %u): %.2F\n",
- bulk_sizes[sz], sc_avg);
- printf("MP/MC burst enq/dequeue (size: %u): %.2F\n",
- bulk_sizes[sz], mc_avg);
+ test_ring_print_test_string(api_type, esize, bulk_sizes[sz],
+ ((double)(end - start)) / iterations);
}
-}
-/* Times enqueue and dequeue on a single lcore */
-static void
-test_bulk_enqueue_dequeue(struct rte_ring *r)
-{
- const unsigned iter_shift = 23;
- const unsigned iterations = 1<<iter_shift;
- unsigned sz, i = 0;
- void *burst[MAX_BURST] = {0};
+ rte_free(burst);
- for (sz = 0; sz < sizeof(bulk_sizes)/sizeof(bulk_sizes[0]); sz++) {
- const uint64_t sc_start = rte_rdtsc();
- for (i = 0; i < iterations; i++) {
- rte_ring_sp_enqueue_bulk(r, burst,
- bulk_sizes[sz], NULL);
- rte_ring_sc_dequeue_bulk(r, burst,
- bulk_sizes[sz], NULL);
- }
- const uint64_t sc_end = rte_rdtsc();
-
- const uint64_t mc_start = rte_rdtsc();
- for (i = 0; i < iterations; i++) {
- rte_ring_mp_enqueue_bulk(r, burst,
- bulk_sizes[sz], NULL);
- rte_ring_mc_dequeue_bulk(r, burst,
- bulk_sizes[sz], NULL);
- }
- const uint64_t mc_end = rte_rdtsc();
-
- double sc_avg = ((double)(sc_end-sc_start) /
- (iterations * bulk_sizes[sz]));
- double mc_avg = ((double)(mc_end-mc_start) /
- (iterations * bulk_sizes[sz]));
-
- printf("SP/SC bulk enq/dequeue (size: %u): %.2F\n", bulk_sizes[sz],
- sc_avg);
- printf("MP/MC bulk enq/dequeue (size: %u): %.2F\n", bulk_sizes[sz],
- mc_avg);
- }
+ return 0;
}
-static int
-test_ring_perf(void)
+/* Run all tests for a given element size */
+static __rte_always_inline int
+test_ring_perf_esize(const int esize)
{
struct lcore_pair cores;
struct rte_ring *r = NULL;
- r = rte_ring_create(RING_NAME, RING_SIZE, rte_socket_id(), 0);
+ /*
+ * Performance test for legacy/_elem APIs
+ * SP-SC/MP-MC, single
+ */
+ r = test_ring_create(RING_NAME, esize, RING_SIZE, rte_socket_id(), 0);
if (r == NULL)
return -1;
- printf("### Testing single element and burst enq/deq ###\n");
- test_single_enqueue_dequeue(r);
- test_burst_enqueue_dequeue(r);
+ printf("\n### Testing single element enq/deq ###\n");
+ if (test_single_enqueue_dequeue(r, esize,
+ TEST_RING_THREAD_SPSC | TEST_RING_ELEM_SINGLE) < 0)
+ return -1;
+ if (test_single_enqueue_dequeue(r, esize,
+ TEST_RING_THREAD_MPMC | TEST_RING_ELEM_SINGLE) < 0)
+ return -1;
+
+ printf("\n### Testing burst enq/deq ###\n");
+ if (test_burst_bulk_enqueue_dequeue(r, esize,
+ TEST_RING_THREAD_SPSC | TEST_RING_ELEM_BURST) < 0)
+ return -1;
+ if (test_burst_bulk_enqueue_dequeue(r, esize,
+ TEST_RING_THREAD_MPMC | TEST_RING_ELEM_BURST) < 0)
+ return -1;
- printf("\n### Testing empty dequeue ###\n");
- test_empty_dequeue(r);
+ printf("\n### Testing bulk enq/deq ###\n");
+ if (test_burst_bulk_enqueue_dequeue(r, esize,
+ TEST_RING_THREAD_SPSC | TEST_RING_ELEM_BULK) < 0)
+ return -1;
+ if (test_burst_bulk_enqueue_dequeue(r, esize,
+ TEST_RING_THREAD_MPMC | TEST_RING_ELEM_BULK) < 0)
+ return -1;
- printf("\n### Testing using a single lcore ###\n");
- test_bulk_enqueue_dequeue(r);
+ printf("\n### Testing empty bulk deq ###\n");
+ test_empty_dequeue(r, esize,
+ TEST_RING_THREAD_SPSC | TEST_RING_ELEM_BULK);
+ test_empty_dequeue(r, esize,
+ TEST_RING_THREAD_MPMC | TEST_RING_ELEM_BULK);
if (get_two_hyperthreads(&cores) == 0) {
printf("\n### Testing using two hyperthreads ###\n");
- run_on_core_pair(&cores, r, enqueue_bulk, dequeue_bulk);
+ if (run_on_core_pair(&cores, r, esize) < 0)
+ return -1;
}
if (get_two_cores(&cores) == 0) {
printf("\n### Testing using two physical cores ###\n");
- run_on_core_pair(&cores, r, enqueue_bulk, dequeue_bulk);
+ if (run_on_core_pair(&cores, r, esize) < 0)
+ return -1;
}
if (get_two_sockets(&cores) == 0) {
printf("\n### Testing using two NUMA nodes ###\n");
- run_on_core_pair(&cores, r, enqueue_bulk, dequeue_bulk);
+ if (run_on_core_pair(&cores, r, esize) < 0)
+ return -1;
}
printf("\n### Testing using all slave nodes ###\n");
- run_on_all_cores(r);
+ if (run_on_all_cores(r, esize) < 0)
+ return -1;
rte_ring_free(r);
+
+ return 0;
+}
+
+static int
+test_ring_perf(void)
+{
+ /* Run all the tests for different element sizes */
+ if (test_ring_perf_esize(-1) == -1)
+ return -1;
+
+ if (test_ring_perf_esize(16) == -1)
+ return -1;
+
return 0;
}