@@ -67,6 +67,9 @@ struct rte_swx_ctl_pipeline_info {
/** Number of meter arrays. */
uint32_t n_metarrays;
+
+ /** Number of RSS objects. */
+ uint32_t n_rss;
};
/**
@@ -1521,6 +1524,95 @@ rte_swx_ctl_meter_stats_read_with_key(struct rte_swx_pipeline *p,
uint8_t *table_key,
struct rte_swx_ctl_meter_stats *stats);
+/*
+ * RSS Query and Configuration API.
+ */
+
+/** RSS object info. */
+struct rte_swx_ctl_rss_info {
+ /** RSS object name. */
+ char name[RTE_SWX_CTL_NAME_SIZE];
+};
+
+/**
+ * RSS object info get
+ *
+ * @param[in] p
+ * Pipeline handle.
+ * @param[in] rss_obj_id
+ * RSS object ID (0 .. *n_rss* - 1).
+ * @param[out] rss
+ * RSS object info.
+ * @return
+ * 0 on success or the following error codes otherwise:
+ * -EINVAL: Invalid argument.
+ */
+__rte_experimental
+int
+rte_swx_ctl_rss_info_get(struct rte_swx_pipeline *p,
+ uint32_t rss_obj_id,
+ struct rte_swx_ctl_rss_info *rss);
+
+/**
+ * RSS object key size read
+ *
+ * @param[in] p
+ * Pipeline handle.
+ * @param[in] rss_name
+ * RSS object name.
+ * @param[out] key_size
+ * RSS key size in bytes.
+ * @return
+ * 0 on success or the following error codes otherwise:
+ * -EINVAL: Invalid argument.
+ */
+__rte_experimental
+int
+rte_swx_ctl_pipeline_rss_key_size_read(struct rte_swx_pipeline *p,
+ const char *rss_name,
+ uint32_t *key_size);
+
+/**
+ * RSS object key read
+ *
+ * @param[in] p
+ * Pipeline handle.
+ * @param[in] rss_name
+ * RSS object name.
+ * @param[out] key
+ * RSS key. Must be pre-allocated by the caller to store *key_size* bytes.
+ * @return
+ * 0 on success or the following error codes otherwise:
+ * -EINVAL: Invalid argument.
+ */
+__rte_experimental
+int
+rte_swx_ctl_pipeline_rss_key_read(struct rte_swx_pipeline *p,
+ const char *rss_name,
+ uint8_t *key);
+
+/**
+ * RSS object key write
+ *
+ * @param[in] p
+ * Pipeline handle.
+ * @param[in] rss_name
+ * RSS object name.
+ * @param[in] key_size
+ * RSS key size in bytes. Must be at least 4 and a power of 2.
+ * @param[in] key
+ * RSS key.
+ * @return
+ * 0 on success or the following error codes otherwise:
+ * -EINVAL: Invalid argument.
+ */
+__rte_experimental
+int
+rte_swx_ctl_pipeline_rss_key_write(struct rte_swx_pipeline *p,
+ const char *rss_name,
+ uint32_t key_size,
+ uint8_t *key);
+
/**
* Pipeline control free
*
@@ -1255,6 +1255,100 @@ hash_func_free(struct rte_swx_pipeline *p)
}
}
+/*
+ * RSS.
+ */
+static struct rss *
+rss_find(struct rte_swx_pipeline *p, const char *name)
+{
+ struct rss *elem;
+
+ TAILQ_FOREACH(elem, &p->rss, node)
+ if (strcmp(elem->name, name) == 0)
+ return elem;
+
+ return NULL;
+}
+
+static struct rss *
+rss_find_by_id(struct rte_swx_pipeline *p, uint32_t rss_obj_id)
+{
+ struct rss *elem;
+
+ TAILQ_FOREACH(elem, &p->rss, node)
+ if (elem->id == rss_obj_id)
+ return elem;
+
+ return NULL;
+}
+
+int
+rte_swx_pipeline_rss_config(struct rte_swx_pipeline *p, const char *name)
+{
+ struct rss *r;
+
+ CHECK(p, EINVAL);
+
+ CHECK_NAME(name, EINVAL);
+ CHECK(!rss_find(p, name), EEXIST);
+
+ /* Memory allocation. */
+ r = calloc(1, sizeof(struct rss));
+ CHECK(r, ENOMEM);
+
+ /* Node initialization. */
+ strcpy(r->name, name);
+ r->id = p->n_rss;
+
+ /* Node add to tailq. */
+ TAILQ_INSERT_TAIL(&p->rss, r, node);
+ p->n_rss++;
+
+ return 0;
+}
+
+static uint32_t rss_runtime_default[] = {1, 0};
+
+static int
+rss_build(struct rte_swx_pipeline *p)
+{
+ struct rss *rss;
+
+ /* Memory allocation. */
+ p->rss_runtime = calloc(p->n_rss, sizeof(struct rss_runtime *));
+ CHECK(p->rss_runtime, ENOMEM);
+
+ /* RSS. */
+ TAILQ_FOREACH(rss, &p->rss, node)
+ p->rss_runtime[rss->id] = (struct rss_runtime *)rss_runtime_default;
+
+ return 0;
+}
+
+static void
+rss_build_free(struct rte_swx_pipeline *p)
+{
+ free(p->rss_runtime);
+ p->rss_runtime = NULL;
+}
+
+static void
+rss_free(struct rte_swx_pipeline *p)
+{
+ rss_build_free(p);
+
+ for ( ; ; ) {
+ struct rss *elem;
+
+ elem = TAILQ_FIRST(&p->rss);
+ if (!elem)
+ break;
+
+ TAILQ_REMOVE(&p->rss, elem, node);
+ free(elem);
+ }
+}
+
/*
* Header.
*/
@@ -3002,6 +3096,63 @@ instr_hash_func_exec(struct rte_swx_pipeline *p)
thread_ip_inc(p);
}
+/*
+ * rss.
+ */
+static int
+instr_rss_translate(struct rte_swx_pipeline *p,
+ struct action *action,
+ char **tokens,
+ int n_tokens,
+ struct instruction *instr,
+ struct instruction_data *data __rte_unused)
+{
+ struct rss *rss;
+ struct field *dst, *src_first, *src_last;
+ uint32_t src_struct_id_first = 0, src_struct_id_last = 0;
+
+ CHECK(n_tokens == 5, EINVAL);
+
+ rss = rss_find(p, tokens[1]);
+ CHECK(rss, EINVAL);
+
+ dst = metadata_field_parse(p, tokens[2]);
+ CHECK(dst, EINVAL);
+ CHECK(dst->n_bits <= 64, EINVAL);
+
+ src_first = struct_field_parse(p, action, tokens[3], &src_struct_id_first);
+ CHECK(src_first, EINVAL);
+
+ src_last = struct_field_parse(p, action, tokens[4], &src_struct_id_last);
+ CHECK(src_last, EINVAL);
+ CHECK(!src_last->var_size, EINVAL);
+ CHECK(src_struct_id_first == src_struct_id_last, EINVAL);
+
+ instr->type = INSTR_RSS;
+ instr->rss.rss_obj_id = (uint8_t)rss->id;
+ instr->rss.dst.offset = (uint8_t)dst->offset / 8;
+ instr->rss.dst.n_bits = (uint8_t)dst->n_bits;
+ instr->rss.src.struct_id = (uint8_t)src_struct_id_first;
+ instr->rss.src.offset = (uint16_t)src_first->offset / 8;
+ instr->rss.src.n_bytes = (uint16_t)((src_last->offset + src_last->n_bits -
+ src_first->offset) / 8);
+
+ return 0;
+}
+
+static inline void
+instr_rss_exec(struct rte_swx_pipeline *p)
+{
+ struct thread *t = &p->threads[p->thread_id];
+ struct instruction *ip = t->ip;
+
+ /* Extern function execute. */
+ __instr_rss_exec(p, t, ip);
+
+ /* Thread. */
+ thread_ip_inc(p);
+}
+
/*
* mov.
*/
@@ -6397,6 +6548,14 @@ instr_translate(struct rte_swx_pipeline *p,
instr,
data);
+ if (!strcmp(tokens[tpos], "rss"))
+ return instr_rss_translate(p,
+ action,
+ &tokens[tpos],
+ n_tokens - tpos,
+ instr,
+ data);
+
if (!strcmp(tokens[tpos], "jmp"))
return instr_jmp_translate(p,
action,
@@ -7371,6 +7530,7 @@ static instr_exec_t instruction_table[] = {
[INSTR_EXTERN_OBJ] = instr_extern_obj_exec,
[INSTR_EXTERN_FUNC] = instr_extern_func_exec,
[INSTR_HASH_FUNC] = instr_hash_func_exec,
+ [INSTR_RSS] = instr_rss_exec,
[INSTR_JMP] = instr_jmp_exec,
[INSTR_JMP_VALID] = instr_jmp_valid_exec,
@@ -10001,6 +10161,7 @@ rte_swx_pipeline_free(struct rte_swx_pipeline *p)
instruction_table_free(p);
metadata_free(p);
header_free(p);
+ rss_free(p);
hash_func_free(p);
extern_func_free(p);
extern_obj_free(p);
@@ -10149,6 +10310,7 @@ rte_swx_pipeline_config(struct rte_swx_pipeline **p, const char *name, int numa_
TAILQ_INIT(&pipeline->extern_objs);
TAILQ_INIT(&pipeline->extern_funcs);
TAILQ_INIT(&pipeline->hash_funcs);
+ TAILQ_INIT(&pipeline->rss);
TAILQ_INIT(&pipeline->headers);
TAILQ_INIT(&pipeline->actions);
TAILQ_INIT(&pipeline->table_types);
@@ -10263,6 +10425,10 @@ rte_swx_pipeline_build(struct rte_swx_pipeline *p)
if (status)
goto error;
+ status = rss_build(p);
+ if (status)
+ goto error;
+
status = header_build(p);
if (status)
goto error;
@@ -10318,6 +10484,7 @@ rte_swx_pipeline_build(struct rte_swx_pipeline *p)
instruction_table_build_free(p);
metadata_build_free(p);
header_build_free(p);
+ rss_build_free(p);
hash_func_build_free(p);
extern_func_build_free(p);
extern_obj_build_free(p);
@@ -10382,6 +10549,7 @@ rte_swx_ctl_pipeline_info_get(struct rte_swx_pipeline *p,
pipeline->n_learners = p->n_learners;
pipeline->n_regarrays = p->n_regarrays;
pipeline->n_metarrays = p->n_metarrays;
+ pipeline->n_rss = p->n_rss;
return 0;
}
@@ -11406,6 +11574,117 @@ rte_swx_ctl_meter_stats_read_with_key(struct rte_swx_pipeline *p,
return rte_swx_ctl_meter_stats_read(p, metarray_name, entry_id, stats);
}
+int
+rte_swx_ctl_rss_info_get(struct rte_swx_pipeline *p,
+ uint32_t rss_obj_id,
+ struct rte_swx_ctl_rss_info *info)
+{
+ struct rss *rss;
+
+ /* Check the input arguments. */
+ if (!p || !info)
+ return -EINVAL;
+
+ rss = rss_find_by_id(p, rss_obj_id);
+ if (!rss)
+ return -EINVAL;
+
+ /* Read from the internal data structures. */
+ strcpy(info->name, rss->name);
+ return 0;
+}
+
+int
+rte_swx_ctl_pipeline_rss_key_size_read(struct rte_swx_pipeline *p,
+ const char *rss_name,
+ uint32_t *key_size)
+{
+ struct rss *rss;
+ struct rss_runtime *r;
+
+ /* Check the input arguments. */
+ CHECK(p, EINVAL);
+
+ CHECK_NAME(rss_name, EINVAL);
+ rss = rss_find(p, rss_name);
+ CHECK(rss, EINVAL);
+ r = p->rss_runtime[rss->id];
+
+ CHECK(key_size, EINVAL);
+
+ /* Read from the internal data structures. */
+ *key_size = r->key_size * 4;
+
+ return 0;
+}
+
+int
+rte_swx_ctl_pipeline_rss_key_read(struct rte_swx_pipeline *p,
+ const char *rss_name,
+ uint8_t *key)
+{
+ struct rss *rss;
+ struct rss_runtime *r;
+ uint32_t *key32 = (uint32_t *)key, i;
+
+ /* Check the input arguments. */
+ CHECK(p, EINVAL);
+
+ CHECK_NAME(rss_name, EINVAL);
+ rss = rss_find(p, rss_name);
+ CHECK(rss, EINVAL);
+ r = p->rss_runtime[rss->id];
+
+ CHECK(key, EINVAL);
+
+ /* Read from the internal data structures. */
+ for (i = 0; i < r->key_size; i++)
+ key32[i] = rte_be_to_cpu_32(r->key[i]);
+
+ return 0;
+}
+
+int
+rte_swx_ctl_pipeline_rss_key_write(struct rte_swx_pipeline *p,
+ const char *rss_name,
+ uint32_t key_size,
+ uint8_t *key)
+{
+ struct rss *rss;
+ struct rss_runtime *r, *r_new;
+ uint32_t *key32 = (uint32_t *)key, i;
+
+ /* Check the input arguments. */
+ CHECK(p, EINVAL);
+
+ CHECK_NAME(rss_name, EINVAL);
+ rss = rss_find(p, rss_name);
+ CHECK(rss, EINVAL);
+ r = p->rss_runtime[rss->id];
+
+ CHECK((key_size >= 4) && rte_is_power_of_2(key_size), EINVAL);
+ CHECK(key, EINVAL);
+ key_size /= 4;
+
+ /* Allocate new RSS run-time entry. */
+ r_new = malloc(sizeof(struct rss_runtime) + key_size * sizeof(uint32_t));
+ if (!r_new)
+ return -ENOMEM;
+
+ /* Fill in the new RSS run-time entry. */
+ r_new->key_size = key_size;
+ for (i = 0; i < key_size; i++)
+ r_new->key[i] = rte_cpu_to_be_32(key32[i]);
+
+ /* Commit the RSS run-time change atomically. */
+ p->rss_runtime[rss->id] = r_new;
+
+ /* Free the old RSS run-time entry. */
+ free(r);
+
+ return 0;
+}
+
/*
* Pipeline compilation.
*/
@@ -11579,6 +11858,7 @@ instr_type_to_name(struct instruction *instr)
case INSTR_EXTERN_OBJ: return "INSTR_EXTERN_OBJ";
case INSTR_EXTERN_FUNC: return "INSTR_EXTERN_FUNC";
case INSTR_HASH_FUNC: return "INSTR_HASH_FUNC";
+ case INSTR_RSS: return "INSTR_RSS";
case INSTR_JMP: return "INSTR_JMP";
case INSTR_JMP_VALID: return "INSTR_JMP_VALID";
@@ -12029,6 +12309,34 @@ instr_hash_export(struct instruction *instr, FILE *f)
instr->hash_func.src.n_bytes);
}
+static void
+instr_rss_export(struct instruction *instr, FILE *f)
+{
+ fprintf(f,
+ "\t{\n"
+ "\t\t.type = %s,\n"
+ "\t\t.rss = {\n"
+ "\t\t\t.rss_obj_id = %u,\n"
+ "\t\t\t.dst = {\n"
+ "\t\t\t\t.offset = %u,\n"
+ "\t\t\t\t.n_bits = %u,\n"
+ "\t\t\t},\n"
+ "\t\t\t.src = {\n"
+ "\t\t\t\t.struct_id = %u,\n"
+ "\t\t\t\t.offset = %u,\n"
+ "\t\t\t\t.n_bytes = %u,\n"
+ "\t\t\t},\n"
+ "\t\t},\n"
+ "\t},\n",
+ instr_type_to_name(instr),
+ instr->rss.rss_obj_id,
+ instr->rss.dst.offset,
+ instr->rss.dst.n_bits,
+ instr->rss.src.struct_id,
+ instr->rss.src.offset,
+ instr->rss.src.n_bytes);
+}
+
static void
instr_reg_export(struct instruction *instr __rte_unused, FILE *f __rte_unused)
{
@@ -12590,6 +12898,7 @@ static instruction_export_t export_table[] = {
[INSTR_EXTERN_OBJ] = instr_extern_export,
[INSTR_EXTERN_FUNC] = instr_extern_export,
[INSTR_HASH_FUNC] = instr_hash_export,
+ [INSTR_RSS] = instr_rss_export,
[INSTR_JMP] = instr_jmp_export,
[INSTR_JMP_VALID] = instr_jmp_export,
@@ -12817,6 +13126,7 @@ instr_type_to_func(struct instruction *instr)
case INSTR_EXTERN_OBJ: return NULL;
case INSTR_EXTERN_FUNC: return NULL;
case INSTR_HASH_FUNC: return "__instr_hash_func_exec";
+ case INSTR_RSS: return "__instr_rss_exec";
case INSTR_JMP: return NULL;
case INSTR_JMP_VALID: return NULL;
@@ -352,6 +352,28 @@ rte_swx_pipeline_hash_func_register(struct rte_swx_pipeline *p,
const char *name,
rte_swx_hash_func_t func);
+/*
+ * RSS.
+ */
+
+/**
+ * Pipeline Receive Side Scaling (RSS) object configure
+ *
+ * @param[in] p
+ * Pipeline handle.
+ * @param[in] name
+ * Name for the new RSS object.
+ * @return
+ * 0 on success or the following error codes otherwise:
+ * -EINVAL: Invalid argument;
+ * -ENOMEM: Not enough space/cannot allocate memory;
+ * -EEXIST: RSS object with this name already exists.
+ */
+__rte_experimental
+int
+rte_swx_pipeline_rss_config(struct rte_swx_pipeline *p,
+ const char *name);
+
/*
* Packet headers and meta-data
*/
@@ -199,6 +199,22 @@ struct hash_func_runtime {
rte_swx_hash_func_t func;
};
+/*
+ * RSS.
+ */
+struct rss {
+ TAILQ_ENTRY(rss) node;
+ char name[RTE_SWX_NAME_SIZE];
+ uint32_t id;
+};
+
+TAILQ_HEAD(rss_tailq, rss);
+
+struct rss_runtime {
+ uint32_t key_size; /* key size in 32-bit increments. */
+ uint32_t key[0]; /* key as array of 32-bit increments, each in NBO format. */
+};
+
/*
* Header.
*/
@@ -524,6 +540,15 @@ enum instruction_type {
*/
INSTR_HASH_FUNC,
+ /* rss RSS_OBJ_NAME dst src_first src_last
+ * Compute the RSS hash value over range of struct fields.
+ * dst = M
+ * src_first = HMEFT
+ * src_last = HMEFT
+ * src_first and src_last must be fields within the same struct
+ */
+ INSTR_RSS,
+
/* jmp LABEL
* Unconditional jump
*/
@@ -677,6 +702,21 @@ struct instr_hash_func {
} src;
};
+struct instr_rss {
+ uint8_t rss_obj_id;
+
+ struct {
+ uint8_t offset;
+ uint8_t n_bits;
+ } dst;
+
+ struct {
+ uint8_t struct_id;
+ uint16_t offset;
+ uint16_t n_bytes;
+ } src;
+};
+
struct instr_dst_src {
struct instr_operand dst;
union {
@@ -763,6 +803,7 @@ struct instruction {
struct instr_extern_obj ext_obj;
struct instr_extern_func ext_func;
struct instr_hash_func hash_func;
+ struct instr_rss rss;
struct instr_jmp jmp;
};
};
@@ -1480,6 +1521,7 @@ struct rte_swx_pipeline {
struct extern_obj_tailq extern_objs;
struct extern_func_tailq extern_funcs;
struct hash_func_tailq hash_funcs;
+ struct rss_tailq rss;
struct header_tailq headers;
struct struct_type *metadata_st;
uint32_t metadata_struct_id;
@@ -1502,6 +1544,7 @@ struct rte_swx_pipeline {
struct selector_statistics *selector_stats;
struct learner_statistics *learner_stats;
struct hash_func_runtime *hash_func_runtime;
+ struct rss_runtime **rss_runtime;
struct regarray_runtime *regarray_runtime;
struct metarray_runtime *metarray_runtime;
struct instruction *instructions;
@@ -1518,6 +1561,7 @@ struct rte_swx_pipeline {
uint32_t n_extern_objs;
uint32_t n_extern_funcs;
uint32_t n_hash_funcs;
+ uint32_t n_rss;
uint32_t n_actions;
uint32_t n_tables;
uint32_t n_selectors;
@@ -2467,6 +2511,54 @@ __instr_hash_func_exec(struct rte_swx_pipeline *p,
METADATA_WRITE(t, dst_offset, n_dst_bits, result);
}
+/*
+ * rss.
+ */
+static inline uint32_t
+rss_func(uint32_t *key, uint32_t key_size, uint32_t *data, uint32_t data_size)
+{
+ uint32_t hash_val = 0, key_mask = key_size - 1, i;
+
+ for (i = 0; i < data_size; i++) {
+ uint32_t d;
+
+ for (d = data[i]; d; d &= (d - 1)) {
+ uint32_t key0, key1, pos;
+
+ pos = rte_bsf32(d);
+ key0 = key[i & key_mask] << (31 - pos);
+ key1 = key[(i + 1) & key_mask] >> (pos + 1);
+ hash_val ^= key0 | key1;
+ }
+ }
+
+ return hash_val;
+}
+
+static inline void
+__instr_rss_exec(struct rte_swx_pipeline *p,
+ struct thread *t,
+ const struct instruction *ip)
+{
+ uint32_t rss_obj_id = ip->rss.rss_obj_id;
+ uint32_t dst_offset = ip->rss.dst.offset;
+ uint32_t n_dst_bits = ip->rss.dst.n_bits;
+ uint32_t src_struct_id = ip->rss.src.struct_id;
+ uint32_t src_offset = ip->rss.src.offset;
+ uint32_t n_src_bytes = ip->rss.src.n_bytes;
+
+ struct rss_runtime *r = p->rss_runtime[rss_obj_id];
+ uint8_t *src_ptr = t->structs[src_struct_id];
+ uint32_t result;
+
+ TRACE("[Thread %2u] rss %u\n",
+ p->thread_id,
+ rss_obj_id);
+
+ result = rss_func(r->key, r->key_size, (uint32_t *)&src_ptr[src_offset], n_src_bytes >> 2);
+ METADATA_WRITE(t, dst_offset, n_dst_bits, result);
+}
+
/*
* mov.
*/
@@ -157,6 +157,10 @@ EXPERIMENTAL {
rte_swx_pipeline_find;
#added in 23.03
+ rte_swx_ctl_rss_info_get;
+ rte_swx_ctl_pipeline_rss_key_size_read;
+ rte_swx_ctl_pipeline_rss_key_read;
+ rte_swx_ctl_pipeline_rss_key_write;
rte_swx_ipsec_create;
rte_swx_ipsec_find;
rte_swx_ipsec_free;
@@ -164,4 +168,5 @@ EXPERIMENTAL {
rte_swx_ipsec_sa_add;
rte_swx_ipsec_sa_delete;
rte_swx_ipsec_sa_read;
+ rte_swx_pipeline_rss_config;
};