[v6,08/10] ipsec: add support for SA telemetry
Checks
Commit Message
Add telemetry support for ipsec SAs
Signed-off-by: Declan Doherty <declan.doherty@intel.com>
Signed-off-by: Radu Nicolau <radu.nicolau@intel.com>
Signed-off-by: Abhijit Sinha <abhijit.sinha@intel.com>
Signed-off-by: Daniel Martin Buckley <daniel.m.buckley@intel.com>
Acked-by: Fan Zhang <roy.fan.zhang@intel.com>
---
lib/ipsec/esp_inb.c | 1 +
lib/ipsec/esp_outb.c | 12 +-
lib/ipsec/meson.build | 2 +-
lib/ipsec/rte_ipsec.h | 23 ++++
lib/ipsec/sa.c | 255 +++++++++++++++++++++++++++++++++++++++++-
lib/ipsec/sa.h | 21 ++++
lib/ipsec/version.map | 9 ++
7 files changed, 317 insertions(+), 6 deletions(-)
Comments
> Signed-off-by: Declan Doherty <declan.doherty@intel.com>
> Signed-off-by: Radu Nicolau <radu.nicolau@intel.com>
> Signed-off-by: Abhijit Sinha <abhijit.sinha@intel.com>
> Signed-off-by: Daniel Martin Buckley <daniel.m.buckley@intel.com>
> Acked-by: Fan Zhang <roy.fan.zhang@intel.com>
> ---
> lib/ipsec/esp_inb.c | 1 +
> lib/ipsec/esp_outb.c | 12 +-
> lib/ipsec/meson.build | 2 +-
> lib/ipsec/rte_ipsec.h | 23 ++++
> lib/ipsec/sa.c | 255 +++++++++++++++++++++++++++++++++++++++++-
> lib/ipsec/sa.h | 21 ++++
> lib/ipsec/version.map | 9 ++
> 7 files changed, 317 insertions(+), 6 deletions(-)
>
> diff --git a/lib/ipsec/esp_inb.c b/lib/ipsec/esp_inb.c
> index a6ab8fbdd5..8cb4c16302 100644
> --- a/lib/ipsec/esp_inb.c
> +++ b/lib/ipsec/esp_inb.c
> @@ -722,6 +722,7 @@ esp_inb_pkt_process(struct rte_ipsec_sa *sa, struct rte_mbuf *mb[],
>
> /* process packets, extract seq numbers */
> k = process(sa, mb, sqn, dr, num, sqh_len);
> + sa->statistics.count += k;
>
> /* handle unprocessed mbufs */
> if (k != num && k != 0)
> diff --git a/lib/ipsec/esp_outb.c b/lib/ipsec/esp_outb.c
> index 9fc7075796..2c02c3bb12 100644
> --- a/lib/ipsec/esp_outb.c
> +++ b/lib/ipsec/esp_outb.c
> @@ -617,7 +617,7 @@ uint16_t
> esp_outb_sqh_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
> uint16_t num)
> {
> - uint32_t i, k, icv_len, *icv;
> + uint32_t i, k, icv_len, *icv, bytes;
> struct rte_mbuf *ml;
> struct rte_ipsec_sa *sa;
> uint32_t dr[num];
> @@ -626,10 +626,12 @@ esp_outb_sqh_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
>
> k = 0;
> icv_len = sa->icv_len;
> + bytes = 0;
>
> for (i = 0; i != num; i++) {
> if ((mb[i]->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED) == 0) {
> ml = rte_pktmbuf_lastseg(mb[i]);
> + bytes += mb[i]->data_len;
Shouldn't it be pkt_len?
> /* remove high-order 32 bits of esn from packet len */
> mb[i]->pkt_len -= sa->sqh_len;
> ml->data_len -= sa->sqh_len;
> @@ -640,6 +642,8 @@ esp_outb_sqh_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
> } else
> dr[i - k] = i;
> }
> + sa->statistics.count += k;
> + sa->statistics.bytes += bytes - (sa->hdr_len * k);
I don't think you need to do multiplication here.
It can be postponed for reporting phase (sa->hdr_len is a constant value per sa).
>
> /* handle unprocessed mbufs */
> if (k != num) {
> @@ -659,16 +663,19 @@ static inline void
> inline_outb_mbuf_prepare(const struct rte_ipsec_session *ss,
> struct rte_mbuf *mb[], uint16_t num)
> {
> - uint32_t i, ol_flags;
> + uint32_t i, ol_flags, bytes = 0;
Lets keep coding style consistent: please do assignment as separate statement.
>
> ol_flags = ss->security.ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA;
> for (i = 0; i != num; i++) {
>
> mb[i]->ol_flags |= PKT_TX_SEC_OFFLOAD;
> + bytes += mb[i]->data_len;
pkt_len?
> if (ol_flags != 0)
> rte_security_set_pkt_metadata(ss->security.ctx,
> ss->security.ses, mb[i], NULL);
> }
> + ss->sa->statistics.count += num;
> + ss->sa->statistics.bytes += bytes - (ss->sa->hdr_len * num);
> }
>
> /* check if packet will exceed MSS and segmentation is required */
> @@ -752,6 +759,7 @@ inline_outb_tun_pkt_process(const struct rte_ipsec_session *ss,
> sqn += nb_segs[i] - 1;
> }
>
> +
Empty line.
> /* copy not processed mbufs beyond good ones */
> if (k != num && k != 0)
> move_bad_mbufs(mb, dr, num, num - k);
> diff --git a/lib/ipsec/meson.build b/lib/ipsec/meson.build
> index 1497f573bb..f5e44cfe47 100644
> --- a/lib/ipsec/meson.build
> +++ b/lib/ipsec/meson.build
> @@ -6,4 +6,4 @@ sources = files('esp_inb.c', 'esp_outb.c', 'sa.c', 'ses.c', 'ipsec_sad.c')
> headers = files('rte_ipsec.h', 'rte_ipsec_sa.h', 'rte_ipsec_sad.h')
> indirect_headers += files('rte_ipsec_group.h')
>
> -deps += ['mbuf', 'net', 'cryptodev', 'security', 'hash']
> +deps += ['mbuf', 'net', 'cryptodev', 'security', 'hash', 'telemetry']
> diff --git a/lib/ipsec/rte_ipsec.h b/lib/ipsec/rte_ipsec.h
> index dd60d95915..2bb52f4b8f 100644
> --- a/lib/ipsec/rte_ipsec.h
> +++ b/lib/ipsec/rte_ipsec.h
> @@ -158,6 +158,29 @@ rte_ipsec_pkt_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
> return ss->pkt_func.process(ss, mb, num);
> }
>
> +
> +struct rte_ipsec_telemetry;
> +
> +/**
> + * Initialize IPsec library telemetry.
> + * @return
> + * 0 on success, negative value otherwise.
> + */
> +__rte_experimental
> +int
> +rte_ipsec_telemetry_init(void);
> +
> +/**
> + * Enable per SA telemetry for a specific SA.
> + * @param sa
> + * Pointer to the *rte_ipsec_sa* object that will have telemetry enabled.
> + * @return
> + * 0 on success, negative value otherwise.
> + */
> +__rte_experimental
> +int
> +rte_ipsec_telemetry_sa_add(struct rte_ipsec_sa *sa);
> +
Why we don't have sa_delete() here?
What user supposed to do when he destroys an sa?
Another question what concurrency model is implied here?
> #include <rte_ipsec_group.h>
>
> #ifdef __cplusplus
> diff --git a/lib/ipsec/sa.c b/lib/ipsec/sa.c
> index 8e369e4618..5b55bbc098 100644
> --- a/lib/ipsec/sa.c
> +++ b/lib/ipsec/sa.c
> @@ -7,7 +7,7 @@
> #include <rte_ip.h>
> #include <rte_errno.h>
> #include <rte_cryptodev.h>
> -
> +#include <rte_telemetry.h>
As a generic one - can we move all telemetry related functions into new .c file
(sa_telemtry or so)? No point to have it here.
> #include "sa.h"
> #include "ipsec_sqn.h"
> #include "crypto.h"
> @@ -25,6 +25,7 @@ struct crypto_xform {
> struct rte_crypto_aead_xform *aead;
> };
>
> +
> /*
> * helper routine, fills internal crypto_xform structure.
> */
> @@ -532,6 +533,249 @@ rte_ipsec_sa_size(const struct rte_ipsec_sa_prm *prm)
> wsz = prm->ipsec_xform.replay_win_sz;
> return ipsec_sa_size(type, &wsz, &nb);
> }
> +struct rte_ipsec_telemetry {
> + bool initialized;
Why 'initilized' is needed at all?
I think there is a static initializer for list: LIST_HEAD_INITIALIZER
> + LIST_HEAD(, rte_ipsec_sa) sa_list_head;
> +};
> +
> +#include <rte_malloc.h>
> +
> +static struct rte_ipsec_telemetry rte_ipsec_telemetry_instance = {
> + .initialized = false };
> +
> +static int
> +handle_telemetry_cmd_ipsec_sa_list(const char *cmd __rte_unused,
> + const char *params __rte_unused,
> + struct rte_tel_data *data)
> +{
> + struct rte_ipsec_telemetry *telemetry = &rte_ipsec_telemetry_instance;
> + struct rte_ipsec_sa *sa;
> +
> + rte_tel_data_start_array(data, RTE_TEL_U64_VAL);
> +
> + LIST_FOREACH(sa, &telemetry->sa_list_head, telemetry_next) {
> + rte_tel_data_add_array_u64(data, htonl(sa->spi));
Should be ntohl() I believe.
BTW, why not use rte_be_to_cpu... functions here?
> + }
> +
> + return 0;
> +}
> +
> +/**
> + * Handle IPsec SA statistics telemetry request
> + *
> + * Return dict of SA's with dict of key/value counters
> + *
> + * {
> + * "SA_SPI_XX": {"count": 0, "bytes": 0, "errors": 0},
> + * "SA_SPI_YY": {"count": 0, "bytes": 0, "errors": 0}
> + * }
> + *
> + */
> +static int
> +handle_telemetry_cmd_ipsec_sa_stats(const char *cmd __rte_unused,
> + const char *params,
> + struct rte_tel_data *data)
> +{
> + struct rte_ipsec_telemetry *telemetry = &rte_ipsec_telemetry_instance;
> + struct rte_ipsec_sa *sa;
> + bool user_specified_spi = false;
> + uint32_t sa_spi;
> +
> + if (params) {
> + user_specified_spi = true;
> + sa_spi = htonl((uint32_t)atoi(params));
strtoul() would be a better choice here.
Another nit - you probably don't need user_specified_spi.
As I remember SPI=0 is a reserved value, so I think It would be enough to:
sa_spi=0; if (params) {sa_spi=..}
> + }
> +
> + rte_tel_data_start_dict(data);
> +
> + LIST_FOREACH(sa, &telemetry->sa_list_head, telemetry_next) {
> + char sa_name[64];
> +
> + static const char *name_pkt_cnt = "count";
> + static const char *name_byte_cnt = "bytes";
> + static const char *name_error_cnt = "errors";
> + struct rte_tel_data *sa_data;
> +
> + /* If user provided SPI only get telemetry for that SA */
> + if (user_specified_spi && (sa_spi != sa->spi))
> + continue;
> +
> + /* allocate telemetry data struct for SA telemetry */
> + sa_data = rte_tel_data_alloc();
> + if (!sa_data)
> + return -ENOMEM;
> +
> + rte_tel_data_start_dict(sa_data);
> +
> + /* add telemetry key/values pairs */
> + rte_tel_data_add_dict_u64(sa_data, name_pkt_cnt,
> + sa->statistics.count);
> +
> + rte_tel_data_add_dict_u64(sa_data, name_byte_cnt,
> + sa->statistics.bytes);
> +
> + rte_tel_data_add_dict_u64(sa_data, name_error_cnt,
> + sa->statistics.errors.count);
> +
> + /* generate telemetry label */
> + snprintf(sa_name, sizeof(sa_name), "SA_SPI_%i", htonl(sa->spi));
Again - ntohl().
> +
> + /* add SA telemetry to dictionary container */
> + rte_tel_data_add_dict_container(data, sa_name, sa_data, 0);
> + }
> +
> + return 0;
> +}
> +
> +static int
> +handle_telemetry_cmd_ipsec_sa_configuration(const char *cmd __rte_unused,
> + const char *params,
> + struct rte_tel_data *data)
> +{
> + struct rte_ipsec_telemetry *telemetry = &rte_ipsec_telemetry_instance;
> + struct rte_ipsec_sa *sa;
> + uint32_t sa_spi;
> +
> + if (params)
> + sa_spi = htonl((uint32_t)atoi(params));
> + else
> + return -EINVAL;
> +
> + rte_tel_data_start_dict(data);
> +
> + LIST_FOREACH(sa, &telemetry->sa_list_head, telemetry_next) {
> + uint64_t mode;
> +
> + if (sa_spi != sa->spi)
> + continue;
> +
> + /* add SA configuration key/values pairs */
> + rte_tel_data_add_dict_string(data, "Type",
> + (sa->type & RTE_IPSEC_SATP_PROTO_MASK) ==
> + RTE_IPSEC_SATP_PROTO_AH ? "AH" : "ESP");
> +
> + rte_tel_data_add_dict_string(data, "Direction",
> + (sa->type & RTE_IPSEC_SATP_DIR_MASK) ==
> + RTE_IPSEC_SATP_DIR_IB ? "Inbound" : "Outbound");
> +
> + mode = sa->type & RTE_IPSEC_SATP_MODE_MASK;
> +
> + if (mode == RTE_IPSEC_SATP_MODE_TRANS) {
> + rte_tel_data_add_dict_string(data, "Mode", "Transport");
> + } else {
> + rte_tel_data_add_dict_string(data, "Mode", "Tunnel");
> +
> + if ((sa->type & RTE_IPSEC_SATP_NATT_MASK) ==
> + RTE_IPSEC_SATP_NATT_ENABLE) {
> + if (sa->type & RTE_IPSEC_SATP_MODE_TUNLV4) {
> + rte_tel_data_add_dict_string(data,
> + "Tunnel-Type",
> + "IPv4-UDP");
> + } else if (sa->type &
> + RTE_IPSEC_SATP_MODE_TUNLV6) {
> + rte_tel_data_add_dict_string(data,
> + "Tunnel-Type",
> + "IPv4-UDP");
> + }
> + } else {
> + if (sa->type & RTE_IPSEC_SATP_MODE_TUNLV4) {
> + rte_tel_data_add_dict_string(data,
> + "Tunnel-Type",
> + "IPv4-UDP");
> + } else if (sa->type &
> + RTE_IPSEC_SATP_MODE_TUNLV6) {
> + rte_tel_data_add_dict_string(data,
> + "Tunnel-Type",
> + "IPv4-UDP");
> + }
> + }
> + }
> +
> + rte_tel_data_add_dict_string(data,
> + "extended-sequence-number",
> + (sa->type & RTE_IPSEC_SATP_ESN_MASK) ==
> + RTE_IPSEC_SATP_ESN_ENABLE ?
> + "enabled" : "disabled");
> +
> + if ((sa->type & RTE_IPSEC_SATP_DIR_MASK) ==
> + RTE_IPSEC_SATP_DIR_IB)
> +
> + if (sa->sqn.inb.rsn[sa->sqn.inb.rdidx])
> + rte_tel_data_add_dict_u64(data,
> + "sequence-number",
> + sa->sqn.inb.rsn[sa->sqn.inb.rdidx]->sqn);
> + else
> + rte_tel_data_add_dict_u64(data,
> + "sequence-number", 0);
> + else
> + rte_tel_data_add_dict_u64(data, "sequence-number",
> + sa->sqn.outb);
> +
> + rte_tel_data_add_dict_string(data,
> + "explicit-congestion-notification",
> + (sa->type & RTE_IPSEC_SATP_ECN_MASK) ==
> + RTE_IPSEC_SATP_ECN_ENABLE ?
> + "enabled" : "disabled");
> +
> + rte_tel_data_add_dict_string(data,
> + "copy-DSCP",
> + (sa->type & RTE_IPSEC_SATP_DSCP_MASK) ==
> + RTE_IPSEC_SATP_DSCP_ENABLE ?
> + "enabled" : "disabled");
> +
> + rte_tel_data_add_dict_string(data, "TSO",
> + sa->tso.enabled ? "enabled" : "disabled");
> +
> + if (sa->tso.enabled)
> + rte_tel_data_add_dict_u64(data, "TSO-MSS", sa->tso.mss);
> +
> + }
> +
> + return 0;
> +}
> +int
> +rte_ipsec_telemetry_init(void)
> +{
> + struct rte_ipsec_telemetry *telemetry = &rte_ipsec_telemetry_instance;
> + int rc = 0;
> +
> + if (telemetry->initialized)
> + return rc;
> +
> + LIST_INIT(&telemetry->sa_list_head);
> +
> + rc = rte_telemetry_register_cmd("/ipsec/sa/list",
> + handle_telemetry_cmd_ipsec_sa_list,
> + "Return list of IPsec Security Associations with telemetry enabled.");
> + if (rc)
> + return rc;
> +
> + rc = rte_telemetry_register_cmd("/ipsec/sa/stats",
> + handle_telemetry_cmd_ipsec_sa_stats,
> + "Returns IPsec Security Association stastistics. Parameters: int sa_spi");
> + if (rc)
> + return rc;
> +
> + rc = rte_telemetry_register_cmd("/ipsec/sa/details",
> + handle_telemetry_cmd_ipsec_sa_configuration,
> + "Returns IPsec Security Association configuration. Parameters: int sa_spi");
> + if (rc)
> + return rc;
> +
> + telemetry->initialized = true;
> +
> + return rc;
> +}
> +
> +int
> +rte_ipsec_telemetry_sa_add(struct rte_ipsec_sa *sa)
> +{
> + struct rte_ipsec_telemetry *telemetry = &rte_ipsec_telemetry_instance;
> +
> + LIST_INSERT_HEAD(&telemetry->sa_list_head, sa, telemetry_next);
> +
> + return 0;
> +}
>
> int
> rte_ipsec_sa_init(struct rte_ipsec_sa *sa, const struct rte_ipsec_sa_prm *prm,
> @@ -644,19 +888,24 @@ uint16_t
> pkt_flag_process(const struct rte_ipsec_session *ss,
> struct rte_mbuf *mb[], uint16_t num)
> {
> - uint32_t i, k;
> + uint32_t i, k, bytes = 0;
> uint32_t dr[num];
>
> RTE_SET_USED(ss);
>
> k = 0;
> for (i = 0; i != num; i++) {
> - if ((mb[i]->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED) == 0)
> + if ((mb[i]->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED) == 0) {
> k++;
> + bytes += mb[i]->data_len;
> + }
> else
> dr[i - k] = i;
> }
>
> + ss->sa->statistics.count += k;
> + ss->sa->statistics.bytes += bytes - (ss->sa->hdr_len * k);
> +
> /* handle unprocessed mbufs */
> if (k != num) {
> rte_errno = EBADMSG;
> diff --git a/lib/ipsec/sa.h b/lib/ipsec/sa.h
> index 3f38921eb3..b9b7ebec5b 100644
> --- a/lib/ipsec/sa.h
> +++ b/lib/ipsec/sa.h
> @@ -122,9 +122,30 @@ struct rte_ipsec_sa {
> uint16_t mss;
> } tso;
>
> + LIST_ENTRY(rte_ipsec_sa) telemetry_next;
> + /**< list entry for telemetry enabled SA */
I am not really fond of idea to have telemetry list stuff embedded into rte_ipsec_sa structure.
Creates all sort of concurrency problem for adding/removing SA, while reading telemetry data, etc.
Another issue if SA is shared my multiple-processes.
Instead would be much cleaner if telemetry list will contain just a pointer to SA.
Then it would be user responsibility to add del/add sa to the telelmetry list in an appropriate time.
Also MT working model for this new API needs to be documented properly.
> +
> +
> + RTE_MARKER cachealign_statistics __rte_cache_min_aligned;
What is the reason for all these extra alignments?
> +
> + /* Statistics */
> + struct {
> + uint64_t count;
> + uint64_t bytes;
> +
> + struct {
> + uint64_t count;
> + uint64_t authentication_failed;
> + } errors;
> + } statistics;
> +
> + RTE_MARKER cachealign_tunnel_header __rte_cache_min_aligned;
> +
> /* template for tunnel header */
> uint8_t hdr[IPSEC_MAX_HDR_SIZE];
>
> +
> + RTE_MARKER cachealign_tunnel_seq_num_replay_win __rte_cache_min_aligned;
> /*
> * sqn and replay window
> * In case of SA handled by multiple threads *sqn* cacheline
> diff --git a/lib/ipsec/version.map b/lib/ipsec/version.map
> index ba8753eac4..fed6b6aba1 100644
> --- a/lib/ipsec/version.map
> +++ b/lib/ipsec/version.map
> @@ -19,3 +19,12 @@ DPDK_22 {
>
> local: *;
> };
> +
> +EXPERIMENTAL {
> + global:
> +
> + # added in 21.11
> + rte_ipsec_telemetry_init;
> + rte_ipsec_telemetry_sa_add;
> +
> +};
> --
> 2.25.1
@@ -722,6 +722,7 @@ esp_inb_pkt_process(struct rte_ipsec_sa *sa, struct rte_mbuf *mb[],
/* process packets, extract seq numbers */
k = process(sa, mb, sqn, dr, num, sqh_len);
+ sa->statistics.count += k;
/* handle unprocessed mbufs */
if (k != num && k != 0)
@@ -617,7 +617,7 @@ uint16_t
esp_outb_sqh_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
uint16_t num)
{
- uint32_t i, k, icv_len, *icv;
+ uint32_t i, k, icv_len, *icv, bytes;
struct rte_mbuf *ml;
struct rte_ipsec_sa *sa;
uint32_t dr[num];
@@ -626,10 +626,12 @@ esp_outb_sqh_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
k = 0;
icv_len = sa->icv_len;
+ bytes = 0;
for (i = 0; i != num; i++) {
if ((mb[i]->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED) == 0) {
ml = rte_pktmbuf_lastseg(mb[i]);
+ bytes += mb[i]->data_len;
/* remove high-order 32 bits of esn from packet len */
mb[i]->pkt_len -= sa->sqh_len;
ml->data_len -= sa->sqh_len;
@@ -640,6 +642,8 @@ esp_outb_sqh_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
} else
dr[i - k] = i;
}
+ sa->statistics.count += k;
+ sa->statistics.bytes += bytes - (sa->hdr_len * k);
/* handle unprocessed mbufs */
if (k != num) {
@@ -659,16 +663,19 @@ static inline void
inline_outb_mbuf_prepare(const struct rte_ipsec_session *ss,
struct rte_mbuf *mb[], uint16_t num)
{
- uint32_t i, ol_flags;
+ uint32_t i, ol_flags, bytes = 0;
ol_flags = ss->security.ol_flags & RTE_SECURITY_TX_OLOAD_NEED_MDATA;
for (i = 0; i != num; i++) {
mb[i]->ol_flags |= PKT_TX_SEC_OFFLOAD;
+ bytes += mb[i]->data_len;
if (ol_flags != 0)
rte_security_set_pkt_metadata(ss->security.ctx,
ss->security.ses, mb[i], NULL);
}
+ ss->sa->statistics.count += num;
+ ss->sa->statistics.bytes += bytes - (ss->sa->hdr_len * num);
}
/* check if packet will exceed MSS and segmentation is required */
@@ -752,6 +759,7 @@ inline_outb_tun_pkt_process(const struct rte_ipsec_session *ss,
sqn += nb_segs[i] - 1;
}
+
/* copy not processed mbufs beyond good ones */
if (k != num && k != 0)
move_bad_mbufs(mb, dr, num, num - k);
@@ -6,4 +6,4 @@ sources = files('esp_inb.c', 'esp_outb.c', 'sa.c', 'ses.c', 'ipsec_sad.c')
headers = files('rte_ipsec.h', 'rte_ipsec_sa.h', 'rte_ipsec_sad.h')
indirect_headers += files('rte_ipsec_group.h')
-deps += ['mbuf', 'net', 'cryptodev', 'security', 'hash']
+deps += ['mbuf', 'net', 'cryptodev', 'security', 'hash', 'telemetry']
@@ -158,6 +158,29 @@ rte_ipsec_pkt_process(const struct rte_ipsec_session *ss, struct rte_mbuf *mb[],
return ss->pkt_func.process(ss, mb, num);
}
+
+struct rte_ipsec_telemetry;
+
+/**
+ * Initialize IPsec library telemetry.
+ * @return
+ * 0 on success, negative value otherwise.
+ */
+__rte_experimental
+int
+rte_ipsec_telemetry_init(void);
+
+/**
+ * Enable per SA telemetry for a specific SA.
+ * @param sa
+ * Pointer to the *rte_ipsec_sa* object that will have telemetry enabled.
+ * @return
+ * 0 on success, negative value otherwise.
+ */
+__rte_experimental
+int
+rte_ipsec_telemetry_sa_add(struct rte_ipsec_sa *sa);
+
#include <rte_ipsec_group.h>
#ifdef __cplusplus
@@ -7,7 +7,7 @@
#include <rte_ip.h>
#include <rte_errno.h>
#include <rte_cryptodev.h>
-
+#include <rte_telemetry.h>
#include "sa.h"
#include "ipsec_sqn.h"
#include "crypto.h"
@@ -25,6 +25,7 @@ struct crypto_xform {
struct rte_crypto_aead_xform *aead;
};
+
/*
* helper routine, fills internal crypto_xform structure.
*/
@@ -532,6 +533,249 @@ rte_ipsec_sa_size(const struct rte_ipsec_sa_prm *prm)
wsz = prm->ipsec_xform.replay_win_sz;
return ipsec_sa_size(type, &wsz, &nb);
}
+struct rte_ipsec_telemetry {
+ bool initialized;
+ LIST_HEAD(, rte_ipsec_sa) sa_list_head;
+};
+
+#include <rte_malloc.h>
+
+static struct rte_ipsec_telemetry rte_ipsec_telemetry_instance = {
+ .initialized = false };
+
+static int
+handle_telemetry_cmd_ipsec_sa_list(const char *cmd __rte_unused,
+ const char *params __rte_unused,
+ struct rte_tel_data *data)
+{
+ struct rte_ipsec_telemetry *telemetry = &rte_ipsec_telemetry_instance;
+ struct rte_ipsec_sa *sa;
+
+ rte_tel_data_start_array(data, RTE_TEL_U64_VAL);
+
+ LIST_FOREACH(sa, &telemetry->sa_list_head, telemetry_next) {
+ rte_tel_data_add_array_u64(data, htonl(sa->spi));
+ }
+
+ return 0;
+}
+
+/**
+ * Handle IPsec SA statistics telemetry request
+ *
+ * Return dict of SA's with dict of key/value counters
+ *
+ * {
+ * "SA_SPI_XX": {"count": 0, "bytes": 0, "errors": 0},
+ * "SA_SPI_YY": {"count": 0, "bytes": 0, "errors": 0}
+ * }
+ *
+ */
+static int
+handle_telemetry_cmd_ipsec_sa_stats(const char *cmd __rte_unused,
+ const char *params,
+ struct rte_tel_data *data)
+{
+ struct rte_ipsec_telemetry *telemetry = &rte_ipsec_telemetry_instance;
+ struct rte_ipsec_sa *sa;
+ bool user_specified_spi = false;
+ uint32_t sa_spi;
+
+ if (params) {
+ user_specified_spi = true;
+ sa_spi = htonl((uint32_t)atoi(params));
+ }
+
+ rte_tel_data_start_dict(data);
+
+ LIST_FOREACH(sa, &telemetry->sa_list_head, telemetry_next) {
+ char sa_name[64];
+
+ static const char *name_pkt_cnt = "count";
+ static const char *name_byte_cnt = "bytes";
+ static const char *name_error_cnt = "errors";
+ struct rte_tel_data *sa_data;
+
+ /* If user provided SPI only get telemetry for that SA */
+ if (user_specified_spi && (sa_spi != sa->spi))
+ continue;
+
+ /* allocate telemetry data struct for SA telemetry */
+ sa_data = rte_tel_data_alloc();
+ if (!sa_data)
+ return -ENOMEM;
+
+ rte_tel_data_start_dict(sa_data);
+
+ /* add telemetry key/values pairs */
+ rte_tel_data_add_dict_u64(sa_data, name_pkt_cnt,
+ sa->statistics.count);
+
+ rte_tel_data_add_dict_u64(sa_data, name_byte_cnt,
+ sa->statistics.bytes);
+
+ rte_tel_data_add_dict_u64(sa_data, name_error_cnt,
+ sa->statistics.errors.count);
+
+ /* generate telemetry label */
+ snprintf(sa_name, sizeof(sa_name), "SA_SPI_%i", htonl(sa->spi));
+
+ /* add SA telemetry to dictionary container */
+ rte_tel_data_add_dict_container(data, sa_name, sa_data, 0);
+ }
+
+ return 0;
+}
+
+static int
+handle_telemetry_cmd_ipsec_sa_configuration(const char *cmd __rte_unused,
+ const char *params,
+ struct rte_tel_data *data)
+{
+ struct rte_ipsec_telemetry *telemetry = &rte_ipsec_telemetry_instance;
+ struct rte_ipsec_sa *sa;
+ uint32_t sa_spi;
+
+ if (params)
+ sa_spi = htonl((uint32_t)atoi(params));
+ else
+ return -EINVAL;
+
+ rte_tel_data_start_dict(data);
+
+ LIST_FOREACH(sa, &telemetry->sa_list_head, telemetry_next) {
+ uint64_t mode;
+
+ if (sa_spi != sa->spi)
+ continue;
+
+ /* add SA configuration key/values pairs */
+ rte_tel_data_add_dict_string(data, "Type",
+ (sa->type & RTE_IPSEC_SATP_PROTO_MASK) ==
+ RTE_IPSEC_SATP_PROTO_AH ? "AH" : "ESP");
+
+ rte_tel_data_add_dict_string(data, "Direction",
+ (sa->type & RTE_IPSEC_SATP_DIR_MASK) ==
+ RTE_IPSEC_SATP_DIR_IB ? "Inbound" : "Outbound");
+
+ mode = sa->type & RTE_IPSEC_SATP_MODE_MASK;
+
+ if (mode == RTE_IPSEC_SATP_MODE_TRANS) {
+ rte_tel_data_add_dict_string(data, "Mode", "Transport");
+ } else {
+ rte_tel_data_add_dict_string(data, "Mode", "Tunnel");
+
+ if ((sa->type & RTE_IPSEC_SATP_NATT_MASK) ==
+ RTE_IPSEC_SATP_NATT_ENABLE) {
+ if (sa->type & RTE_IPSEC_SATP_MODE_TUNLV4) {
+ rte_tel_data_add_dict_string(data,
+ "Tunnel-Type",
+ "IPv4-UDP");
+ } else if (sa->type &
+ RTE_IPSEC_SATP_MODE_TUNLV6) {
+ rte_tel_data_add_dict_string(data,
+ "Tunnel-Type",
+ "IPv4-UDP");
+ }
+ } else {
+ if (sa->type & RTE_IPSEC_SATP_MODE_TUNLV4) {
+ rte_tel_data_add_dict_string(data,
+ "Tunnel-Type",
+ "IPv4-UDP");
+ } else if (sa->type &
+ RTE_IPSEC_SATP_MODE_TUNLV6) {
+ rte_tel_data_add_dict_string(data,
+ "Tunnel-Type",
+ "IPv4-UDP");
+ }
+ }
+ }
+
+ rte_tel_data_add_dict_string(data,
+ "extended-sequence-number",
+ (sa->type & RTE_IPSEC_SATP_ESN_MASK) ==
+ RTE_IPSEC_SATP_ESN_ENABLE ?
+ "enabled" : "disabled");
+
+ if ((sa->type & RTE_IPSEC_SATP_DIR_MASK) ==
+ RTE_IPSEC_SATP_DIR_IB)
+
+ if (sa->sqn.inb.rsn[sa->sqn.inb.rdidx])
+ rte_tel_data_add_dict_u64(data,
+ "sequence-number",
+ sa->sqn.inb.rsn[sa->sqn.inb.rdidx]->sqn);
+ else
+ rte_tel_data_add_dict_u64(data,
+ "sequence-number", 0);
+ else
+ rte_tel_data_add_dict_u64(data, "sequence-number",
+ sa->sqn.outb);
+
+ rte_tel_data_add_dict_string(data,
+ "explicit-congestion-notification",
+ (sa->type & RTE_IPSEC_SATP_ECN_MASK) ==
+ RTE_IPSEC_SATP_ECN_ENABLE ?
+ "enabled" : "disabled");
+
+ rte_tel_data_add_dict_string(data,
+ "copy-DSCP",
+ (sa->type & RTE_IPSEC_SATP_DSCP_MASK) ==
+ RTE_IPSEC_SATP_DSCP_ENABLE ?
+ "enabled" : "disabled");
+
+ rte_tel_data_add_dict_string(data, "TSO",
+ sa->tso.enabled ? "enabled" : "disabled");
+
+ if (sa->tso.enabled)
+ rte_tel_data_add_dict_u64(data, "TSO-MSS", sa->tso.mss);
+
+ }
+
+ return 0;
+}
+int
+rte_ipsec_telemetry_init(void)
+{
+ struct rte_ipsec_telemetry *telemetry = &rte_ipsec_telemetry_instance;
+ int rc = 0;
+
+ if (telemetry->initialized)
+ return rc;
+
+ LIST_INIT(&telemetry->sa_list_head);
+
+ rc = rte_telemetry_register_cmd("/ipsec/sa/list",
+ handle_telemetry_cmd_ipsec_sa_list,
+ "Return list of IPsec Security Associations with telemetry enabled.");
+ if (rc)
+ return rc;
+
+ rc = rte_telemetry_register_cmd("/ipsec/sa/stats",
+ handle_telemetry_cmd_ipsec_sa_stats,
+ "Returns IPsec Security Association stastistics. Parameters: int sa_spi");
+ if (rc)
+ return rc;
+
+ rc = rte_telemetry_register_cmd("/ipsec/sa/details",
+ handle_telemetry_cmd_ipsec_sa_configuration,
+ "Returns IPsec Security Association configuration. Parameters: int sa_spi");
+ if (rc)
+ return rc;
+
+ telemetry->initialized = true;
+
+ return rc;
+}
+
+int
+rte_ipsec_telemetry_sa_add(struct rte_ipsec_sa *sa)
+{
+ struct rte_ipsec_telemetry *telemetry = &rte_ipsec_telemetry_instance;
+
+ LIST_INSERT_HEAD(&telemetry->sa_list_head, sa, telemetry_next);
+
+ return 0;
+}
int
rte_ipsec_sa_init(struct rte_ipsec_sa *sa, const struct rte_ipsec_sa_prm *prm,
@@ -644,19 +888,24 @@ uint16_t
pkt_flag_process(const struct rte_ipsec_session *ss,
struct rte_mbuf *mb[], uint16_t num)
{
- uint32_t i, k;
+ uint32_t i, k, bytes = 0;
uint32_t dr[num];
RTE_SET_USED(ss);
k = 0;
for (i = 0; i != num; i++) {
- if ((mb[i]->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED) == 0)
+ if ((mb[i]->ol_flags & PKT_RX_SEC_OFFLOAD_FAILED) == 0) {
k++;
+ bytes += mb[i]->data_len;
+ }
else
dr[i - k] = i;
}
+ ss->sa->statistics.count += k;
+ ss->sa->statistics.bytes += bytes - (ss->sa->hdr_len * k);
+
/* handle unprocessed mbufs */
if (k != num) {
rte_errno = EBADMSG;
@@ -122,9 +122,30 @@ struct rte_ipsec_sa {
uint16_t mss;
} tso;
+ LIST_ENTRY(rte_ipsec_sa) telemetry_next;
+ /**< list entry for telemetry enabled SA */
+
+
+ RTE_MARKER cachealign_statistics __rte_cache_min_aligned;
+
+ /* Statistics */
+ struct {
+ uint64_t count;
+ uint64_t bytes;
+
+ struct {
+ uint64_t count;
+ uint64_t authentication_failed;
+ } errors;
+ } statistics;
+
+ RTE_MARKER cachealign_tunnel_header __rte_cache_min_aligned;
+
/* template for tunnel header */
uint8_t hdr[IPSEC_MAX_HDR_SIZE];
+
+ RTE_MARKER cachealign_tunnel_seq_num_replay_win __rte_cache_min_aligned;
/*
* sqn and replay window
* In case of SA handled by multiple threads *sqn* cacheline
@@ -19,3 +19,12 @@ DPDK_22 {
local: *;
};
+
+EXPERIMENTAL {
+ global:
+
+ # added in 21.11
+ rte_ipsec_telemetry_init;
+ rte_ipsec_telemetry_sa_add;
+
+};