@@ -19,7 +19,10 @@ CFLAGS += -O3
CFLAGS += $(WERROR_FLAGS)
endif
+CFLAGS += -I$(RTE_SDK)/drivers/bus/dpaa/include
CFLAGS += -I$(RTE_SDK)/drivers/crypto/caam_jr
+#sharing the hw flib headers from dpaa2_sec pmd
+CFLAGS += -I$(RTE_SDK)/drivers/crypto/dpaa2_sec/
CFLAGS += -I$(RTE_SDK)/lib/librte_eal/common/include
CFLAGS += -I$(RTE_SDK)/lib/librte_eal/linuxapp/eal
@@ -30,11 +33,14 @@ EXPORT_MAP := rte_pmd_caam_jr_version.map
LIBABIVER := 1
# library source files
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_CAAM_JR) += caam_jr_hw.c
+SRCS-$(CONFIG_RTE_LIBRTE_PMD_CAAM_JR) += caam_jr_uio.c
SRCS-$(CONFIG_RTE_LIBRTE_PMD_CAAM_JR) += caam_jr.c
# library dependencies
LDLIBS += -lrte_eal -lrte_mbuf -lrte_mempool -lrte_ring
LDLIBS += -lrte_cryptodev
+LDLIBS += -lrte_bus_dpaa
LDLIBS += -lrte_bus_vdev
include $(RTE_SDK)/mk/rte.lib.mk
@@ -16,13 +16,146 @@
#include <rte_malloc.h>
#include <rte_security_driver.h>
#include <rte_hexdump.h>
+#include <caam_jr_config.h>
+/* RTA header files */
+#include <hw/desc/common.h>
+#include <of.h>
+#include <caam_jr_hw_specific.h>
+#include <caam_jr_pvt.h>
#include <caam_jr_log.h>
#define CRYPTODEV_NAME_CAAM_JR_PMD crypto_caam_jr
static uint8_t cryptodev_driver_id;
int caam_jr_logtype;
+enum rta_sec_era rta_sec_era;
+
+/* Lists the states possible for the SEC user space driver. */
+enum sec_driver_state_e {
+ SEC_DRIVER_STATE_IDLE, /* Driver not initialized */
+ SEC_DRIVER_STATE_STARTED, /* Driver initialized and can be used*/
+ SEC_DRIVER_STATE_RELEASE, /* Driver release is in progress */
+};
+
+/* Job rings used for communication with SEC HW */
+static struct sec_job_ring_t g_job_rings[MAX_SEC_JOB_RINGS];
+
+/* The current state of SEC user space driver */
+static enum sec_driver_state_e g_driver_state = SEC_DRIVER_STATE_IDLE;
+
+/* The number of job rings used by SEC user space driver */
+static int g_job_rings_no;
+static int g_job_rings_max;
+
+/* @brief Poll the HW for already processed jobs in the JR
+ * and silently discard the available jobs or notify them to UA
+ * with indicated error code.
+ *
+ * @param [in,out] job_ring The job ring to poll.
+ * @param [in] do_notify Can be #TRUE or #FALSE. Indicates if
+ * descriptors are to be discarded
+ * or notified to UA with given error_code.
+ * @param [out] notified_descs Number of notified descriptors. Can be NULL
+ * if do_notify is #FALSE
+ */
+static void hw_flush_job_ring(struct sec_job_ring_t *job_ring,
+ uint32_t do_notify,
+ uint32_t *notified_descs)
+{
+ int32_t jobs_no_to_discard = 0;
+ int32_t discarded_descs_no = 0;
+
+ CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Flushing jr notify desc=[%d]",
+ job_ring, job_ring->pidx, job_ring->cidx, do_notify);
+
+ jobs_no_to_discard = hw_get_no_finished_jobs(job_ring);
+
+ /* Discard all jobs */
+ CAAM_JR_DEBUG("Jr[%p] pi[%d] ci[%d].Discarding %d descs",
+ job_ring, job_ring->pidx, job_ring->cidx,
+ jobs_no_to_discard);
+
+ while (jobs_no_to_discard > discarded_descs_no) {
+ /* Get completed descriptor */
+#if 0
+ /*TODO if we want to do something with desc*/
+ /* Since the memory is contigous, then P2V translation is a
+ * mere addition to the base descriptor physical address
+ */
+ current_desc = job_ring->output_ring[job_ring->cidx].desc;
+#endif
+
+ discarded_descs_no++;
+ /* Now increment the consumer index for the current job ring,
+ * AFTER saving job in temporary location!
+ * Increment the consumer index for the current job ring
+ */
+ job_ring->cidx = SEC_CIRCULAR_COUNTER(job_ring->cidx,
+ SEC_JOB_RING_SIZE);
+
+ hw_remove_entries(job_ring, 1);
+ }
+
+ if (do_notify == true) {
+ ASSERT(notified_descs != NULL);
+ *notified_descs = discarded_descs_no;
+ }
+}
+
+
+/* @brief Flush job rings of any processed descs.
+ * The processed descs are silently dropped,
+ * WITHOUT being notified to UA.
+ */
+static void close_job_ring(struct sec_job_ring_t *job_ring)
+{
+ if (job_ring->irq_fd) {
+ /* Producer index is frozen. If consumer index is not equal
+ * with producer index, then we have descs to flush.
+ */
+ while (job_ring->pidx != job_ring->cidx)
+ hw_flush_job_ring(job_ring, false, NULL);
+
+ /* free the uio job ring */
+ free_job_ring(job_ring->irq_fd);
+ job_ring->irq_fd = 0;
+ caam_jr_dma_free(job_ring->input_ring);
+ caam_jr_dma_free(job_ring->output_ring);
+
+ g_job_rings_no--;
+
+ }
+}
+
+/** @brief Release the software and hardware resources tied to a job ring.
+ * @param [in] job_ring The job ring
+ *
+ * @retval 0 for success
+ * @retval -1 for error
+ */
+static int shutdown_job_ring(struct sec_job_ring_t *job_ring)
+{
+ int ret = 0;
+
+ ASSERT(job_ring != NULL);
+ ret = hw_shutdown_job_ring(job_ring);
+ SEC_ASSERT(ret == 0, ret,
+ "Failed to shutdown hardware job ring %p",
+ job_ring);
+
+ if (job_ring->coalescing_en)
+ hw_job_ring_disable_coalescing(job_ring);
+
+ if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL) {
+ ret = caam_jr_disable_irqs(job_ring->irq_fd);
+ SEC_ASSERT(ret == 0, ret,
+ "Failed to disable irqs for job ring %p",
+ job_ring);
+ }
+
+ return ret;
+}
/*
* @brief Release the resources used by the SEC user space driver.
@@ -42,31 +175,195 @@ int caam_jr_logtype;
static int
caam_jr_dev_uninit(struct rte_cryptodev *dev)
{
+ struct sec_job_ring_t *internals;
if (dev == NULL)
return -ENODEV;
+ internals = dev->data->dev_private;
+ rte_free(dev->security_ctx);
+
+
+ /* If any descriptors in flight , poll and wait
+ * until all descriptors are received and silently discarded.
+ */
+ if (internals) {
+ shutdown_job_ring(internals);
+ close_job_ring(internals);
+ rte_mempool_free(internals->ctx_pool);
+ }
CAAM_JR_INFO("Closing DPAA_SEC device %s", dev->data->name);
+ /* last caam jr instance) */
+ if (g_job_rings_no == 0)
+ g_driver_state = SEC_DRIVER_STATE_IDLE;
- return 0;
+ return SEC_SUCCESS;
}
+/* @brief Initialize the software and hardware resources tied to a job ring.
+ * @param [in] jr_mode; Model to be used by SEC Driver to receive
+ * notifications from SEC. Can be either
+ * of the three: #SEC_NOTIFICATION_TYPE_NAPI
+ * #SEC_NOTIFICATION_TYPE_IRQ or
+ * #SEC_NOTIFICATION_TYPE_POLL
+ * @param [in] NAPI_mode The NAPI work mode to configure a job ring at
+ * startup. Used only when #SEC_NOTIFICATION_TYPE
+ * is set to #SEC_NOTIFICATION_TYPE_NAPI.
+ * @param [in] irq_coalescing_timer This value determines the maximum
+ * amount of time after processing a
+ * descriptor before raising an interrupt.
+ * @param [in] irq_coalescing_count This value determines how many
+ * descriptors are completed before
+ * raising an interrupt.
+ * @param [in] reg_base_addr, The job ring base address register
+ * @param [in] irq_id The job ring interrupt identification number.
+ * @retval job_ring_handle for successful job ring configuration
+ * @retval NULL on error
+ *
+ */
+static void *
+init_job_ring(void *reg_base_addr, uint32_t irq_id)
+{
+ struct sec_job_ring_t *job_ring = NULL;
+ int i, ret = 0;
+ int jr_mode = SEC_NOTIFICATION_TYPE_POLL;
+ int napi_mode = 0;
+ int irq_coalescing_timer = 0;
+ int irq_coalescing_count = 0;
+
+ for (i = 0; i < MAX_SEC_JOB_RINGS; i++) {
+ if (g_job_rings[i].irq_fd == 0) {
+ job_ring = &g_job_rings[i];
+ g_job_rings_no++;
+ break;
+ }
+ }
+ if (job_ring == NULL) {
+ CAAM_JR_ERR("No free job ring\n");
+ return NULL;
+ }
+
+ job_ring->register_base_addr = reg_base_addr;
+ job_ring->jr_mode = jr_mode;
+ job_ring->napi_mode = 0;
+ job_ring->irq_fd = irq_id;
+
+ /* Allocate mem for input and output ring */
+
+ /* Allocate memory for input ring */
+ job_ring->input_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
+ SEC_DMA_MEM_INPUT_RING_SIZE);
+ memset(job_ring->input_ring, 0, SEC_DMA_MEM_INPUT_RING_SIZE);
+
+ /* Allocate memory for output ring */
+ job_ring->output_ring = caam_jr_dma_mem_alloc(L1_CACHE_BYTES,
+ SEC_DMA_MEM_OUTPUT_RING_SIZE);
+ memset(job_ring->output_ring, 0, SEC_DMA_MEM_OUTPUT_RING_SIZE);
+
+ /* Reset job ring in SEC hw and configure job ring registers */
+ ret = hw_reset_job_ring(job_ring);
+ if (ret != 0) {
+ CAAM_JR_ERR("Failed to reset hardware job ring");
+ goto cleanup;
+ }
+
+ if (jr_mode == SEC_NOTIFICATION_TYPE_NAPI) {
+ /* When SEC US driver works in NAPI mode, the UA can select
+ * if the driver starts with IRQs on or off.
+ */
+ if (napi_mode == SEC_STARTUP_INTERRUPT_MODE) {
+ CAAM_JR_INFO("Enabling DONE IRQ generationon job ring - %p",
+ job_ring);
+ ret = caam_jr_enable_irqs(job_ring->irq_fd);
+ if (ret != 0) {
+ CAAM_JR_ERR("Failed to enable irqs for job ring");
+ goto cleanup;
+ }
+ }
+ } else if (jr_mode == SEC_NOTIFICATION_TYPE_IRQ) {
+ /* When SEC US driver works in pure interrupt mode,
+ * IRQ's are always enabled.
+ */
+ CAAM_JR_INFO("Enabling DONE IRQ generation on job ring - %p",
+ job_ring);
+ ret = caam_jr_enable_irqs(job_ring->irq_fd);
+ if (ret != 0) {
+ CAAM_JR_ERR("Failed to enable irqs for job ring");
+ goto cleanup;
+ }
+ }
+ if (irq_coalescing_timer || irq_coalescing_count) {
+ hw_job_ring_set_coalescing_param(job_ring,
+ irq_coalescing_timer,
+ irq_coalescing_count);
+
+ hw_job_ring_enable_coalescing(job_ring);
+ job_ring->coalescing_en = 1;
+ }
+
+ job_ring->jr_state = SEC_JOB_RING_STATE_STARTED;
+ job_ring->max_nb_queue_pairs = RTE_CAAM_MAX_NB_SEC_QPS;
+ job_ring->max_nb_sessions = RTE_CAAM_JR_PMD_MAX_NB_SESSIONS;
+
+ return job_ring;
+cleanup:
+ caam_jr_dma_free(job_ring->output_ring);
+ caam_jr_dma_free(job_ring->input_ring);
+ return NULL;
+}
+
+
static int
caam_jr_dev_init(const char *name,
struct rte_vdev_device *vdev,
struct rte_cryptodev_pmd_init_params *init_params)
{
struct rte_cryptodev *dev;
+ struct uio_job_ring *job_ring;
+ char str[RTE_CRYPTODEV_NAME_MAX_LEN];
PMD_INIT_FUNC_TRACE();
+ /* Validate driver state */
+ if (g_driver_state == SEC_DRIVER_STATE_IDLE) {
+ g_job_rings_max = sec_configure();
+ if (!g_job_rings_max) {
+ CAAM_JR_ERR("No job ring detected on UIO !!!!");
+ return -1;
+ }
+ /* Update driver state */
+ g_driver_state = SEC_DRIVER_STATE_STARTED;
+ }
+
+ if (g_job_rings_no >= g_job_rings_max) {
+ CAAM_JR_ERR("No more job rings available max=%d!!!!",
+ g_job_rings_max);
+ return -1;
+ }
+
+ job_ring = config_job_ring();
+ if (job_ring == NULL) {
+ CAAM_JR_ERR("failed to create job ring");
+ goto init_error;
+ }
+
+ snprintf(str, sizeof(str), "caam_jr%d", job_ring->jr_id);
+
dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
if (dev == NULL) {
CAAM_JR_ERR("failed to create cryptodev vdev");
goto cleanup;
}
+ /*TODO free it during teardown*/
+ dev->data->dev_private = init_job_ring(job_ring->register_base_addr,
+ job_ring->uio_fd);
+
+ if (!dev->data->dev_private) {
+ CAAM_JR_ERR("Ring memory allocation failed\n");
+ goto cleanup2;
+ }
dev->driver_id = cryptodev_driver_id;
dev->dev_ops = NULL;
@@ -84,7 +381,12 @@ caam_jr_dev_init(const char *name,
return 0;
+cleanup2:
+ caam_jr_dev_uninit(dev);
+ rte_cryptodev_pmd_release_device(dev);
cleanup:
+ free_job_ring(job_ring->uio_fd);
+init_error:
CAAM_JR_ERR("driver %s: cryptodev_caam_jr_create failed",
init_params->name);
@@ -97,7 +399,7 @@ cryptodev_caam_jr_probe(struct rte_vdev_device *vdev)
{
struct rte_cryptodev_pmd_init_params init_params = {
"",
- 128,
+ sizeof(struct sec_job_ring_t),
rte_socket_id(),
RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS
};
@@ -111,6 +413,29 @@ cryptodev_caam_jr_probe(struct rte_vdev_device *vdev)
input_args = rte_vdev_device_args(vdev);
rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
+ /* if sec device version is not configured */
+ if (!rta_get_sec_era()) {
+ const struct device_node *caam_node;
+
+ for_each_compatible_node(caam_node, NULL, "fsl,sec-v4.0") {
+ const uint32_t *prop = of_get_property(caam_node,
+ "fsl,sec-era",
+ NULL);
+ if (prop) {
+ rta_set_sec_era(
+ INTL_SEC_ERA(cpu_to_caam32(*prop)));
+ break;
+ }
+ }
+ }
+#ifdef RTE_LIBRTE_PMD_CAAM_JR_BE
+ if (rta_get_sec_era() > RTA_SEC_ERA_8) {
+ RTE_LOG(ERR, PMD,
+ "CAAM is compiled in BE mode for device with sec era > 8???\n");
+ return -EINVAL;
+ }
+#endif
+
return caam_jr_dev_init(name, vdev, &init_params);
}
new file mode 100644
@@ -0,0 +1,207 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017-2018 NXP
+ */
+
+#ifndef CAAM_JR_CONFIG_H
+#define CAAM_JR_CONFIG_H
+
+#include <rte_byteorder.h>
+
+#ifdef RTE_LIBRTE_PMD_CAAM_JR_BE
+#define CAAM_BYTE_ORDER __BIG_ENDIAN
+#else
+#define CAAM_BYTE_ORDER __LITTLE_ENDIAN
+#endif
+
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+#define CORE_BYTE_ORDER __BIG_ENDIAN
+#else
+#define CORE_BYTE_ORDER __LITTLE_ENDIAN
+#endif
+
+typedef uint64_t dma_addr_t;
+
+#if CORE_BYTE_ORDER != CAAM_BYTE_ORDER
+
+#define cpu_to_caam64 rte_cpu_to_be_64
+#define cpu_to_caam32 rte_cpu_to_be_32
+#else
+#define cpu_to_caam64
+#define cpu_to_caam32
+
+#endif
+
+/*
+ * SEC is configured to start work in polling mode,
+ * when configured for NAPI notification style.
+ */
+#define SEC_STARTUP_POLLING_MODE 0
+/*
+ * SEC is configured to start work in interrupt mode,
+ * when configured for NAPI notification style.
+ */
+#define SEC_STARTUP_INTERRUPT_MODE 1
+
+/*
+ * SEC driver will use NAPI model to receive notifications
+ * for processed packets from SEC engine hardware:
+ * - IRQ for low traffic
+ * - polling for high traffic.
+ */
+#define SEC_NOTIFICATION_TYPE_NAPI 0
+/*
+ * SEC driver will use ONLY interrupts to receive notifications
+ * for processed packets from SEC engine hardware.
+ */
+#define SEC_NOTIFICATION_TYPE_IRQ 1
+/*
+ * SEC driver will use ONLY polling to receive notifications
+ * for processed packets from SEC engine hardware.
+ */
+#define SEC_NOTIFICATION_TYPE_POLL 2
+
+/*
+ * SEC USER SPACE DRIVER related configuration.
+ */
+
+/*
+ * Determines how SEC user space driver will receive notifications
+ * for processed packets from SEC engine.
+ * Valid values are: #SEC_NOTIFICATION_TYPE_POLL, #SEC_NOTIFICATION_TYPE_IRQ
+ * and #SEC_NOTIFICATION_TYPE_NAPI.
+ */
+#define SEC_NOTIFICATION_TYPE SEC_NOTIFICATION_TYPE_POLL
+
+/* Maximum number of job rings supported by SEC hardware */
+#define MAX_SEC_JOB_RINGS 4
+
+/* Maximum number of QP per job ring */
+#define RTE_CAAM_MAX_NB_SEC_QPS 1
+
+/*
+ * Size of cryptographic context that is used directly in communicating
+ * with SEC device. SEC device works only with physical addresses. This
+ * is the maximum size for a SEC descriptor ( = 64 words).
+ */
+#define SEC_CRYPTO_DESCRIPTOR_SIZE 256
+
+/*
+ * Size of job descriptor submitted to SEC device for each packet to
+ * be processed.
+ * Job descriptor contains 3 DMA address pointers:
+ * - to shared descriptor, to input buffer and to output buffer.
+ * The job descriptor contains other SEC specific commands as well:
+ * - HEADER command, SEQ IN PTR command SEQ OUT PTR command and opaque data
+ * each measuring 4 bytes.
+ * Job descriptor size, depending on physical address representation:
+ * - 32 bit - size is 28 bytes - cacheline-aligned size is 64 bytes
+ * - 36 bit - size is 40 bytes - cacheline-aligned size is 64 bytes
+ * @note: Job descriptor must be cacheline-aligned to ensure efficient
+ * memory access.
+ * @note: If other format is used for job descriptor, then the size must be
+ * revised.
+ */
+#define SEC_JOB_DESCRIPTOR_SIZE 64
+
+/*
+ * Size of one entry in the input ring of a job ring.
+ * Input ring contains pointers to job descriptors.
+ * The memory used for an input ring and output ring must be physically
+ * contiguous.
+ */
+#define SEC_JOB_INPUT_RING_ENTRY_SIZE sizeof(dma_addr_t)
+
+/*
+ * Size of one entry in the output ring of a job ring.
+ * Output ring entry is a pointer to a job descriptor followed by a 4 byte
+ * status word.
+ * The memory used for an input ring and output ring must be physically
+ * contiguous.
+ * @note If desired to use also the optional SEQ OUT indication in output ring
+ * entries,
+ * then 4 more bytes must be added to the size.
+ */
+#define SEC_JOB_OUTPUT_RING_ENTRY_SIZE (SEC_JOB_INPUT_RING_ENTRY_SIZE + 4)
+
+/*
+ * DMA memory required for an input ring of a job ring.
+ */
+#define SEC_DMA_MEM_INPUT_RING_SIZE ((SEC_JOB_INPUT_RING_ENTRY_SIZE) * \
+ (SEC_JOB_RING_SIZE))
+
+/*
+ * DMA memory required for an output ring of a job ring.
+ * Required extra 4 byte for status word per each entry.
+ */
+#define SEC_DMA_MEM_OUTPUT_RING_SIZE ((SEC_JOB_OUTPUT_RING_ENTRY_SIZE) * \
+ (SEC_JOB_RING_SIZE))
+
+/* DMA memory required for a job ring, including both input and output rings. */
+#define SEC_DMA_MEM_JOB_RING_SIZE ((SEC_DMA_MEM_INPUT_RING_SIZE) + \
+ (SEC_DMA_MEM_OUTPUT_RING_SIZE))
+
+/*
+ * When calling sec_init() UA will provide an area of virtual memory
+ * of size #SEC_DMA_MEMORY_SIZE to be used internally by the driver
+ * to allocate data (like SEC descriptors) that needs to be passed to
+ * SEC device in physical addressing and later on retrieved from SEC device.
+ * At initialization the UA provides specialized ptov/vtop functions/macros to
+ * translate addresses allocated from this memory area.
+ */
+#define SEC_DMA_MEMORY_SIZE ((SEC_DMA_MEM_JOB_RING_SIZE) * \
+ (MAX_SEC_JOB_RINGS))
+
+#define L1_CACHE_BYTES 64
+
+/* SEC JOB RING related configuration. */
+
+/*
+ * Configure the size of the JOB RING.
+ * The maximum size of the ring in hardware limited to 1024.
+ * However the number of packets in flight in a time interval of 1ms can
+ * be calculated from the traffic rate (Mbps) and packet size.
+ * Here it was considered a packet size of 64 bytes.
+ *
+ * @note Round up to nearest power of 2 for optimized update
+ * of producer/consumer indexes of each job ring
+ */
+#define SEC_JOB_RING_SIZE 512
+
+/*
+ * Interrupt coalescing related configuration.
+ * NOTE: SEC hardware enabled interrupt
+ * coalescing is not supported on SEC version 3.1!
+ * SEC version 4.4 has support for interrupt
+ * coalescing.
+ */
+
+#if SEC_NOTIFICATION_TYPE != SEC_NOTIFICATION_TYPE_POLL
+
+#define SEC_INT_COALESCING_ENABLE 1
+/*
+ * Interrupt Coalescing Descriptor Count Threshold.
+ * While interrupt coalescing is enabled (ICEN=1), this value determines
+ * how many Descriptors are completed before raising an interrupt.
+ *
+ * Valid values for this field are from 0 to 255.
+ * Note that a value of 1 functionally defeats the advantages of interrupt
+ * coalescing since the threshold value is reached each time that a
+ * Job Descriptor is completed. A value of 0 is treated in the same
+ * manner as a value of 1.
+ */
+#define SEC_INTERRUPT_COALESCING_DESCRIPTOR_COUNT_THRESH 10
+
+/*
+ * Interrupt Coalescing Timer Threshold.
+ * While interrupt coalescing is enabled (ICEN=1), this value determines the
+ * maximum amount of time after processing a Descriptor before raising an
+ * interrupt.
+ * The threshold value is represented in units equal to 64 CAAM interface
+ * clocks. Valid values for this field are from 1 to 65535.
+ * A value of 0 results in behavior identical to that when interrupt
+ * coalescing is disabled.
+ */
+#define SEC_INTERRUPT_COALESCING_TIMER_THRESH 100
+#endif /* SEC_NOTIFICATION_TYPE_POLL */
+
+#endif /* CAAM_JR_CONFIG_H */
new file mode 100644
@@ -0,0 +1,365 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017-2018 NXP
+ */
+
+#include <fcntl.h>
+#include <unistd.h>
+#include <inttypes.h>
+#include <rte_common.h>
+#include <rte_memory.h>
+#include <rte_malloc.h>
+#include <rte_crypto.h>
+#include <rte_security.h>
+#include <caam_jr_config.h>
+
+/* RTA header files */
+#include <hw/desc/common.h>
+#include <hw/desc/algo.h>
+#include <hw/desc/ipsec.h>
+
+#include <caam_jr_hw_specific.h>
+#include <caam_jr_pvt.h>
+#include <caam_jr_log.h>
+
+/* Used to retry resetting a job ring in SEC hardware. */
+#define SEC_TIMEOUT 100000
+
+/* @brief Process Jump Halt Condition related errors
+ *
+ * @param [in] error_code The error code in the descriptor status word
+ */
+static inline void hw_handle_jmp_halt_cond_err(union hw_error_code
+ error_code) {
+ CAAM_JR_DEBUG("JMP: %d, Descriptor Index: 0x%x, Condition: 0x%x",
+ error_code.error_desc.jmp_halt_cond_src.jmp,
+ error_code.error_desc.jmp_halt_cond_src.desc_idx,
+ error_code.error_desc.jmp_halt_cond_src.cond);
+ (void)error_code;
+}
+
+/* @brief Process DECO related errors
+ *
+ * @param [in] error_code The error code in the descriptor status word
+ */
+static inline void hw_handle_deco_err(union hw_error_code
+ error_code) {
+ CAAM_JR_DEBUG("JMP: %d, Descriptor Index: 0x%x",
+ error_code.error_desc.deco_src.jmp,
+ error_code.error_desc.deco_src.desc_idx);
+
+ switch (error_code.error_desc.deco_src.desc_err) {
+ case SEC_HW_ERR_DECO_HFN_THRESHOLD:
+ CAAM_JR_DEBUG(" Warning: Descriptor completed normally,"
+ "but 3GPP HFN matches or exceeds the Threshold ");
+ break;
+ default:
+ CAAM_JR_DEBUG("Error 0x%04x not implemented",
+ error_code.error_desc.deco_src.desc_err);
+ break;
+ }
+}
+
+/* @brief Process Jump Halt User Status related errors
+ *
+ * @param [in] error_code The error code in the descriptor status word
+ */
+static inline void hw_handle_jmp_halt_user_err(union hw_error_code
+ error_code __rte_unused)
+{
+ CAAM_JR_DEBUG(" Not implemented");
+}
+
+/* @brief Process CCB related errors
+ *
+ * @param [in] error_code The error code in the descriptor status word
+ */
+static inline void
+hw_handle_ccb_err(union hw_error_code hw_error_code __rte_unused)
+{
+ CAAM_JR_DEBUG(" Not implemented");
+}
+
+/* @brief Process Job Ring related errors
+ *
+ * @param [in] error_code The error code in the descriptor status word
+ */
+static inline
+void hw_handle_jr_err(union hw_error_code hw_error_code __rte_unused)
+{
+ CAAM_JR_DEBUG(" Not implemented");
+}
+
+int hw_reset_job_ring(struct sec_job_ring_t *job_ring)
+{
+ int ret = 0;
+
+ ASSERT(job_ring->register_base_addr != NULL);
+
+ /* First reset the job ring in hw */
+ ret = hw_shutdown_job_ring(job_ring);
+ SEC_ASSERT(ret == 0, ret, "Failed resetting job ring in hardware");
+
+ /* In order to have the HW JR in a workable state
+ * after a reset, I need to re-write the input
+ * queue size, input start address, output queue
+ * size and output start address
+ */
+ /* Write the JR input queue size to the HW register */
+ hw_set_input_ring_size(job_ring, SEC_JOB_RING_SIZE);
+
+ /* Write the JR output queue size to the HW register */
+ hw_set_output_ring_size(job_ring, SEC_JOB_RING_SIZE);
+
+ /* Write the JR input queue start address */
+ hw_set_input_ring_start_addr(job_ring,
+ caam_jr_dma_vtop(job_ring->input_ring));
+ CAAM_JR_DEBUG(" Set input ring base address to : Virtual: 0x%" PRIx64
+ ",Physical: 0x%" PRIx64 ", Read from HW: 0x%" PRIx64,
+ (uint64_t)(uintptr_t)job_ring->input_ring,
+ caam_jr_dma_vtop(job_ring->input_ring),
+ hw_get_inp_queue_base(job_ring));
+
+ /* Write the JR output queue start address */
+ hw_set_output_ring_start_addr(job_ring,
+ caam_jr_dma_vtop(job_ring->output_ring));
+ CAAM_JR_DEBUG(" Set output ring base address to: Virtual: 0x%" PRIx64
+ ",Physical: 0x%" PRIx64 ", Read from HW: 0x%" PRIx64,
+ (uint64_t)(uintptr_t)job_ring->output_ring,
+ caam_jr_dma_vtop(job_ring->output_ring),
+ hw_get_out_queue_base(job_ring));
+ return ret;
+}
+
+int hw_shutdown_job_ring(struct sec_job_ring_t *job_ring)
+{
+ unsigned int timeout = SEC_TIMEOUT;
+ uint32_t tmp = 0;
+ int usleep_interval = 10;
+
+ if (job_ring->register_base_addr == NULL) {
+ CAAM_JR_ERR("Jr[%p] has reg base addr as NULL.driver not init",
+ job_ring);
+ return 0;
+ }
+
+ CAAM_JR_INFO("Resetting Job ring %p", job_ring);
+
+ /*
+ * Mask interrupts since we are going to poll
+ * for reset completion status
+ * Also, at POR, interrupts are ENABLED on a JR, thus
+ * this is the point where I can disable them without
+ * changing the code logic too much
+ */
+ caam_jr_disable_irqs(job_ring->irq_fd);
+
+ /* initiate flush (required prior to reset) */
+ SET_JR_REG(JRCR, job_ring, JR_REG_JRCR_VAL_RESET);
+
+ /* dummy read */
+ tmp = GET_JR_REG(JRCR, job_ring);
+
+ do {
+ tmp = GET_JR_REG(JRINT, job_ring);
+ usleep(usleep_interval);
+ } while (((tmp & JRINT_ERR_HALT_MASK) ==
+ JRINT_ERR_HALT_INPROGRESS) && --timeout);
+
+ CAAM_JR_INFO("JRINT is %x", tmp);
+
+ if ((tmp & JRINT_ERR_HALT_MASK) != JRINT_ERR_HALT_COMPLETE ||
+ timeout == 0) {
+ CAAM_JR_ERR("0x%x, %d", tmp, timeout);
+ /* unmask interrupts */
+ if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL)
+ caam_jr_enable_irqs(job_ring->irq_fd);
+ return -1;
+ }
+
+ /* Initiate reset */
+ timeout = SEC_TIMEOUT;
+ SET_JR_REG(JRCR, job_ring, JR_REG_JRCR_VAL_RESET);
+
+ do {
+ tmp = GET_JR_REG(JRCR, job_ring);
+ usleep(usleep_interval);
+ } while ((tmp & JR_REG_JRCR_VAL_RESET) && --timeout);
+
+ CAAM_JR_DEBUG("JRCR is %x", tmp);
+
+ if (timeout == 0) {
+ CAAM_JR_ERR("Failed to reset hw job ring %p", job_ring);
+ /* unmask interrupts */
+ if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL)
+ caam_jr_enable_irqs(job_ring->irq_fd);
+ return -1;
+ }
+ /* unmask interrupts */
+ if (job_ring->jr_mode != SEC_NOTIFICATION_TYPE_POLL)
+ caam_jr_enable_irqs(job_ring->irq_fd);
+ return 0;
+
+}
+
+void hw_handle_job_ring_error(struct sec_job_ring_t *job_ring __rte_unused,
+ uint32_t error_code)
+{
+ union hw_error_code hw_err_code;
+
+ hw_err_code.error = error_code;
+
+ switch (hw_err_code.error_desc.value.ssrc) {
+ case SEC_HW_ERR_SSRC_NO_SRC:
+ ASSERT(hw_err_code.error_desc.no_status_src.res == 0);
+ CAAM_JR_ERR("No Status Source ");
+ break;
+ case SEC_HW_ERR_SSRC_CCB_ERR:
+ CAAM_JR_ERR("CCB Status Source");
+ hw_handle_ccb_err(hw_err_code);
+ break;
+ case SEC_HW_ERR_SSRC_JMP_HALT_U:
+ CAAM_JR_ERR("Jump Halt User Status Source");
+ hw_handle_jmp_halt_user_err(hw_err_code);
+ break;
+ case SEC_HW_ERR_SSRC_DECO:
+ CAAM_JR_ERR("DECO Status Source");
+ hw_handle_deco_err(hw_err_code);
+ break;
+ case SEC_HW_ERR_SSRC_JR:
+ CAAM_JR_ERR("Job Ring Status Source");
+ hw_handle_jr_err(hw_err_code);
+ break;
+ case SEC_HW_ERR_SSRC_JMP_HALT_COND:
+ CAAM_JR_ERR("Jump Halt Condition Codes");
+ hw_handle_jmp_halt_cond_err(hw_err_code);
+ break;
+ default:
+ ASSERT(0);
+ CAAM_JR_ERR("Unknown SSRC");
+ break;
+ }
+}
+
+void hw_job_ring_error_print(struct sec_job_ring_t *job_ring, int code)
+{
+ switch (code) {
+ case JRINT_ERR_WRITE_STATUS:
+ CAAM_JR_ERR("Error writing status to Output Ring ");
+ break;
+ case JRINT_ERR_BAD_INPUT_BASE:
+ CAAM_JR_ERR(
+ "Bad Input Ring Base (%p) (not on a 4-byte boundary) ",
+ (void *)job_ring);
+ break;
+ case JRINT_ERR_BAD_OUTPUT_BASE:
+ CAAM_JR_ERR(
+ "Bad Output Ring Base (%p) (not on a 4-byte boundary) ",
+ (void *)job_ring);
+ break;
+ case JRINT_ERR_WRITE_2_IRBA:
+ CAAM_JR_ERR(
+ "Invalid write to Input Ring Base Address Register ");
+ break;
+ case JRINT_ERR_WRITE_2_ORBA:
+ CAAM_JR_ERR(
+ "Invalid write to Output Ring Base Address Register ");
+ break;
+ case JRINT_ERR_RES_B4_HALT:
+ CAAM_JR_ERR(
+ "Job Ring [%p] released before Job Ring is halted",
+ (void *)job_ring);
+ break;
+ case JRINT_ERR_REM_TOO_MANY:
+ CAAM_JR_ERR("Removed too many jobs from job ring [%p]",
+ (void *)job_ring);
+ break;
+ case JRINT_ERR_ADD_TOO_MANY:
+ CAAM_JR_ERR("Added too many jobs on job ring [%p]", job_ring);
+ break;
+ default:
+ CAAM_JR_ERR(" Unknown SEC JR Error :%d",
+ code);
+ break;
+ }
+}
+
+int hw_job_ring_set_coalescing_param(struct sec_job_ring_t *job_ring,
+ uint16_t irq_coalescing_timer,
+ uint8_t irq_coalescing_count)
+{
+ uint32_t reg_val = 0;
+
+ ASSERT(job_ring != NULL);
+
+ if (job_ring->register_base_addr == NULL) {
+ CAAM_JR_ERR("Jr[%p] has reg base addr as NULL.driver not init",
+ job_ring);
+ return -1;
+ }
+ /* Set descriptor count coalescing */
+ reg_val |= (irq_coalescing_count << JR_REG_JRCFG_LO_ICDCT_SHIFT);
+
+ /* Set coalescing timer value */
+ reg_val |= (irq_coalescing_timer << JR_REG_JRCFG_LO_ICTT_SHIFT);
+
+ /* Update parameters in HW */
+ SET_JR_REG_LO(JRCFG, job_ring, reg_val);
+
+ CAAM_JR_DEBUG("Set coalescing params on jr %p timer:%d, desc count: %d",
+ job_ring, irq_coalescing_timer, irq_coalescing_timer);
+
+ return 0;
+}
+
+int hw_job_ring_enable_coalescing(struct sec_job_ring_t *job_ring)
+{
+ uint32_t reg_val = 0;
+
+ ASSERT(job_ring != NULL);
+ if (job_ring->register_base_addr == NULL) {
+ CAAM_JR_ERR("Jr[%p] has reg base addr as NULL.driver not init",
+ job_ring);
+ return -1;
+ }
+
+ /* Get the current value of the register */
+ reg_val = GET_JR_REG_LO(JRCFG, job_ring);
+
+ /* Enable coalescing */
+ reg_val |= JR_REG_JRCFG_LO_ICEN_EN;
+
+ /* Write in hw */
+ SET_JR_REG_LO(JRCFG, job_ring, reg_val);
+
+ CAAM_JR_DEBUG("Enabled coalescing on jr %p ",
+ job_ring);
+
+ return 0;
+}
+
+int hw_job_ring_disable_coalescing(struct sec_job_ring_t *job_ring)
+{
+ uint32_t reg_val = 0;
+
+ ASSERT(job_ring != NULL);
+
+ if (job_ring->register_base_addr == NULL) {
+ CAAM_JR_ERR("Jr[%p] has reg base addr as NULL.driver not init",
+ job_ring);
+ return -1;
+ }
+
+ /* Get the current value of the register */
+ reg_val = GET_JR_REG_LO(JRCFG, job_ring);
+
+ /* Disable coalescing */
+ reg_val &= ~JR_REG_JRCFG_LO_ICEN_EN;
+
+ /* Write in hw */
+ SET_JR_REG_LO(JRCFG, job_ring, reg_val);
+
+ CAAM_JR_DEBUG("Disabled coalescing on jr %p ", job_ring);
+
+ return 0;
+
+}
new file mode 100644
@@ -0,0 +1,503 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017 NXP
+ */
+
+#ifndef CAAM_JR_HW_SPECIFIC_H
+#define CAAM_JR_HW_SPECIFIC_H
+
+#include <caam_jr_config.h>
+
+/*
+ * Offset to the registers of a job ring.
+ * Is different for each job ring.
+ */
+#define CHAN_BASE(jr) ((size_t)(jr)->register_base_addr)
+
+#ifndef unlikely
+#define unlikely(x) __builtin_expect(!!(x), 0)
+#endif
+
+#define SEC_JOB_RING_IS_FULL(pi, ci, ring_max_size, ring_threshold) \
+ ((((pi) + 1 + ((ring_max_size) - (ring_threshold))) & \
+ (ring_max_size - 1)) == ((ci)))
+
+#define SEC_CIRCULAR_COUNTER(x, max) (((x) + 1) & (max - 1))
+
+/*
+ * Assert that cond is true. If !cond is true, display str and the vararg list
+ * in a printf-like syntax. also, if !cond is true, return altRet.
+ *
+ * \param cond A boolean expression to be asserted true
+ * \param altRet The value to be returned if cond doesn't hold true
+ * \param str A quoted char string
+ *
+ * E.g.:
+ * SEC_ASSERT(ret > 0, 0, "ERROR initializing app: code = %d\n", ret);
+ */
+#define SEC_ASSERT(cond, altRet, ...) {\
+ if (unlikely(!(cond))) {\
+ CAAM_JR_ERR(__VA_ARGS__); \
+ return altRet; \
+ } \
+}
+
+#define SEC_DP_ASSERT(cond, altRet, ...) {\
+ if (unlikely(!(cond))) {\
+ CAAM_JR_DP_ERR(__VA_ARGS__); \
+ return altRet; \
+ } \
+}
+
+#define ASSERT(x)
+
+/*
+ * Constants representing various job ring registers
+ */
+#if CAAM_BYTE_ORDER == __BIG_ENDIAN
+#define JR_REG_IRBA_OFFSET 0x0000
+#define JR_REG_IRBA_OFFSET_LO 0x0004
+#else
+#define JR_REG_IRBA_OFFSET 0x0004
+#define JR_REG_IRBA_OFFSET_LO 0x0000
+#endif
+
+#define JR_REG_IRSR_OFFSET 0x000C
+#define JR_REG_IRSA_OFFSET 0x0014
+#define JR_REG_IRJA_OFFSET 0x001C
+
+#if CAAM_BYTE_ORDER == __BIG_ENDIAN
+#define JR_REG_ORBA_OFFSET 0x0020
+#define JR_REG_ORBA_OFFSET_LO 0x0024
+#else
+#define JR_REG_ORBA_OFFSET 0x0024
+#define JR_REG_ORBA_OFFSET_LO 0x0020
+#endif
+
+#define JR_REG_ORSR_OFFSET 0x002C
+#define JR_REG_ORJR_OFFSET 0x0034
+#define JR_REG_ORSFR_OFFSET 0x003C
+#define JR_REG_JROSR_OFFSET 0x0044
+#define JR_REG_JRINT_OFFSET 0x004C
+
+#define JR_REG_JRCFG_OFFSET 0x0050
+#define JR_REG_JRCFG_OFFSET_LO 0x0054
+
+#define JR_REG_IRRI_OFFSET 0x005C
+#define JR_REG_ORWI_OFFSET 0x0064
+#define JR_REG_JRCR_OFFSET 0x006C
+
+/*
+ * Constants for error handling on job ring
+ */
+#define JR_REG_JRINT_ERR_TYPE_SHIFT 8
+#define JR_REG_JRINT_ERR_ORWI_SHIFT 16
+#define JR_REG_JRINIT_JRE_SHIFT 1
+
+#define JRINT_JRE (1 << JR_REG_JRINIT_JRE_SHIFT)
+#define JRINT_ERR_WRITE_STATUS (1 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_BAD_INPUT_BASE (3 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_BAD_OUTPUT_BASE (4 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_WRITE_2_IRBA (5 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_WRITE_2_ORBA (6 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_RES_B4_HALT (7 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_REM_TOO_MANY (8 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_ADD_TOO_MANY (9 << JR_REG_JRINT_ERR_TYPE_SHIFT)
+#define JRINT_ERR_HALT_MASK 0x0C
+#define JRINT_ERR_HALT_INPROGRESS 0x04
+#define JRINT_ERR_HALT_COMPLETE 0x08
+
+#define JR_REG_JRCR_VAL_RESET 0x00000001
+
+#define JR_REG_JRCFG_LO_ICTT_SHIFT 0x10
+#define JR_REG_JRCFG_LO_ICDCT_SHIFT 0x08
+#define JR_REG_JRCFG_LO_ICEN_EN 0x02
+
+/*
+ * Constants for Descriptor Processing errors
+ */
+#define SEC_HW_ERR_SSRC_NO_SRC 0x00
+#define SEC_HW_ERR_SSRC_CCB_ERR 0x02
+#define SEC_HW_ERR_SSRC_JMP_HALT_U 0x03
+#define SEC_HW_ERR_SSRC_DECO 0x04
+#define SEC_HW_ERR_SSRC_JR 0x06
+#define SEC_HW_ERR_SSRC_JMP_HALT_COND 0x07
+
+#define SEC_HW_ERR_DECO_HFN_THRESHOLD 0xF1
+#define SEC_HW_ERR_CCB_ICV_CHECK_FAIL 0x0A
+
+/*
+ * Constants for descriptors
+ */
+/* Return higher 32 bits of physical address */
+#define PHYS_ADDR_HI(phys_addr) \
+ (uint32_t)(((uint64_t)phys_addr) >> 32)
+
+/* Return lower 32 bits of physical address */
+#define PHYS_ADDR_LO(phys_addr) \
+ (uint32_t)(((uint64_t)phys_addr) & 0xFFFFFFFF)
+
+/*
+ * Macros for extracting error codes for the job ring
+ */
+#define JR_REG_JRINT_ERR_TYPE_EXTRACT(value) ((value) & 0x00000F00)
+#define JR_REG_JRINT_ERR_ORWI_EXTRACT(value) \
+ (((value) & 0x3FFF0000) >> JR_REG_JRINT_ERR_ORWI_SHIFT)
+#define JR_REG_JRINT_JRE_EXTRACT(value) ((value) & JRINT_JRE)
+
+/*
+ * Macros for managing the job ring
+ */
+/* Read pointer to job ring input ring start address */
+#if defined(RTE_ARCH_ARM64)
+#define hw_get_inp_queue_base(jr) ((((dma_addr_t)GET_JR_REG(IRBA, (jr))) << 32) | \
+ (GET_JR_REG_LO(IRBA, (jr))))
+
+/* Read pointer to job ring output ring start address */
+#define hw_get_out_queue_base(jr) (((dma_addr_t)(GET_JR_REG(ORBA, (jr))) << 32) | \
+ (GET_JR_REG_LO(ORBA, (jr))))
+#else
+#define hw_get_inp_queue_base(jr) ((dma_addr_t)(GET_JR_REG_LO(IRBA, (jr))))
+
+#define hw_get_out_queue_base(jr) ((dma_addr_t)(GET_JR_REG_LO(ORBA, (jr))))
+#endif
+
+/*
+ * IRJA - Input Ring Jobs Added Register shows
+ * how many new jobs were added to the Input Ring.
+ */
+#define hw_enqueue_desc_on_job_ring(job_ring) SET_JR_REG(IRJA, (job_ring), 1)
+
+#define hw_set_input_ring_size(job_ring, size) SET_JR_REG(IRSR, job_ring, (size))
+
+#define hw_set_output_ring_size(job_ring, size) SET_JR_REG(ORSR, job_ring, (size))
+
+#if defined(RTE_ARCH_ARM64)
+#define hw_set_input_ring_start_addr(job_ring, start_addr) \
+{ \
+ SET_JR_REG(IRBA, job_ring, PHYS_ADDR_HI(start_addr)); \
+ SET_JR_REG_LO(IRBA, job_ring, PHYS_ADDR_LO(start_addr));\
+}
+
+#define hw_set_output_ring_start_addr(job_ring, start_addr) \
+{ \
+ SET_JR_REG(ORBA, job_ring, PHYS_ADDR_HI(start_addr)); \
+ SET_JR_REG_LO(ORBA, job_ring, PHYS_ADDR_LO(start_addr));\
+}
+
+#else
+#define hw_set_input_ring_start_addr(job_ring, start_addr) \
+{ \
+ SET_JR_REG(IRBA, job_ring, 0); \
+ SET_JR_REG_LO(IRBA, job_ring, PHYS_ADDR_LO(start_addr));\
+}
+
+#define hw_set_output_ring_start_addr(job_ring, start_addr) \
+{ \
+ SET_JR_REG(ORBA, job_ring, 0); \
+ SET_JR_REG_LO(ORBA, job_ring, PHYS_ADDR_LO(start_addr));\
+}
+#endif
+
+/* ORJR - Output Ring Jobs Removed Register shows how many jobs were
+ * removed from the Output Ring for processing by software. This is done after
+ * the software has processed the entries.
+ */
+#define hw_remove_entries(jr, no_entries) SET_JR_REG(ORJR, (jr), (no_entries))
+
+/* IRSA - Input Ring Slots Available register holds the number of entries in
+ * the Job Ring's input ring. Once a job is enqueued, the value returned is
+ * decremented by the hardware by the number of jobs enqueued.
+ */
+#define hw_get_available_slots(jr) GET_JR_REG(IRSA, jr)
+
+/* ORSFR - Output Ring Slots Full register holds the number of jobs which were
+ * processed by the SEC and can be retrieved by the software. Once a job has
+ * been processed by software, the user will call hw_remove_one_entry in order
+ * to notify the SEC that the entry was processed.
+ */
+#define hw_get_no_finished_jobs(jr) GET_JR_REG(ORSFR, jr)
+
+/*
+ * Macros for manipulating JR registers
+ */
+#if CORE_BYTE_ORDER == CAAM_BYTE_ORDER
+#define sec_read_32(addr) (*(volatile unsigned int *)(addr))
+#define sec_write_32(addr, val) (*(volatile unsigned int *)(addr) = (val))
+
+#else
+#define sec_read_32(addr) rte_bswap32((*(volatile unsigned int *)(addr)))
+#define sec_write_32(addr, val) \
+ (*(volatile unsigned int *)(addr) = rte_bswap32(val))
+#endif
+
+#if CAAM_BYTE_ORDER == __LITTLE_ENDIAN
+#define sec_read_64(addr) (((u64)sec_read_32((u32 *)(addr) + 1) << 32) | \
+ (sec_read_32((u32 *)(addr))))
+
+#define sec_write_64(addr, val) { \
+ sec_write_32((u32 *)(addr) + 1, (u32)((val) >> 32)); \
+ sec_write_32((u32 *)(addr), (u32)(val)); \
+}
+#else /* CAAM_BYTE_ORDER == __BIG_ENDIAN */
+#define sec_read_64(addr) (((u64)sec_read_32((u32 *)(addr)) << 32) | \
+ (sec_read_32((u32 *)(addr) + 1)))
+
+#define sec_write_64(addr, val) { \
+ sec_write_32((u32 *)(addr), (u32)((val) >> 32)); \
+ sec_write_32((u32 *)(addr) + 1, (u32)(val)); \
+}
+#endif
+
+#if defined(RTE_ARCH_ARM64)
+#define sec_read_addr(a) sec_read_64((a))
+#define sec_write_addr(a, v) sec_write_64((a), (v))
+#else
+#define sec_read_addr(a) sec_read_32((a))
+#define sec_write_addr(a, v) sec_write_32((a), (v))
+#endif
+
+#define JR_REG(name, jr) (CHAN_BASE(jr) + JR_REG_##name##_OFFSET)
+#define JR_REG_LO(name, jr) (CHAN_BASE(jr) + JR_REG_##name##_OFFSET_LO)
+
+#define GET_JR_REG(name, jr) (sec_read_32(JR_REG(name, (jr))))
+#define GET_JR_REG_LO(name, jr) (sec_read_32(JR_REG_LO(name, (jr))))
+
+#define SET_JR_REG(name, jr, value) \
+ (sec_write_32(JR_REG(name, (jr)), value))
+#define SET_JR_REG_LO(name, jr, value) \
+ (sec_write_32(JR_REG_LO(name, (jr)), value))
+
+/* Lists the possible states for a job ring. */
+typedef enum sec_job_ring_state_e {
+ SEC_JOB_RING_STATE_STARTED, /* Job ring is initialized */
+ SEC_JOB_RING_STATE_RESET, /* Job ring reset is in progress */
+} sec_job_ring_state_t;
+
+/* code or cmd block to caam */
+struct sec_cdb {
+ struct {
+ union {
+ uint32_t word;
+ struct {
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+ uint16_t rsvd63_48;
+ unsigned int rsvd47_39:9;
+ unsigned int idlen:7;
+#else
+ unsigned int idlen:7;
+ unsigned int rsvd47_39:9;
+ uint16_t rsvd63_48;
+#endif
+ } field;
+ } __rte_packed hi;
+
+ union {
+ uint32_t word;
+ struct {
+#if RTE_BYTE_ORDER == RTE_BIG_ENDIAN
+ unsigned int rsvd31_30:2;
+ unsigned int fsgt:1;
+ unsigned int lng:1;
+ unsigned int offset:2;
+ unsigned int abs:1;
+ unsigned int add_buf:1;
+ uint8_t pool_id;
+ uint16_t pool_buffer_size;
+#else
+ uint16_t pool_buffer_size;
+ uint8_t pool_id;
+ unsigned int add_buf:1;
+ unsigned int abs:1;
+ unsigned int offset:2;
+ unsigned int lng:1;
+ unsigned int fsgt:1;
+ unsigned int rsvd31_30:2;
+#endif
+ } field;
+ } __rte_packed lo;
+ } __rte_packed sh_hdr;
+
+ uint32_t sh_desc[SEC_JOB_DESCRIPTOR_SIZE];
+};
+
+struct caam_jr_qp {
+ struct sec_job_ring_t *ring;
+ uint64_t rx_pkts;
+ uint64_t rx_errs;
+ uint64_t rx_poll_err;
+ uint64_t tx_pkts;
+ uint64_t tx_errs;
+ uint64_t tx_ring_full;
+};
+
+struct sec_job_ring_t {
+ /* TODO: Add wrapper macro to make it obvious this is the consumer index
+ * on the output ring
+ */
+ uint32_t cidx; /* Consumer index for job ring (jobs array).
+ * @note: cidx and pidx are accessed from
+ * different threads. Place the cidx and pidx
+ * inside the structure so that they lay on
+ * different cachelines, to avoid false sharing
+ * between threads when the threads run on
+ * different cores!
+ */
+ /* TODO: Add wrapper macro to make it obvious this is the producer index
+ * on the input ring
+ */
+ uint32_t pidx; /* Producer index for job ring (jobs array) */
+
+ phys_addr_t *input_ring;/* Ring of output descriptors received from SEC.
+ * Size of array is power of 2 to allow fast
+ * update of producer/consumer indexes with
+ * bitwise operations.
+ */
+
+ struct sec_outring_entry *output_ring;
+ /* Ring of output descriptors received from SEC.
+ * Size of array is power of 2 to allow fast
+ * update of producer/consumer indexes with
+ * bitwise operations.
+ */
+
+ uint32_t irq_fd; /* The file descriptor used for polling from
+ * user space for interrupts notifications
+ */
+ uint32_t jr_mode; /* Model used by SEC Driver to receive
+ * notifications from SEC. Can be either
+ * of the three: #SEC_NOTIFICATION_TYPE_NAPI
+ * #SEC_NOTIFICATION_TYPE_IRQ or
+ * #SEC_NOTIFICATION_TYPE_POLL
+ */
+ uint32_t napi_mode; /* Job ring mode if NAPI mode is chosen
+ * Used only when jr_mode is set to
+ * #SEC_NOTIFICATION_TYPE_NAPI
+ */
+ void *register_base_addr; /* Base address for SEC's
+ * register memory for this job ring.
+ */
+ uint8_t coalescing_en; /* notifies if coelescing is
+ * enabled for the job ring
+ */
+ sec_job_ring_state_t jr_state; /* The state of this job ring */
+
+ struct rte_mempool *ctx_pool; /* per dev mempool for caam_jr_op_ctx */
+ unsigned int max_nb_queue_pairs;
+ unsigned int max_nb_sessions;
+ struct caam_jr_qp qps[RTE_CAAM_MAX_NB_SEC_QPS]; /* i/o queue for sec */
+};
+
+/* Union describing the possible error codes that
+ * can be set in the descriptor status word
+ */
+union hw_error_code {
+ uint32_t error;
+ union {
+ struct {
+ uint32_t ssrc:4;
+ uint32_t ssed_val:28;
+ } __rte_packed value;
+ struct {
+ uint32_t ssrc:4;
+ uint32_t res:28;
+ } __rte_packed no_status_src;
+ struct {
+ uint32_t ssrc:4;
+ uint32_t jmp:1;
+ uint32_t res:11;
+ uint32_t desc_idx:8;
+ uint32_t cha_id:4;
+ uint32_t err_id:4;
+ } __rte_packed ccb_status_src;
+ struct {
+ uint32_t ssrc:4;
+ uint32_t jmp:1;
+ uint32_t res:11;
+ uint32_t desc_idx:8;
+ uint32_t offset:8;
+ } __rte_packed jmp_halt_user_src;
+ struct {
+ uint32_t ssrc:4;
+ uint32_t jmp:1;
+ uint32_t res:11;
+ uint32_t desc_idx:8;
+ uint32_t desc_err:8;
+ } __rte_packed deco_src;
+ struct {
+ uint32_t ssrc:4;
+ uint32_t res:17;
+ uint32_t naddr:3;
+ uint32_t desc_err:8;
+ } __rte_packed jr_src;
+ struct {
+ uint32_t ssrc:4;
+ uint32_t jmp:1;
+ uint32_t res:11;
+ uint32_t desc_idx:8;
+ uint32_t cond:8;
+ } __rte_packed jmp_halt_cond_src;
+ } __rte_packed error_desc;
+} __rte_packed;
+
+/* @brief Initialize a job ring/channel in SEC device.
+ * Write configuration register/s to properly initialize a job ring.
+ *
+ * @param [in] job_ring The job ring
+ *
+ * @retval 0 for success
+ * @retval other for error
+ */
+int hw_reset_job_ring(struct sec_job_ring_t *job_ring);
+
+/* @brief Reset a job ring/channel in SEC device.
+ * Write configuration register/s to reset a job ring.
+ *
+ * @param [in] job_ring The job ring
+ *
+ * @retval 0 for success
+ * @retval -1 in case job ring reset failed
+ */
+int hw_shutdown_job_ring(struct sec_job_ring_t *job_ring);
+
+/* @brief Handle a job ring/channel error in SEC device.
+ * Identify the error type and clear error bits if required.
+ *
+ * @param [in] job_ring The job ring
+ * @param [in] sec_error_code The job ring's error code
+ */
+void hw_handle_job_ring_error(struct sec_job_ring_t *job_ring,
+ uint32_t sec_error_code);
+
+/* @brief Handle a job ring error in the device.
+ * Identify the error type and printout a explanatory
+ * messages.
+ *
+ * @param [in] job_ring The job ring
+ *
+ */
+void hw_job_ring_error_print(struct sec_job_ring_t *job_ring, int code);
+
+/* @brief Set interrupt coalescing parameters on the Job Ring.
+ * @param [in] job_ring The job ring
+ * @param [in] irq_coalesing_timer Interrupt coalescing timer threshold.
+ * This value determines the maximum
+ * amount of time after processing a
+ * descriptor before raising an interrupt.
+ * @param [in] irq_coalescing_count Interrupt coalescing descriptor count
+ * threshold.
+ */
+int hw_job_ring_set_coalescing_param(struct sec_job_ring_t *job_ring,
+ uint16_t irq_coalescing_timer,
+ uint8_t irq_coalescing_count);
+
+/* @brief Enable interrupt coalescing on a job ring
+ * @param [in] job_ring The job ring
+ */
+int hw_job_ring_enable_coalescing(struct sec_job_ring_t *job_ring);
+
+/* @brief Disable interrupt coalescing on a job ring
+ * @param [in] job_ring The job ring
+ */
+int hw_job_ring_disable_coalescing(struct sec_job_ring_t *job_ring);
+
+#endif /* CAAM_JR_HW_SPECIFIC_H */
new file mode 100644
@@ -0,0 +1,285 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017-2018 NXP
+ */
+
+#ifndef CAAM_JR_PVT_H
+#define CAAM_JR_PVT_H
+
+/* NXP CAAM JR PMD device name */
+
+#define CAAM_JR_ALG_UNSUPPORT (-1)
+
+/* Minimum job descriptor consists of a oneword job descriptor HEADER and
+ * a pointer to the shared descriptor.
+ */
+#define MIN_JOB_DESC_SIZE (CAAM_CMD_SZ + CAAM_PTR_SZ)
+#define CAAM_JOB_DESC_SIZE 13
+
+/* CTX_POOL_NUM_BUFS is set as per the ipsec-secgw application */
+#define CTX_POOL_NUM_BUFS 32000
+#define CTX_POOL_CACHE_SIZE 512
+
+#define DIR_ENC 1
+#define DIR_DEC 0
+
+#define JR_MAX_NB_MAX_DIGEST 32
+
+#define RTE_CAAM_JR_PMD_MAX_NB_SESSIONS 2048
+
+
+/* Return codes for SEC user space driver APIs */
+enum sec_return_code_e {
+ SEC_SUCCESS = 0, /* Operation executed successfully.*/
+ SEC_INVALID_INPUT_PARAM, /* API received an invalid input
+ * parameter
+ */
+ SEC_OUT_OF_MEMORY, /* Memory allocation failed. */
+ SEC_DESCRIPTOR_IN_FLIGHT, /* API function indicates there are
+ * descriptors in flight
+ * for SEC to process.
+ */
+ SEC_LAST_DESCRIPTOR_IN_FLIGHT, /* API function indicates there is one
+ * last descriptor in flight
+ * for SEC to process that.
+ */
+ SEC_PROCESSING_ERROR, /* Indicates a SEC processing error
+ * occurred on a Job Ring which requires
+ * a SEC user space driver shutdown. Can
+ * be returned from sec_poll_job_ring().
+ * Then the only other API that can be
+ * called after this error is
+ * sec_release().
+ */
+ SEC_DESC_PROCESSING_ERROR, /* Indicates a SEC descriptor processing
+ * error occurred on a Job Ring. Can be
+ * returned from sec_poll_job_ring().
+ * The driver was able to reset job ring
+ * and job ring can be used like in a
+ * normal case.
+ */
+ SEC_JR_IS_FULL, /* Job Ring is full. There is no more
+ * room in the JR for new descriptors.
+ * This can happen if the descriptor RX
+ * rate is higher than SEC's capacity.
+ */
+ SEC_DRIVER_RELEASE_IN_PROGRESS, /* SEC driver shutdown is in progress,
+ * descriptors processing or polling is
+ * allowed.
+ */
+ SEC_DRIVER_ALREADY_INITIALIZED, /* SEC driver is already initialized.*/
+ SEC_DRIVER_NOT_INITIALIZED, /* SEC driver is NOT initialized. */
+ SEC_JOB_RING_RESET_IN_PROGRESS, /* Job ring is resetting due to a
+ * per-descriptor SEC processing error
+ * ::SEC_desc_PROCESSING_ERROR. Reset is
+ * finished when sec_poll_job_ring()
+ * return. Then the job ring can be used
+ * again.
+ */
+ SEC_RESET_ENGINE_FAILED, /* Resetting of SEC Engine by SEC Kernel
+ * Driver Failed
+ */
+ SEC_ENABLE_IRQS_FAILED, /* Enabling of IRQs in SEC Kernel Driver
+ * Failed
+ */
+ SEC_DISABLE_IRQS_FAILED, /* Disabling of IRQs in SEC Kernel
+ * Driver Failed
+ */
+ /* END OF VALID VALUES */
+
+ SEC_RETURN_CODE_MAX_VALUE, /* Invalid value for return code. It is
+ * used to mark the end of the return
+ * code values. @note ALL new return
+ * code values MUST be added before
+ * ::SEC_RETURN_CODE_MAX_VALUE!
+ */
+};
+
+enum caam_jr_op_type {
+ CAAM_JR_NONE, /* No Cipher operations*/
+ CAAM_JR_CIPHER,/* CIPHER operations */
+ CAAM_JR_AUTH, /* Authentication Operations */
+ CAAM_JR_AEAD, /* Authenticated Encryption with associated data */
+ CAAM_JR_IPSEC, /* IPSEC protocol operations*/
+ CAAM_JR_PDCP, /* PDCP protocol operations*/
+ CAAM_JR_PKC, /* Public Key Cryptographic Operations */
+ CAAM_JR_MAX
+};
+
+struct caam_jr_session {
+ uint8_t dir; /* Operation Direction */
+ enum rte_crypto_cipher_algorithm cipher_alg; /* Cipher Algorithm*/
+ enum rte_crypto_auth_algorithm auth_alg; /* Authentication Algorithm*/
+ enum rte_crypto_aead_algorithm aead_alg; /* AEAD Algorithm*/
+ union {
+ struct {
+ uint8_t *data; /* pointer to key data */
+ size_t length; /* key length in bytes */
+ } aead_key;
+ struct {
+ struct {
+ uint8_t *data; /* pointer to key data */
+ size_t length; /* key length in bytes */
+ } cipher_key;
+ struct {
+ uint8_t *data; /* pointer to key data */
+ size_t length; /* key length in bytes */
+ } auth_key;
+ };
+ };
+ struct {
+ uint16_t length;
+ uint16_t offset;
+ } iv; /* Initialisation vector parameters */
+ uint16_t auth_only_len; /* Length of data for Auth only */
+ uint32_t digest_length;
+ struct ip ip4_hdr;
+ struct caam_jr_qp *qp;
+ struct sec_cdb *cdb; /* cmd block associated with qp */
+ struct rte_mempool *ctx_pool; /* session mempool for caam_jr_op_ctx */
+};
+
+/*
+ * 16-byte hardware scatter/gather table
+ */
+
+#define SEC4_SG_LEN_EXT 0x80000000 /* Entry points to table */
+#define SEC4_SG_LEN_FIN 0x40000000 /* Last ent in table */
+#define SEC4_SG_BPID_MASK 0x000000ff
+#define SEC4_SG_BPID_SHIFT 16
+#define SEC4_SG_LEN_MASK 0x3fffffff /* Excludes EXT and FINAL */
+#define SEC4_SG_OFFSET_MASK 0x00001fff
+
+struct sec4_sg_entry {
+ uint64_t ptr;
+ uint32_t len;
+ uint32_t bpid_offset;
+};
+
+#define MAX_SG_ENTRIES 16
+#define SG_CACHELINE_0 0
+#define SG_CACHELINE_1 4
+#define SG_CACHELINE_2 8
+#define SG_CACHELINE_3 12
+
+/* Structure encompassing a job descriptor which is to be processed
+ * by SEC. User should also initialise this structure with the callback
+ * function pointer which will be called by driver after recieving proccessed
+ * descriptor from SEC. User data is also passed in this data structure which
+ * will be sent as an argument to the user callback function.
+ */
+struct job_descriptor {
+ uint32_t desc[CAAM_JOB_DESC_SIZE];
+};
+
+struct caam_jr_op_ctx {
+ struct job_descriptor jobdes;
+ /* sg[0] output, sg[1] input, others are possible sub frames */
+ struct sec4_sg_entry sg[MAX_SG_ENTRIES];
+ struct rte_crypto_op *op;
+ struct rte_mempool *ctx_pool; /* mempool pointer for caam_jr_op_ctx */
+ int64_t vtop_offset;
+ uint8_t digest[JR_MAX_NB_MAX_DIGEST];
+};
+
+/**
+ * Checksum
+ *
+ * @param buffer calculate chksum for buffer
+ * @param len buffer length
+ *
+ * @return checksum value in host cpu order
+ */
+static inline uint16_t
+calc_chksum(void *buffer, int len)
+{
+ uint16_t *buf = (uint16_t *)buffer;
+ uint32_t sum = 0;
+ uint16_t result;
+
+ for (sum = 0; len > 1; len -= 2)
+ sum += *buf++;
+
+ if (len == 1)
+ sum += *(unsigned char *)buf;
+
+ sum = (sum >> 16) + (sum & 0xFFFF);
+ sum += (sum >> 16);
+ result = ~sum;
+
+ return result;
+}
+struct uio_job_ring {
+ uint32_t jr_id;
+ uint32_t uio_fd;
+ void *register_base_addr;
+ int map_size;
+ int uio_minor_number;
+};
+
+int sec_cleanup(void);
+int sec_configure(void);
+struct uio_job_ring *config_job_ring(void);
+void free_job_ring(uint32_t uio_fd);
+
+/* For Dma memory allocation of specified length and alignment */
+static inline void *caam_jr_dma_mem_alloc(size_t align, size_t len)
+{
+ return rte_malloc("mem_alloc", len, align);
+}
+
+/* For freeing dma memory */
+static inline void caam_jr_dma_free(void *ptr)
+{
+ rte_free(ptr);
+}
+
+static inline rte_iova_t
+caam_jr_mem_vtop(void *vaddr)
+{
+ const struct rte_memseg *ms;
+
+ ms = rte_mem_virt2memseg(vaddr, NULL);
+ if (ms)
+ return ms->iova + RTE_PTR_DIFF(vaddr, ms->addr);
+ return (size_t)NULL;
+}
+
+static inline void *
+caam_jr_dma_ptov(rte_iova_t paddr)
+{
+ return rte_mem_iova2virt(paddr);
+}
+
+/* Virtual to physical address conversion */
+static inline rte_iova_t caam_jr_dma_vtop(void *ptr)
+{
+ //return rte_malloc_virt2iova(ptr);
+ return caam_jr_mem_vtop(ptr);
+}
+
+/** @brief Request to SEC kernel driver to enable interrupts for
+ * descriptor finished processing
+ * Use UIO to communicate with SEC kernel driver: write command
+ * value that indicates an IRQ enable action into UIO file descriptor
+ * of this job ring.
+ *
+ * @param [in] uio_fd Job Ring UIO File descriptor
+ * @retval 0 for success
+ * @retval -1 value for error
+ */
+uint32_t caam_jr_enable_irqs(uint32_t uio_fd);
+
+/** @brief Request to SEC kernel driver to disable interrupts for descriptor
+ * finished processing
+ * Use UIO to communicate with SEC kernel driver: write command
+ * value that indicates an IRQ disable action into UIO file descriptor
+ * of this job ring.
+ *
+ * @param [in] uio_fd UIO File descripto
+ * @retval 0 for success
+ * @retval -1 value for error
+ *
+ */
+uint32_t caam_jr_disable_irqs(uint32_t uio_fd);
+
+#endif
new file mode 100644
@@ -0,0 +1,491 @@
+/* SPDX-License-Identifier: BSD-3-Clause
+ * Copyright 2017-2018 NXP
+ */
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <dirent.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <rte_common.h>
+#include <rte_malloc.h>
+#include <rte_crypto.h>
+#include <rte_security.h>
+#include <caam_jr_config.h>
+
+/* RTA header files */
+#include <hw/desc/common.h>
+#include <hw/desc/algo.h>
+#include <hw/desc/ipsec.h>
+
+#include <caam_jr_hw_specific.h>
+#include <caam_jr_pvt.h>
+#include <caam_jr_log.h>
+
+/* Prefix path to sysfs directory where UIO device attributes are exported.
+ * Path for UIO device X is /sys/class/uio/uioX
+ */
+#define SEC_UIO_DEVICE_SYS_ATTR_PATH "/sys/class/uio"
+
+/* Subfolder in sysfs where mapping attributes are exported
+ * for each UIO device. Path for mapping Y for device X is:
+ * /sys/class/uio/uioX/maps/mapY
+ */
+#define SEC_UIO_DEVICE_SYS_MAP_ATTR "maps/map"
+
+/* Name of UIO device file prefix. Each UIO device will have a device file
+ * /dev/uioX, where X is the minor device number.
+ */
+#define SEC_UIO_DEVICE_FILE_NAME "/dev/uio"
+
+/*
+ * Name of UIO device. Each user space SEC job ring will have a corresponding
+ * UIO device with the name sec-channelX, where X is the job ring id.
+ * Maximum length is #SEC_UIO_MAX_DEVICE_NAME_LENGTH.
+ *
+ * @note Must be kept in synch with SEC kernel driver
+ * define #SEC_UIO_DEVICE_NAME !
+ */
+#define SEC_UIO_DEVICE_NAME "fsl-jr"
+
+/* Maximum length for the name of an UIO device file.
+ * Device file name format is: /dev/uioX.
+ */
+#define SEC_UIO_MAX_DEVICE_FILE_NAME_LENGTH 30
+
+/* Maximum length for the name of an attribute file for an UIO device.
+ * Attribute files are exported in sysfs and have the name formatted as:
+ * /sys/class/uio/uioX/<attribute_file_name>
+ */
+#define SEC_UIO_MAX_ATTR_FILE_NAME 100
+
+/* Command that is used by SEC user space driver and SEC kernel driver
+ * to signal a request from the former to the later to disable job DONE
+ * and error IRQs on a certain job ring.
+ * The configuration is done at SEC Controller's level.
+ * @note Need to be kept in synch with #SEC_UIO_DISABLE_IRQ_CMD from
+ * linux/drivers/crypto/talitos.c !
+ */
+#define SEC_UIO_DISABLE_IRQ_CMD 0
+
+/* Command that is used by SEC user space driver and SEC kernel driver
+ * to signal a request from the former to the later to enable job DONE
+ * and error IRQs on a certain job ring.
+ * The configuration is done at SEC Controller's level.
+ * @note Need to be kept in synch with #SEC_UIO_ENABLE_IRQ_CMD from
+ * linux/drivers/crypto/talitos.c !
+ */
+#define SEC_UIO_ENABLE_IRQ_CMD 1
+
+/** Command that is used by SEC user space driver and SEC kernel driver
+ * to signal a request from the former to the later to do a SEC engine reset.
+ * @note Need to be kept in synch with #SEC_UIO_RESET_SEC_ENGINE_CMD from
+ * linux/drivers/crypto/talitos.c !
+ */
+#define SEC_UIO_RESET_SEC_ENGINE_CMD 3
+
+/* The id for the mapping used to export SEC's registers to
+ * user space through UIO devices.
+ */
+#define SEC_UIO_MAP_ID 0
+
+static struct uio_job_ring g_uio_job_ring[MAX_SEC_JOB_RINGS];
+static int g_uio_jr_num;
+
+/** @brief Checks if a file name contains a certain substring.
+ * If so, it extracts the number following the substring.
+ * This function assumes a filename format of: [text][number].
+ * @param [in] filename File name
+ * @param [in] match String to match in file name
+ * @param [out] number The number extracted from filename
+ *
+ * @retval true if file name matches the criteria
+ * @retval false if file name does not match the criteria
+ */
+static bool
+file_name_match_extract(const char filename[], const char match[], int *number)
+{
+ char *substr = NULL;
+
+ substr = strstr(filename, match);
+ if (substr == NULL)
+ return false;
+
+ /* substring <match> was found in <filename>
+ * read number following <match> substring in <filename>
+ */
+ sscanf(filename + strlen(match), "%d", number);
+ return true;
+}
+
+/** @brief Reads first line from a file.
+ * Composes file name as: root/subdir/filename
+ *
+ * @param [in] root Root path
+ * @param [in] subdir Subdirectory name
+ * @param [in] filename File name
+ * @param [out] line The first line read from file.
+ *
+ * @retval 0 for succes
+ * @retval other value for error
+ */
+static int
+file_read_first_line(const char root[], const char subdir[],
+ const char filename[], char *line)
+{
+ char absolute_file_name[SEC_UIO_MAX_ATTR_FILE_NAME];
+ int fd = 0, ret = 0;
+
+ /*compose the file name: root/subdir/filename */
+ memset(absolute_file_name, 0, sizeof(absolute_file_name));
+ snprintf(absolute_file_name, SEC_UIO_MAX_ATTR_FILE_NAME,
+ "%s/%s/%s", root, subdir, filename);
+
+ fd = open(absolute_file_name, O_RDONLY);
+ SEC_ASSERT(fd > 0, fd, "Error opening file %s",
+ absolute_file_name);
+
+ /* read UIO device name from first line in file */
+ ret = read(fd, line, SEC_UIO_MAX_DEVICE_FILE_NAME_LENGTH);
+ close(fd);
+
+ /* NULL-ify string */
+ line[SEC_UIO_MAX_DEVICE_FILE_NAME_LENGTH - 1] = '\0';
+
+ if (ret <= 0) {
+ CAAM_JR_ERR("Error reading from file %s", absolute_file_name);
+ return ret;
+ }
+
+ return 0;
+}
+
+/** @brief Uses UIO control to send commands to SEC kernel driver.
+ * The mechanism is to write a command word into the file descriptor
+ * that the user-space driver obtained for each user-space SEC job ring.
+ * Both user-space driver and kernel driver must have the same understanding
+ * about the command codes.
+ *
+ * @param [in] UIO FD The UIO file descriptor
+ * @param [in] uio_command Command word
+ *
+ * @retval Result of write operation on the job ring's UIO file descriptor.
+ * Should be sizeof(int) for success operations.
+ * Other values can be returned and used, if desired to add special
+ * meaning to return values, but this has to be programmed in SEC
+ * kernel driver as well. No special return values are used.
+ */
+static int sec_uio_send_command(uint32_t uio_fd, int32_t uio_command)
+{
+ int ret;
+
+ /* Use UIO file descriptor we have for this job ring.
+ * Writing a command code to this file descriptor will make the
+ * SEC kernel driver execute the desired command.
+ */
+ ret = write(uio_fd, &uio_command, sizeof(int));
+ return ret;
+}
+
+/** @brief Request to SEC kernel driver to enable interrupts for
+ * descriptor finished processing
+ * Use UIO to communicate with SEC kernel driver: write command
+ * value that indicates an IRQ enable action into UIO file descriptor
+ * of this job ring.
+ *
+ * @param [in] uio_fd Job Ring UIO File descriptor
+ * @retval 0 for success
+ * @retval -1 value for error
+ */
+uint32_t caam_jr_enable_irqs(uint32_t uio_fd)
+{
+ int ret;
+
+ /* Use UIO file descriptor we have for this job ring.
+ * Writing a command code to this file descriptor will make the
+ * SEC kernel driver enable DONE and Error IRQs for this job ring,
+ * at Controller level.
+ */
+ ret = sec_uio_send_command(uio_fd, SEC_UIO_ENABLE_IRQ_CMD);
+ SEC_ASSERT(ret == sizeof(int), -1,
+ "Failed to request SEC engine to enable job done and "
+ "error IRQs through UIO control. UIO FD %d. Reset SEC driver!",
+ uio_fd);
+ CAAM_JR_DEBUG("Enabled IRQs on jr with uio_fd %d", uio_fd);
+ return 0;
+}
+
+
+/** @brief Request to SEC kernel driver to disable interrupts for descriptor
+ * finished processing
+ * Use UIO to communicate with SEC kernel driver: write command
+ * value that indicates an IRQ disable action into UIO file descriptor
+ * of this job ring.
+ *
+ * @param [in] uio_fd UIO File descripto
+ * @retval 0 for success
+ * @retval -1 value for error
+ *
+ */
+uint32_t caam_jr_disable_irqs(uint32_t uio_fd)
+{
+ int ret;
+
+ /* Use UIO file descriptor we have for this job ring.
+ * Writing a command code to this file descriptor will make the
+ * SEC kernel driver disable IRQs for this job ring,
+ * at Controller level.
+ */
+
+ ret = sec_uio_send_command(uio_fd, SEC_UIO_DISABLE_IRQ_CMD);
+ SEC_ASSERT(ret == sizeof(int), -1,
+ "Failed to request SEC engine to disable job done and "
+ "IRQs through UIO control. UIO_FD %d Reset SEC driver!",
+ uio_fd);
+ CAAM_JR_DEBUG("Disabled IRQs on jr with uio_fd %d", uio_fd);
+ return 0;
+}
+
+/** @brief Maps register range assigned for a job ring.
+ *
+ * @param [in] uio_device_fd UIO device file descriptor
+ * @param [in] uio_device_id UIO device id
+ * @param [in] uio_map_id UIO allows maximum 5 different mapping for
+ each device. Maps start with id 0.
+ * @param [out] map_size Map size.
+ * @retval NULL if failed to map registers
+ * @retval Virtual address for mapped register address range
+ */
+static void *
+uio_map_registers(int uio_device_fd, int uio_device_id,
+ int uio_map_id, int *map_size)
+{
+ void *mapped_address = NULL;
+ unsigned int uio_map_size = 0;
+ char uio_sys_root[SEC_UIO_MAX_ATTR_FILE_NAME];
+ char uio_sys_map_subdir[SEC_UIO_MAX_ATTR_FILE_NAME];
+ char uio_map_size_str[32];
+ int ret = 0;
+
+ /* compose the file name: root/subdir/filename */
+ memset(uio_sys_root, 0, sizeof(uio_sys_root));
+ memset(uio_sys_map_subdir, 0, sizeof(uio_sys_map_subdir));
+ memset(uio_map_size_str, 0, sizeof(uio_map_size_str));
+
+ /* Compose string: /sys/class/uio/uioX */
+ sprintf(uio_sys_root, "%s/%s%d", SEC_UIO_DEVICE_SYS_ATTR_PATH,
+ "uio", uio_device_id);
+ /* Compose string: maps/mapY */
+ sprintf(uio_sys_map_subdir, "%s%d", SEC_UIO_DEVICE_SYS_MAP_ATTR,
+ uio_map_id);
+
+ /* Read first (and only) line from file
+ * /sys/class/uio/uioX/maps/mapY/size
+ */
+ ret = file_read_first_line(uio_sys_root, uio_sys_map_subdir,
+ "size", uio_map_size_str);
+ SEC_ASSERT(ret == 0, NULL, "file_read_first_line() failed");
+
+ /* Read mapping size, expressed in hexa(base 16) */
+ uio_map_size = strtol(uio_map_size_str, NULL, 16);
+
+ /* Map the region in user space */
+ mapped_address = mmap(0, /*dynamically choose virtual address */
+ uio_map_size, PROT_READ | PROT_WRITE,
+ MAP_SHARED, uio_device_fd, 0);
+ /* offset = 0 because UIO device has only one mapping
+ * for the entire SEC register memory
+ */
+ if (mapped_address == MAP_FAILED) {
+ CAAM_JR_ERR(
+ "Failed to map registers! errno = %d job ring fd = %d,"
+ "uio device id = %d, uio map id = %d", errno,
+ uio_device_fd, uio_device_id, uio_map_id);
+ return NULL;
+ }
+
+ /*
+ * Save the map size to use it later on for munmap-ing.
+ */
+ *map_size = uio_map_size;
+
+ CAAM_JR_INFO("UIO dev[%d] mapped region [id =%d] size 0x%x at %p",
+ uio_device_id, uio_map_id, uio_map_size, mapped_address);
+
+ return mapped_address;
+}
+
+void free_job_ring(uint32_t uio_fd)
+{
+ struct uio_job_ring *job_ring = NULL;
+ int i;
+
+ if (!job_ring->uio_fd)
+ return;
+
+ for (i = 0; i < MAX_SEC_JOB_RINGS; i++) {
+ if (g_uio_job_ring[i].uio_fd == uio_fd) {
+ job_ring = &g_uio_job_ring[i];
+ break;
+ }
+ }
+ if (job_ring == NULL) {
+ CAAM_JR_ERR("JR not available for fd = %x\n", uio_fd);
+ return;
+ }
+
+ /* Open device file */
+ CAAM_JR_INFO("Closed device file for job ring %d , fd = %d",
+ job_ring->jr_id, job_ring->uio_fd);
+ close(job_ring->uio_fd);
+ g_uio_jr_num--;
+ job_ring->uio_fd = 0;
+
+ if (job_ring->register_base_addr == NULL)
+ return;
+
+ /* Unmap the PCI memory resource of device */
+ if (munmap(job_ring->register_base_addr, job_ring->map_size)) {
+ CAAM_JR_INFO("cannot munmap(%p, 0x%lx): %s",
+ job_ring->register_base_addr,
+ (unsigned long)job_ring->map_size, strerror(errno));
+ } else
+ CAAM_JR_DEBUG(" JR UIO memory unmapped at %p",
+ job_ring->register_base_addr);
+ job_ring->register_base_addr = NULL;
+}
+
+struct uio_job_ring *config_job_ring(void)
+{
+ char uio_device_file_name[32];
+ struct uio_job_ring *job_ring = NULL;
+ int i;
+
+ for (i = 0; i < MAX_SEC_JOB_RINGS; i++) {
+ if (g_uio_job_ring[i].uio_fd == 0) {
+ job_ring = &g_uio_job_ring[i];
+ g_uio_jr_num++;
+ break;
+ }
+ }
+
+ if (job_ring == NULL) {
+ CAAM_JR_ERR("No free job ring\n");
+ return NULL;
+ }
+
+ /* Find UIO device created by SEC kernel driver for this job ring. */
+ memset(uio_device_file_name, 0, sizeof(uio_device_file_name));
+
+ sprintf(uio_device_file_name, "%s%d", SEC_UIO_DEVICE_FILE_NAME,
+ job_ring->uio_minor_number);
+
+ /* Open device file */
+ job_ring->uio_fd = open(uio_device_file_name, O_RDWR);
+ SEC_ASSERT(job_ring->uio_fd > 0, NULL,
+ "Failed to open UIO device file for job ring %d",
+ job_ring->jr_id);
+
+ CAAM_JR_INFO("Open device(%s) file for job ring=%d , uio_fd = %d",
+ uio_device_file_name, job_ring->jr_id, job_ring->uio_fd);
+
+ ASSERT(job_ring->register_base_addr == NULL);
+ job_ring->register_base_addr = uio_map_registers(
+ job_ring->uio_fd, job_ring->uio_minor_number,
+ SEC_UIO_MAP_ID, &job_ring->map_size);
+
+ SEC_ASSERT(job_ring->register_base_addr != NULL, NULL,
+ "Failed to map SEC registers");
+ return job_ring;
+}
+
+int sec_configure(void)
+{
+ char uio_name[32];
+ int config_jr_no = 0, jr_id = -1;
+ int uio_minor_number = -1;
+ int ret;
+ DIR *d = NULL;
+ struct dirent *dir;
+
+ d = opendir(SEC_UIO_DEVICE_SYS_ATTR_PATH);
+ if (d == NULL) {
+ printf("\nError opening directory '%s': %s\n",
+ SEC_UIO_DEVICE_SYS_ATTR_PATH, strerror(errno));
+ return -1;
+ }
+
+ /* Iterate through all subdirs */
+ while ((dir = readdir(d)) != NULL) {
+ if (!strncmp(dir->d_name, ".", 1) ||
+ !strncmp(dir->d_name, "..", 2))
+ continue;
+
+ if (file_name_match_extract
+ (dir->d_name, "uio", &uio_minor_number)) {
+ /*
+ * Open file uioX/name and read first line which contains
+ * the name for the device. Based on the name check if this
+ * UIO device is UIO device for job ring with id jr_id.
+ */
+ memset(uio_name, 0, sizeof(uio_name));
+ ret = file_read_first_line(SEC_UIO_DEVICE_SYS_ATTR_PATH,
+ dir->d_name, "name", uio_name);
+ CAAM_JR_INFO("sec device uio name: %s", uio_name);
+ SEC_ASSERT(ret == 0, -1, "file_read_first_line failed");
+
+ if (file_name_match_extract(uio_name,
+ SEC_UIO_DEVICE_NAME,
+ &jr_id)) {
+ g_uio_job_ring[config_jr_no].jr_id = jr_id;
+ g_uio_job_ring[config_jr_no].uio_minor_number =
+ uio_minor_number;
+ CAAM_JR_INFO("Detected logical JRID:%d", jr_id);
+ config_jr_no++;
+
+ /* todo find the actual ring id
+ * OF_FULLNAME=/soc/crypto@1700000/jr@20000
+ */
+ }
+ }
+ }
+ closedir(d);
+
+ if (config_jr_no == 0) {
+ CAAM_JR_ERR("! No SEC Job Rings assigned for userspace usage!");
+ return 0;
+ }
+ CAAM_JR_INFO("Total JR detected =%d", config_jr_no);
+ return config_jr_no;
+}
+
+int sec_cleanup(void)
+{
+ int i;
+ struct uio_job_ring *job_ring;
+
+ for (i = 0; i < g_uio_jr_num; i++) {
+ job_ring = &g_uio_job_ring[i];
+ /* munmap SEC's register memory */
+ if (job_ring->register_base_addr) {
+ munmap(job_ring->register_base_addr,
+ job_ring->map_size);
+ job_ring->register_base_addr = NULL;
+ }
+ /* I need to close the fd after shutdown UIO commands need to be
+ * sent using the fd
+ */
+ if (job_ring->uio_fd != 0) {
+ CAAM_JR_INFO(
+ "Closed device file for job ring %d , fd = %d",
+ job_ring->jr_id, job_ring->uio_fd);
+ close(job_ring->uio_fd);
+ }
+ }
+ return 0;
+}
@@ -6,6 +6,9 @@ if host_machine.system() != 'linux'
endif
deps += ['bus_vdev', 'bus_dpaa', 'security']
-sources = files('caam_jr.c')
+sources = files('caam_jr_hw.c', 'caam_jr_uio.c', 'caam_jr.c')
allow_experimental_apis = true
+
+includes += include_directories('../dpaa2_sec/')
+includes += include_directories('../../bus/dpaa/include/')