diff options
Diffstat (limited to 'target/linux/layerscape/patches-4.9/804-crypto-support-layerscape.patch')
| -rw-r--r-- | target/linux/layerscape/patches-4.9/804-crypto-support-layerscape.patch | 29223 |
1 files changed, 0 insertions, 29223 deletions
diff --git a/target/linux/layerscape/patches-4.9/804-crypto-support-layerscape.patch b/target/linux/layerscape/patches-4.9/804-crypto-support-layerscape.patch deleted file mode 100644 index 714fba3a7e..0000000000 --- a/target/linux/layerscape/patches-4.9/804-crypto-support-layerscape.patch +++ /dev/null @@ -1,29223 +0,0 @@ -From 2a0aa9bd187f6f5693982a8f79665585af772237 Mon Sep 17 00:00:00 2001 -From: Yangbo Lu <yangbo.lu@nxp.com> -Date: Thu, 5 Jul 2018 17:29:41 +0800 -Subject: [PATCH 16/32] crypto: support layerscape -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - -This is an integrated patch for layerscape sec support. - -Signed-off-by: Radu Alexe <radu.alexe@nxp.com> -Signed-off-by: Fabio Estevam <festevam@gmail.com> -Signed-off-by: Arnd Bergmann <arnd@arndb.de> -Signed-off-by: Radu Alexe <radu.alexe@nxp.com> -Signed-off-by: Tudor Ambarus <tudor-dan.ambarus@nxp.com> -Signed-off-by: Eric Biggers <ebiggers@google.com> -Signed-off-by: Giovanni Cabiddu <giovanni.cabiddu@intel.com> -Signed-off-by: Xulin Sun <xulin.sun@windriver.com> -Signed-off-by: Ard Biesheuvel <ard.biesheuvel@linaro.org> -Signed-off-by: Marcus Folkesson <marcus.folkesson@gmail.com> -Signed-off-by: Tudor Ambarus <tudor-dan.ambarus@nxp.com> -Signed-off-by: Andrew Lutomirski <luto@kernel.org> -Signed-off-by: Wei Yongjun <weiyongjun1@huawei.com> -Signed-off-by: Masahiro Yamada <yamada.masahiro@socionext.com> -Signed-off-by: Marcelo Cerri <marcelo.cerri@canonical.com> -Signed-off-by: Arvind Yadav <arvind.yadav.cs@gmail.com> -Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au> -Signed-off-by: Laura Abbott <labbott@redhat.com> -Signed-off-by: Horia Geantă <horia.geanta@nxp.com> -Singed-off-by: Yangbo Lu <yangbo.lu@nxp.com> ---- - crypto/Kconfig | 30 + - crypto/Makefile | 4 + - crypto/acompress.c | 169 + - crypto/algboss.c | 12 +- - crypto/crypto_user.c | 19 + - crypto/scompress.c | 356 + - crypto/tcrypt.c | 17 +- - crypto/testmgr.c | 1708 ++--- - crypto/testmgr.h | 1125 ++-- - crypto/tls.c | 607 ++ - drivers/crypto/caam/Kconfig | 77 +- - drivers/crypto/caam/Makefile | 16 +- - drivers/crypto/caam/caamalg.c | 2185 ++---- - drivers/crypto/caam/caamalg_desc.c | 1961 ++++++ - drivers/crypto/caam/caamalg_desc.h | 127 + - drivers/crypto/caam/caamalg_qi.c | 3321 +++++++++ - drivers/crypto/caam/caamalg_qi2.c | 5938 +++++++++++++++++ - drivers/crypto/caam/caamalg_qi2.h | 283 + - drivers/crypto/caam/caamhash.c | 555 +- - drivers/crypto/caam/caamhash_desc.c | 108 + - drivers/crypto/caam/caamhash_desc.h | 49 + - drivers/crypto/caam/caampkc.c | 471 +- - drivers/crypto/caam/caampkc.h | 58 + - drivers/crypto/caam/caamrng.c | 16 +- - drivers/crypto/caam/compat.h | 1 + - drivers/crypto/caam/ctrl.c | 358 +- - drivers/crypto/caam/ctrl.h | 2 + - drivers/crypto/caam/desc.h | 84 +- - drivers/crypto/caam/desc_constr.h | 180 +- - drivers/crypto/caam/dpseci.c | 858 +++ - drivers/crypto/caam/dpseci.h | 395 ++ - drivers/crypto/caam/dpseci_cmd.h | 261 + - drivers/crypto/caam/error.c | 127 +- - drivers/crypto/caam/error.h | 10 +- - drivers/crypto/caam/intern.h | 31 +- - drivers/crypto/caam/jr.c | 72 +- - drivers/crypto/caam/jr.h | 2 + - drivers/crypto/caam/key_gen.c | 32 +- - drivers/crypto/caam/key_gen.h | 36 +- - drivers/crypto/caam/pdb.h | 62 + - drivers/crypto/caam/pkc_desc.c | 36 + - drivers/crypto/caam/qi.c | 804 +++ - drivers/crypto/caam/qi.h | 204 + - drivers/crypto/caam/regs.h | 63 +- - drivers/crypto/caam/sg_sw_qm.h | 126 + - drivers/crypto/caam/sg_sw_qm2.h | 81 + - drivers/crypto/caam/sg_sw_sec4.h | 60 +- - drivers/crypto/talitos.c | 8 + - drivers/net/wireless/rsi/rsi_91x_usb.c | 2 +- - drivers/staging/wilc1000/linux_wlan.c | 2 +- - .../staging/wilc1000/wilc_wfi_cfgoperations.c | 2 +- - include/crypto/acompress.h | 269 + - include/crypto/internal/acompress.h | 81 + - include/crypto/internal/scompress.h | 136 + - include/linux/crypto.h | 3 + - include/uapi/linux/cryptouser.h | 5 + - scripts/spelling.txt | 3 + - sound/soc/amd/acp-pcm-dma.c | 2 +- - 58 files changed, 19620 insertions(+), 3990 deletions(-) - create mode 100644 crypto/acompress.c - create mode 100644 crypto/scompress.c - create mode 100644 crypto/tls.c - create mode 100644 drivers/crypto/caam/caamalg_desc.c - create mode 100644 drivers/crypto/caam/caamalg_desc.h - create mode 100644 drivers/crypto/caam/caamalg_qi.c - create mode 100644 drivers/crypto/caam/caamalg_qi2.c - create mode 100644 drivers/crypto/caam/caamalg_qi2.h - create mode 100644 drivers/crypto/caam/caamhash_desc.c - create mode 100644 drivers/crypto/caam/caamhash_desc.h - create mode 100644 drivers/crypto/caam/dpseci.c - create mode 100644 drivers/crypto/caam/dpseci.h - create mode 100644 drivers/crypto/caam/dpseci_cmd.h - create mode 100644 drivers/crypto/caam/qi.c - create mode 100644 drivers/crypto/caam/qi.h - create mode 100644 drivers/crypto/caam/sg_sw_qm.h - create mode 100644 drivers/crypto/caam/sg_sw_qm2.h - create mode 100644 include/crypto/acompress.h - create mode 100644 include/crypto/internal/acompress.h - create mode 100644 include/crypto/internal/scompress.h - ---- a/crypto/Kconfig -+++ b/crypto/Kconfig -@@ -102,6 +102,15 @@ config CRYPTO_KPP - select CRYPTO_ALGAPI - select CRYPTO_KPP2 - -+config CRYPTO_ACOMP2 -+ tristate -+ select CRYPTO_ALGAPI2 -+ -+config CRYPTO_ACOMP -+ tristate -+ select CRYPTO_ALGAPI -+ select CRYPTO_ACOMP2 -+ - config CRYPTO_RSA - tristate "RSA algorithm" - select CRYPTO_AKCIPHER -@@ -138,6 +147,7 @@ config CRYPTO_MANAGER2 - select CRYPTO_BLKCIPHER2 if !CRYPTO_MANAGER_DISABLE_TESTS - select CRYPTO_AKCIPHER2 if !CRYPTO_MANAGER_DISABLE_TESTS - select CRYPTO_KPP2 if !CRYPTO_MANAGER_DISABLE_TESTS -+ select CRYPTO_ACOMP2 if !CRYPTO_MANAGER_DISABLE_TESTS - - config CRYPTO_USER - tristate "Userspace cryptographic algorithm configuration" -@@ -295,6 +305,26 @@ config CRYPTO_ECHAINIV - a sequence number xored with a salt. This is the default - algorithm for CBC. - -+config CRYPTO_TLS -+ tristate "TLS support" -+ select CRYPTO_AEAD -+ select CRYPTO_BLKCIPHER -+ select CRYPTO_MANAGER -+ select CRYPTO_HASH -+ select CRYPTO_NULL -+ select CRYPTO_AUTHENC -+ help -+ Support for TLS 1.0 record encryption and decryption -+ -+ This module adds support for encryption/decryption of TLS 1.0 frames -+ using blockcipher algorithms. The name of the resulting algorithm is -+ "tls10(hmac(<digest>),cbc(<cipher>))". By default, the generic base -+ algorithms are used (e.g. aes-generic, sha1-generic), but hardware -+ accelerated versions will be used automatically if available. -+ -+ User-space applications (OpenSSL, GnuTLS) can offload TLS 1.0 -+ operations through AF_ALG or cryptodev interfaces -+ - comment "Block modes" - - config CRYPTO_CBC ---- a/crypto/Makefile -+++ b/crypto/Makefile -@@ -51,6 +51,9 @@ rsa_generic-y += rsa_helper.o - rsa_generic-y += rsa-pkcs1pad.o - obj-$(CONFIG_CRYPTO_RSA) += rsa_generic.o - -+obj-$(CONFIG_CRYPTO_ACOMP2) += acompress.o -+obj-$(CONFIG_CRYPTO_ACOMP2) += scompress.o -+ - cryptomgr-y := algboss.o testmgr.o - - obj-$(CONFIG_CRYPTO_MANAGER2) += cryptomgr.o -@@ -115,6 +118,7 @@ obj-$(CONFIG_CRYPTO_CRC32C) += crc32c_ge - obj-$(CONFIG_CRYPTO_CRC32) += crc32_generic.o - obj-$(CONFIG_CRYPTO_CRCT10DIF) += crct10dif_common.o crct10dif_generic.o - obj-$(CONFIG_CRYPTO_AUTHENC) += authenc.o authencesn.o -+obj-$(CONFIG_CRYPTO_TLS) += tls.o - obj-$(CONFIG_CRYPTO_LZO) += lzo.o - obj-$(CONFIG_CRYPTO_LZ4) += lz4.o - obj-$(CONFIG_CRYPTO_LZ4HC) += lz4hc.o ---- /dev/null -+++ b/crypto/acompress.c -@@ -0,0 +1,169 @@ -+/* -+ * Asynchronous Compression operations -+ * -+ * Copyright (c) 2016, Intel Corporation -+ * Authors: Weigang Li <weigang.li@intel.com> -+ * Giovanni Cabiddu <giovanni.cabiddu@intel.com> -+ * -+ * This program is free software; you can redistribute it and/or modify it -+ * under the terms of the GNU General Public License as published by the Free -+ * Software Foundation; either version 2 of the License, or (at your option) -+ * any later version. -+ * -+ */ -+#include <linux/errno.h> -+#include <linux/kernel.h> -+#include <linux/module.h> -+#include <linux/seq_file.h> -+#include <linux/slab.h> -+#include <linux/string.h> -+#include <linux/crypto.h> -+#include <crypto/algapi.h> -+#include <linux/cryptouser.h> -+#include <net/netlink.h> -+#include <crypto/internal/acompress.h> -+#include <crypto/internal/scompress.h> -+#include "internal.h" -+ -+static const struct crypto_type crypto_acomp_type; -+ -+#ifdef CONFIG_NET -+static int crypto_acomp_report(struct sk_buff *skb, struct crypto_alg *alg) -+{ -+ struct crypto_report_acomp racomp; -+ -+ strncpy(racomp.type, "acomp", sizeof(racomp.type)); -+ -+ if (nla_put(skb, CRYPTOCFGA_REPORT_ACOMP, -+ sizeof(struct crypto_report_acomp), &racomp)) -+ goto nla_put_failure; -+ return 0; -+ -+nla_put_failure: -+ return -EMSGSIZE; -+} -+#else -+static int crypto_acomp_report(struct sk_buff *skb, struct crypto_alg *alg) -+{ -+ return -ENOSYS; -+} -+#endif -+ -+static void crypto_acomp_show(struct seq_file *m, struct crypto_alg *alg) -+ __attribute__ ((unused)); -+ -+static void crypto_acomp_show(struct seq_file *m, struct crypto_alg *alg) -+{ -+ seq_puts(m, "type : acomp\n"); -+} -+ -+static void crypto_acomp_exit_tfm(struct crypto_tfm *tfm) -+{ -+ struct crypto_acomp *acomp = __crypto_acomp_tfm(tfm); -+ struct acomp_alg *alg = crypto_acomp_alg(acomp); -+ -+ alg->exit(acomp); -+} -+ -+static int crypto_acomp_init_tfm(struct crypto_tfm *tfm) -+{ -+ struct crypto_acomp *acomp = __crypto_acomp_tfm(tfm); -+ struct acomp_alg *alg = crypto_acomp_alg(acomp); -+ -+ if (tfm->__crt_alg->cra_type != &crypto_acomp_type) -+ return crypto_init_scomp_ops_async(tfm); -+ -+ acomp->compress = alg->compress; -+ acomp->decompress = alg->decompress; -+ acomp->dst_free = alg->dst_free; -+ acomp->reqsize = alg->reqsize; -+ -+ if (alg->exit) -+ acomp->base.exit = crypto_acomp_exit_tfm; -+ -+ if (alg->init) -+ return alg->init(acomp); -+ -+ return 0; -+} -+ -+static unsigned int crypto_acomp_extsize(struct crypto_alg *alg) -+{ -+ int extsize = crypto_alg_extsize(alg); -+ -+ if (alg->cra_type != &crypto_acomp_type) -+ extsize += sizeof(struct crypto_scomp *); -+ -+ return extsize; -+} -+ -+static const struct crypto_type crypto_acomp_type = { -+ .extsize = crypto_acomp_extsize, -+ .init_tfm = crypto_acomp_init_tfm, -+#ifdef CONFIG_PROC_FS -+ .show = crypto_acomp_show, -+#endif -+ .report = crypto_acomp_report, -+ .maskclear = ~CRYPTO_ALG_TYPE_MASK, -+ .maskset = CRYPTO_ALG_TYPE_ACOMPRESS_MASK, -+ .type = CRYPTO_ALG_TYPE_ACOMPRESS, -+ .tfmsize = offsetof(struct crypto_acomp, base), -+}; -+ -+struct crypto_acomp *crypto_alloc_acomp(const char *alg_name, u32 type, -+ u32 mask) -+{ -+ return crypto_alloc_tfm(alg_name, &crypto_acomp_type, type, mask); -+} -+EXPORT_SYMBOL_GPL(crypto_alloc_acomp); -+ -+struct acomp_req *acomp_request_alloc(struct crypto_acomp *acomp) -+{ -+ struct crypto_tfm *tfm = crypto_acomp_tfm(acomp); -+ struct acomp_req *req; -+ -+ req = __acomp_request_alloc(acomp); -+ if (req && (tfm->__crt_alg->cra_type != &crypto_acomp_type)) -+ return crypto_acomp_scomp_alloc_ctx(req); -+ -+ return req; -+} -+EXPORT_SYMBOL_GPL(acomp_request_alloc); -+ -+void acomp_request_free(struct acomp_req *req) -+{ -+ struct crypto_acomp *acomp = crypto_acomp_reqtfm(req); -+ struct crypto_tfm *tfm = crypto_acomp_tfm(acomp); -+ -+ if (tfm->__crt_alg->cra_type != &crypto_acomp_type) -+ crypto_acomp_scomp_free_ctx(req); -+ -+ if (req->flags & CRYPTO_ACOMP_ALLOC_OUTPUT) { -+ acomp->dst_free(req->dst); -+ req->dst = NULL; -+ } -+ -+ __acomp_request_free(req); -+} -+EXPORT_SYMBOL_GPL(acomp_request_free); -+ -+int crypto_register_acomp(struct acomp_alg *alg) -+{ -+ struct crypto_alg *base = &alg->base; -+ -+ base->cra_type = &crypto_acomp_type; -+ base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK; -+ base->cra_flags |= CRYPTO_ALG_TYPE_ACOMPRESS; -+ -+ return crypto_register_alg(base); -+} -+EXPORT_SYMBOL_GPL(crypto_register_acomp); -+ -+int crypto_unregister_acomp(struct acomp_alg *alg) -+{ -+ return crypto_unregister_alg(&alg->base); -+} -+EXPORT_SYMBOL_GPL(crypto_unregister_acomp); -+ -+MODULE_LICENSE("GPL"); -+MODULE_DESCRIPTION("Asynchronous compression type"); ---- a/crypto/algboss.c -+++ b/crypto/algboss.c -@@ -247,17 +247,9 @@ static int cryptomgr_schedule_test(struc - memcpy(param->alg, alg->cra_name, sizeof(param->alg)); - type = alg->cra_flags; - -- /* This piece of crap needs to disappear into per-type test hooks. */ --#ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS -- type |= CRYPTO_ALG_TESTED; --#else -- if (!((type ^ CRYPTO_ALG_TYPE_BLKCIPHER) & -- CRYPTO_ALG_TYPE_BLKCIPHER_MASK) && !(type & CRYPTO_ALG_GENIV) && -- ((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == -- CRYPTO_ALG_TYPE_BLKCIPHER ? alg->cra_blkcipher.ivsize : -- alg->cra_ablkcipher.ivsize)) -+ /* Do not test internal algorithms. */ -+ if (type & CRYPTO_ALG_INTERNAL) - type |= CRYPTO_ALG_TESTED; --#endif - - param->type = type; - ---- a/crypto/crypto_user.c -+++ b/crypto/crypto_user.c -@@ -112,6 +112,21 @@ nla_put_failure: - return -EMSGSIZE; - } - -+static int crypto_report_acomp(struct sk_buff *skb, struct crypto_alg *alg) -+{ -+ struct crypto_report_acomp racomp; -+ -+ strncpy(racomp.type, "acomp", sizeof(racomp.type)); -+ -+ if (nla_put(skb, CRYPTOCFGA_REPORT_ACOMP, -+ sizeof(struct crypto_report_acomp), &racomp)) -+ goto nla_put_failure; -+ return 0; -+ -+nla_put_failure: -+ return -EMSGSIZE; -+} -+ - static int crypto_report_akcipher(struct sk_buff *skb, struct crypto_alg *alg) - { - struct crypto_report_akcipher rakcipher; -@@ -186,7 +201,11 @@ static int crypto_report_one(struct cryp - goto nla_put_failure; - - break; -+ case CRYPTO_ALG_TYPE_ACOMPRESS: -+ if (crypto_report_acomp(skb, alg)) -+ goto nla_put_failure; - -+ break; - case CRYPTO_ALG_TYPE_AKCIPHER: - if (crypto_report_akcipher(skb, alg)) - goto nla_put_failure; ---- /dev/null -+++ b/crypto/scompress.c -@@ -0,0 +1,356 @@ -+/* -+ * Synchronous Compression operations -+ * -+ * Copyright 2015 LG Electronics Inc. -+ * Copyright (c) 2016, Intel Corporation -+ * Author: Giovanni Cabiddu <giovanni.cabiddu@intel.com> -+ * -+ * This program is free software; you can redistribute it and/or modify it -+ * under the terms of the GNU General Public License as published by the Free -+ * Software Foundation; either version 2 of the License, or (at your option) -+ * any later version. -+ * -+ */ -+#include <linux/errno.h> -+#include <linux/kernel.h> -+#include <linux/module.h> -+#include <linux/seq_file.h> -+#include <linux/slab.h> -+#include <linux/string.h> -+#include <linux/crypto.h> -+#include <linux/vmalloc.h> -+#include <crypto/algapi.h> -+#include <linux/cryptouser.h> -+#include <net/netlink.h> -+#include <linux/scatterlist.h> -+#include <crypto/scatterwalk.h> -+#include <crypto/internal/acompress.h> -+#include <crypto/internal/scompress.h> -+#include "internal.h" -+ -+static const struct crypto_type crypto_scomp_type; -+static void * __percpu *scomp_src_scratches; -+static void * __percpu *scomp_dst_scratches; -+static int scomp_scratch_users; -+static DEFINE_MUTEX(scomp_lock); -+ -+#ifdef CONFIG_NET -+static int crypto_scomp_report(struct sk_buff *skb, struct crypto_alg *alg) -+{ -+ struct crypto_report_comp rscomp; -+ -+ strncpy(rscomp.type, "scomp", sizeof(rscomp.type)); -+ -+ if (nla_put(skb, CRYPTOCFGA_REPORT_COMPRESS, -+ sizeof(struct crypto_report_comp), &rscomp)) -+ goto nla_put_failure; -+ return 0; -+ -+nla_put_failure: -+ return -EMSGSIZE; -+} -+#else -+static int crypto_scomp_report(struct sk_buff *skb, struct crypto_alg *alg) -+{ -+ return -ENOSYS; -+} -+#endif -+ -+static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg) -+ __attribute__ ((unused)); -+ -+static void crypto_scomp_show(struct seq_file *m, struct crypto_alg *alg) -+{ -+ seq_puts(m, "type : scomp\n"); -+} -+ -+static int crypto_scomp_init_tfm(struct crypto_tfm *tfm) -+{ -+ return 0; -+} -+ -+static void crypto_scomp_free_scratches(void * __percpu *scratches) -+{ -+ int i; -+ -+ if (!scratches) -+ return; -+ -+ for_each_possible_cpu(i) -+ vfree(*per_cpu_ptr(scratches, i)); -+ -+ free_percpu(scratches); -+} -+ -+static void * __percpu *crypto_scomp_alloc_scratches(void) -+{ -+ void * __percpu *scratches; -+ int i; -+ -+ scratches = alloc_percpu(void *); -+ if (!scratches) -+ return NULL; -+ -+ for_each_possible_cpu(i) { -+ void *scratch; -+ -+ scratch = vmalloc_node(SCOMP_SCRATCH_SIZE, cpu_to_node(i)); -+ if (!scratch) -+ goto error; -+ *per_cpu_ptr(scratches, i) = scratch; -+ } -+ -+ return scratches; -+ -+error: -+ crypto_scomp_free_scratches(scratches); -+ return NULL; -+} -+ -+static void crypto_scomp_free_all_scratches(void) -+{ -+ if (!--scomp_scratch_users) { -+ crypto_scomp_free_scratches(scomp_src_scratches); -+ crypto_scomp_free_scratches(scomp_dst_scratches); -+ scomp_src_scratches = NULL; -+ scomp_dst_scratches = NULL; -+ } -+} -+ -+static int crypto_scomp_alloc_all_scratches(void) -+{ -+ if (!scomp_scratch_users++) { -+ scomp_src_scratches = crypto_scomp_alloc_scratches(); -+ if (!scomp_src_scratches) -+ return -ENOMEM; -+ scomp_dst_scratches = crypto_scomp_alloc_scratches(); -+ if (!scomp_dst_scratches) -+ return -ENOMEM; -+ } -+ return 0; -+} -+ -+static void crypto_scomp_sg_free(struct scatterlist *sgl) -+{ -+ int i, n; -+ struct page *page; -+ -+ if (!sgl) -+ return; -+ -+ n = sg_nents(sgl); -+ for_each_sg(sgl, sgl, n, i) { -+ page = sg_page(sgl); -+ if (page) -+ __free_page(page); -+ } -+ -+ kfree(sgl); -+} -+ -+static struct scatterlist *crypto_scomp_sg_alloc(size_t size, gfp_t gfp) -+{ -+ struct scatterlist *sgl; -+ struct page *page; -+ int i, n; -+ -+ n = ((size - 1) >> PAGE_SHIFT) + 1; -+ -+ sgl = kmalloc_array(n, sizeof(struct scatterlist), gfp); -+ if (!sgl) -+ return NULL; -+ -+ sg_init_table(sgl, n); -+ -+ for (i = 0; i < n; i++) { -+ page = alloc_page(gfp); -+ if (!page) -+ goto err; -+ sg_set_page(sgl + i, page, PAGE_SIZE, 0); -+ } -+ -+ return sgl; -+ -+err: -+ sg_mark_end(sgl + i); -+ crypto_scomp_sg_free(sgl); -+ return NULL; -+} -+ -+static int scomp_acomp_comp_decomp(struct acomp_req *req, int dir) -+{ -+ struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); -+ void **tfm_ctx = acomp_tfm_ctx(tfm); -+ struct crypto_scomp *scomp = *tfm_ctx; -+ void **ctx = acomp_request_ctx(req); -+ const int cpu = get_cpu(); -+ u8 *scratch_src = *per_cpu_ptr(scomp_src_scratches, cpu); -+ u8 *scratch_dst = *per_cpu_ptr(scomp_dst_scratches, cpu); -+ int ret; -+ -+ if (!req->src || !req->slen || req->slen > SCOMP_SCRATCH_SIZE) { -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ if (req->dst && !req->dlen) { -+ ret = -EINVAL; -+ goto out; -+ } -+ -+ if (!req->dlen || req->dlen > SCOMP_SCRATCH_SIZE) -+ req->dlen = SCOMP_SCRATCH_SIZE; -+ -+ scatterwalk_map_and_copy(scratch_src, req->src, 0, req->slen, 0); -+ if (dir) -+ ret = crypto_scomp_compress(scomp, scratch_src, req->slen, -+ scratch_dst, &req->dlen, *ctx); -+ else -+ ret = crypto_scomp_decompress(scomp, scratch_src, req->slen, -+ scratch_dst, &req->dlen, *ctx); -+ if (!ret) { -+ if (!req->dst) { -+ req->dst = crypto_scomp_sg_alloc(req->dlen, -+ req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP ? -+ GFP_KERNEL : GFP_ATOMIC); -+ if (!req->dst) -+ goto out; -+ } -+ scatterwalk_map_and_copy(scratch_dst, req->dst, 0, req->dlen, -+ 1); -+ } -+out: -+ put_cpu(); -+ return ret; -+} -+ -+static int scomp_acomp_compress(struct acomp_req *req) -+{ -+ return scomp_acomp_comp_decomp(req, 1); -+} -+ -+static int scomp_acomp_decompress(struct acomp_req *req) -+{ -+ return scomp_acomp_comp_decomp(req, 0); -+} -+ -+static void crypto_exit_scomp_ops_async(struct crypto_tfm *tfm) -+{ -+ struct crypto_scomp **ctx = crypto_tfm_ctx(tfm); -+ -+ crypto_free_scomp(*ctx); -+} -+ -+int crypto_init_scomp_ops_async(struct crypto_tfm *tfm) -+{ -+ struct crypto_alg *calg = tfm->__crt_alg; -+ struct crypto_acomp *crt = __crypto_acomp_tfm(tfm); -+ struct crypto_scomp **ctx = crypto_tfm_ctx(tfm); -+ struct crypto_scomp *scomp; -+ -+ if (!crypto_mod_get(calg)) -+ return -EAGAIN; -+ -+ scomp = crypto_create_tfm(calg, &crypto_scomp_type); -+ if (IS_ERR(scomp)) { -+ crypto_mod_put(calg); -+ return PTR_ERR(scomp); -+ } -+ -+ *ctx = scomp; -+ tfm->exit = crypto_exit_scomp_ops_async; -+ -+ crt->compress = scomp_acomp_compress; -+ crt->decompress = scomp_acomp_decompress; -+ crt->dst_free = crypto_scomp_sg_free; -+ crt->reqsize = sizeof(void *); -+ -+ return 0; -+} -+ -+struct acomp_req *crypto_acomp_scomp_alloc_ctx(struct acomp_req *req) -+{ -+ struct crypto_acomp *acomp = crypto_acomp_reqtfm(req); -+ struct crypto_tfm *tfm = crypto_acomp_tfm(acomp); -+ struct crypto_scomp **tfm_ctx = crypto_tfm_ctx(tfm); -+ struct crypto_scomp *scomp = *tfm_ctx; -+ void *ctx; -+ -+ ctx = crypto_scomp_alloc_ctx(scomp); -+ if (IS_ERR(ctx)) { -+ kfree(req); -+ return NULL; -+ } -+ -+ *req->__ctx = ctx; -+ -+ return req; -+} -+ -+void crypto_acomp_scomp_free_ctx(struct acomp_req *req) -+{ -+ struct crypto_acomp *acomp = crypto_acomp_reqtfm(req); -+ struct crypto_tfm *tfm = crypto_acomp_tfm(acomp); -+ struct crypto_scomp **tfm_ctx = crypto_tfm_ctx(tfm); -+ struct crypto_scomp *scomp = *tfm_ctx; -+ void *ctx = *req->__ctx; -+ -+ if (ctx) -+ crypto_scomp_free_ctx(scomp, ctx); -+} -+ -+static const struct crypto_type crypto_scomp_type = { -+ .extsize = crypto_alg_extsize, -+ .init_tfm = crypto_scomp_init_tfm, -+#ifdef CONFIG_PROC_FS -+ .show = crypto_scomp_show, -+#endif -+ .report = crypto_scomp_report, -+ .maskclear = ~CRYPTO_ALG_TYPE_MASK, -+ .maskset = CRYPTO_ALG_TYPE_MASK, -+ .type = CRYPTO_ALG_TYPE_SCOMPRESS, -+ .tfmsize = offsetof(struct crypto_scomp, base), -+}; -+ -+int crypto_register_scomp(struct scomp_alg *alg) -+{ -+ struct crypto_alg *base = &alg->base; -+ int ret = -ENOMEM; -+ -+ mutex_lock(&scomp_lock); -+ if (crypto_scomp_alloc_all_scratches()) -+ goto error; -+ -+ base->cra_type = &crypto_scomp_type; -+ base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK; -+ base->cra_flags |= CRYPTO_ALG_TYPE_SCOMPRESS; -+ -+ ret = crypto_register_alg(base); -+ if (ret) -+ goto error; -+ -+ mutex_unlock(&scomp_lock); -+ return ret; -+ -+error: -+ crypto_scomp_free_all_scratches(); -+ mutex_unlock(&scomp_lock); -+ return ret; -+} -+EXPORT_SYMBOL_GPL(crypto_register_scomp); -+ -+int crypto_unregister_scomp(struct scomp_alg *alg) -+{ -+ int ret; -+ -+ mutex_lock(&scomp_lock); -+ ret = crypto_unregister_alg(&alg->base); -+ crypto_scomp_free_all_scratches(); -+ mutex_unlock(&scomp_lock); -+ -+ return ret; -+} -+EXPORT_SYMBOL_GPL(crypto_unregister_scomp); -+ -+MODULE_LICENSE("GPL"); -+MODULE_DESCRIPTION("Synchronous compression type"); ---- a/crypto/tcrypt.c -+++ b/crypto/tcrypt.c -@@ -74,7 +74,7 @@ static char *check[] = { - "khazad", "wp512", "wp384", "wp256", "tnepres", "xeta", "fcrypt", - "camellia", "seed", "salsa20", "rmd128", "rmd160", "rmd256", "rmd320", - "lzo", "cts", "zlib", "sha3-224", "sha3-256", "sha3-384", "sha3-512", -- NULL -+ "rsa", NULL - }; - - struct tcrypt_result { -@@ -1336,6 +1336,10 @@ static int do_test(const char *alg, u32 - ret += tcrypt_test("hmac(sha3-512)"); - break; - -+ case 115: -+ ret += tcrypt_test("rsa"); -+ break; -+ - case 150: - ret += tcrypt_test("ansi_cprng"); - break; -@@ -1397,6 +1401,9 @@ static int do_test(const char *alg, u32 - case 190: - ret += tcrypt_test("authenc(hmac(sha512),cbc(des3_ede))"); - break; -+ case 191: -+ ret += tcrypt_test("tls10(hmac(sha1),cbc(aes))"); -+ break; - case 200: - test_cipher_speed("ecb(aes)", ENCRYPT, sec, NULL, 0, - speed_template_16_24_32); -@@ -1411,9 +1418,9 @@ static int do_test(const char *alg, u32 - test_cipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0, - speed_template_32_40_48); - test_cipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0, -- speed_template_32_48_64); -+ speed_template_32_64); - test_cipher_speed("xts(aes)", DECRYPT, sec, NULL, 0, -- speed_template_32_48_64); -+ speed_template_32_64); - test_cipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0, - speed_template_16_24_32); - test_cipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0, -@@ -1844,9 +1851,9 @@ static int do_test(const char *alg, u32 - test_acipher_speed("lrw(aes)", DECRYPT, sec, NULL, 0, - speed_template_32_40_48); - test_acipher_speed("xts(aes)", ENCRYPT, sec, NULL, 0, -- speed_template_32_48_64); -+ speed_template_32_64); - test_acipher_speed("xts(aes)", DECRYPT, sec, NULL, 0, -- speed_template_32_48_64); -+ speed_template_32_64); - test_acipher_speed("cts(cbc(aes))", ENCRYPT, sec, NULL, 0, - speed_template_16_24_32); - test_acipher_speed("cts(cbc(aes))", DECRYPT, sec, NULL, 0, ---- a/crypto/testmgr.c -+++ b/crypto/testmgr.c -@@ -33,6 +33,7 @@ - #include <crypto/drbg.h> - #include <crypto/akcipher.h> - #include <crypto/kpp.h> -+#include <crypto/acompress.h> - - #include "internal.h" - -@@ -62,7 +63,7 @@ int alg_test(const char *driver, const c - */ - #define IDX1 32 - #define IDX2 32400 --#define IDX3 1 -+#define IDX3 1511 - #define IDX4 8193 - #define IDX5 22222 - #define IDX6 17101 -@@ -82,47 +83,54 @@ struct tcrypt_result { - - struct aead_test_suite { - struct { -- struct aead_testvec *vecs; -+ const struct aead_testvec *vecs; - unsigned int count; - } enc, dec; - }; - - struct cipher_test_suite { - struct { -- struct cipher_testvec *vecs; -+ const struct cipher_testvec *vecs; - unsigned int count; - } enc, dec; - }; - - struct comp_test_suite { - struct { -- struct comp_testvec *vecs; -+ const struct comp_testvec *vecs; - unsigned int count; - } comp, decomp; - }; - - struct hash_test_suite { -- struct hash_testvec *vecs; -+ const struct hash_testvec *vecs; - unsigned int count; - }; - - struct cprng_test_suite { -- struct cprng_testvec *vecs; -+ const struct cprng_testvec *vecs; - unsigned int count; - }; - - struct drbg_test_suite { -- struct drbg_testvec *vecs; -+ const struct drbg_testvec *vecs; - unsigned int count; - }; - -+struct tls_test_suite { -+ struct { -+ struct tls_testvec *vecs; -+ unsigned int count; -+ } enc, dec; -+}; -+ - struct akcipher_test_suite { -- struct akcipher_testvec *vecs; -+ const struct akcipher_testvec *vecs; - unsigned int count; - }; - - struct kpp_test_suite { -- struct kpp_testvec *vecs; -+ const struct kpp_testvec *vecs; - unsigned int count; - }; - -@@ -139,12 +147,14 @@ struct alg_test_desc { - struct hash_test_suite hash; - struct cprng_test_suite cprng; - struct drbg_test_suite drbg; -+ struct tls_test_suite tls; - struct akcipher_test_suite akcipher; - struct kpp_test_suite kpp; - } suite; - }; - --static unsigned int IDX[8] = { IDX1, IDX2, IDX3, IDX4, IDX5, IDX6, IDX7, IDX8 }; -+static const unsigned int IDX[8] = { -+ IDX1, IDX2, IDX3, IDX4, IDX5, IDX6, IDX7, IDX8 }; - - static void hexdump(unsigned char *buf, unsigned int len) - { -@@ -202,7 +212,7 @@ static int wait_async_op(struct tcrypt_r - } - - static int ahash_partial_update(struct ahash_request **preq, -- struct crypto_ahash *tfm, struct hash_testvec *template, -+ struct crypto_ahash *tfm, const struct hash_testvec *template, - void *hash_buff, int k, int temp, struct scatterlist *sg, - const char *algo, char *result, struct tcrypt_result *tresult) - { -@@ -259,11 +269,12 @@ out_nostate: - return ret; - } - --static int __test_hash(struct crypto_ahash *tfm, struct hash_testvec *template, -- unsigned int tcount, bool use_digest, -- const int align_offset) -+static int __test_hash(struct crypto_ahash *tfm, -+ const struct hash_testvec *template, unsigned int tcount, -+ bool use_digest, const int align_offset) - { - const char *algo = crypto_tfm_alg_driver_name(crypto_ahash_tfm(tfm)); -+ size_t digest_size = crypto_ahash_digestsize(tfm); - unsigned int i, j, k, temp; - struct scatterlist sg[8]; - char *result; -@@ -274,7 +285,7 @@ static int __test_hash(struct crypto_aha - char *xbuf[XBUFSIZE]; - int ret = -ENOMEM; - -- result = kmalloc(MAX_DIGEST_SIZE, GFP_KERNEL); -+ result = kmalloc(digest_size, GFP_KERNEL); - if (!result) - return ret; - key = kmalloc(MAX_KEYLEN, GFP_KERNEL); -@@ -304,7 +315,7 @@ static int __test_hash(struct crypto_aha - goto out; - - j++; -- memset(result, 0, MAX_DIGEST_SIZE); -+ memset(result, 0, digest_size); - - hash_buff = xbuf[0]; - hash_buff += align_offset; -@@ -379,7 +390,7 @@ static int __test_hash(struct crypto_aha - continue; - - j++; -- memset(result, 0, MAX_DIGEST_SIZE); -+ memset(result, 0, digest_size); - - temp = 0; - sg_init_table(sg, template[i].np); -@@ -457,7 +468,7 @@ static int __test_hash(struct crypto_aha - continue; - - j++; -- memset(result, 0, MAX_DIGEST_SIZE); -+ memset(result, 0, digest_size); - - ret = -EINVAL; - hash_buff = xbuf[0]; -@@ -536,7 +547,8 @@ out_nobuf: - return ret; - } - --static int test_hash(struct crypto_ahash *tfm, struct hash_testvec *template, -+static int test_hash(struct crypto_ahash *tfm, -+ const struct hash_testvec *template, - unsigned int tcount, bool use_digest) - { - unsigned int alignmask; -@@ -564,7 +576,7 @@ static int test_hash(struct crypto_ahash - } - - static int __test_aead(struct crypto_aead *tfm, int enc, -- struct aead_testvec *template, unsigned int tcount, -+ const struct aead_testvec *template, unsigned int tcount, - const bool diff_dst, const int align_offset) - { - const char *algo = crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)); -@@ -955,7 +967,7 @@ out_noxbuf: - } - - static int test_aead(struct crypto_aead *tfm, int enc, -- struct aead_testvec *template, unsigned int tcount) -+ const struct aead_testvec *template, unsigned int tcount) - { - unsigned int alignmask; - int ret; -@@ -987,8 +999,236 @@ static int test_aead(struct crypto_aead - return 0; - } - -+static int __test_tls(struct crypto_aead *tfm, int enc, -+ struct tls_testvec *template, unsigned int tcount, -+ const bool diff_dst) -+{ -+ const char *algo = crypto_tfm_alg_driver_name(crypto_aead_tfm(tfm)); -+ unsigned int i, k, authsize; -+ char *q; -+ struct aead_request *req; -+ struct scatterlist *sg; -+ struct scatterlist *sgout; -+ const char *e, *d; -+ struct tcrypt_result result; -+ void *input; -+ void *output; -+ void *assoc; -+ char *iv; -+ char *key; -+ char *xbuf[XBUFSIZE]; -+ char *xoutbuf[XBUFSIZE]; -+ char *axbuf[XBUFSIZE]; -+ int ret = -ENOMEM; -+ -+ if (testmgr_alloc_buf(xbuf)) -+ goto out_noxbuf; -+ -+ if (diff_dst && testmgr_alloc_buf(xoutbuf)) -+ goto out_nooutbuf; -+ -+ if (testmgr_alloc_buf(axbuf)) -+ goto out_noaxbuf; -+ -+ iv = kzalloc(MAX_IVLEN, GFP_KERNEL); -+ if (!iv) -+ goto out_noiv; -+ -+ key = kzalloc(MAX_KEYLEN, GFP_KERNEL); -+ if (!key) -+ goto out_nokey; -+ -+ sg = kmalloc(sizeof(*sg) * 8 * (diff_dst ? 2 : 1), GFP_KERNEL); -+ if (!sg) -+ goto out_nosg; -+ -+ sgout = sg + 8; -+ -+ d = diff_dst ? "-ddst" : ""; -+ e = enc ? "encryption" : "decryption"; -+ -+ init_completion(&result.completion); -+ -+ req = aead_request_alloc(tfm, GFP_KERNEL); -+ if (!req) { -+ pr_err("alg: tls%s: Failed to allocate request for %s\n", -+ d, algo); -+ goto out; -+ } -+ -+ aead_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, -+ tcrypt_complete, &result); -+ -+ for (i = 0; i < tcount; i++) { -+ input = xbuf[0]; -+ assoc = axbuf[0]; -+ -+ ret = -EINVAL; -+ if (WARN_ON(template[i].ilen > PAGE_SIZE || -+ template[i].alen > PAGE_SIZE)) -+ goto out; -+ -+ memcpy(assoc, template[i].assoc, template[i].alen); -+ memcpy(input, template[i].input, template[i].ilen); -+ -+ if (template[i].iv) -+ memcpy(iv, template[i].iv, MAX_IVLEN); -+ else -+ memset(iv, 0, MAX_IVLEN); -+ -+ crypto_aead_clear_flags(tfm, ~0); -+ -+ if (template[i].klen > MAX_KEYLEN) { -+ pr_err("alg: aead%s: setkey failed on test %d for %s: key size %d > %d\n", -+ d, i, algo, template[i].klen, MAX_KEYLEN); -+ ret = -EINVAL; -+ goto out; -+ } -+ memcpy(key, template[i].key, template[i].klen); -+ -+ ret = crypto_aead_setkey(tfm, key, template[i].klen); -+ if (!ret == template[i].fail) { -+ pr_err("alg: tls%s: setkey failed on test %d for %s: flags=%x\n", -+ d, i, algo, crypto_aead_get_flags(tfm)); -+ goto out; -+ } else if (ret) -+ continue; -+ -+ authsize = 20; -+ ret = crypto_aead_setauthsize(tfm, authsize); -+ if (ret) { -+ pr_err("alg: aead%s: Failed to set authsize to %u on test %d for %s\n", -+ d, authsize, i, algo); -+ goto out; -+ } -+ -+ k = !!template[i].alen; -+ sg_init_table(sg, k + 1); -+ sg_set_buf(&sg[0], assoc, template[i].alen); -+ sg_set_buf(&sg[k], input, (enc ? template[i].rlen : -+ template[i].ilen)); -+ output = input; -+ -+ if (diff_dst) { -+ sg_init_table(sgout, k + 1); -+ sg_set_buf(&sgout[0], assoc, template[i].alen); -+ -+ output = xoutbuf[0]; -+ sg_set_buf(&sgout[k], output, -+ (enc ? template[i].rlen : template[i].ilen)); -+ } -+ -+ aead_request_set_crypt(req, sg, (diff_dst) ? sgout : sg, -+ template[i].ilen, iv); -+ -+ aead_request_set_ad(req, template[i].alen); -+ -+ ret = enc ? crypto_aead_encrypt(req) : crypto_aead_decrypt(req); -+ -+ switch (ret) { -+ case 0: -+ if (template[i].novrfy) { -+ /* verification was supposed to fail */ -+ pr_err("alg: tls%s: %s failed on test %d for %s: ret was 0, expected -EBADMSG\n", -+ d, e, i, algo); -+ /* so really, we got a bad message */ -+ ret = -EBADMSG; -+ goto out; -+ } -+ break; -+ case -EINPROGRESS: -+ case -EBUSY: -+ wait_for_completion(&result.completion); -+ reinit_completion(&result.completion); -+ ret = result.err; -+ if (!ret) -+ break; -+ case -EBADMSG: -+ /* verification failure was expected */ -+ if (template[i].novrfy) -+ continue; -+ /* fall through */ -+ default: -+ pr_err("alg: tls%s: %s failed on test %d for %s: ret=%d\n", -+ d, e, i, algo, -ret); -+ goto out; -+ } -+ -+ q = output; -+ if (memcmp(q, template[i].result, template[i].rlen)) { -+ pr_err("alg: tls%s: Test %d failed on %s for %s\n", -+ d, i, e, algo); -+ hexdump(q, template[i].rlen); -+ pr_err("should be:\n"); -+ hexdump(template[i].result, template[i].rlen); -+ ret = -EINVAL; -+ goto out; -+ } -+ } -+ -+out: -+ aead_request_free(req); -+ -+ kfree(sg); -+out_nosg: -+ kfree(key); -+out_nokey: -+ kfree(iv); -+out_noiv: -+ testmgr_free_buf(axbuf); -+out_noaxbuf: -+ if (diff_dst) -+ testmgr_free_buf(xoutbuf); -+out_nooutbuf: -+ testmgr_free_buf(xbuf); -+out_noxbuf: -+ return ret; -+} -+ -+static int test_tls(struct crypto_aead *tfm, int enc, -+ struct tls_testvec *template, unsigned int tcount) -+{ -+ int ret; -+ /* test 'dst == src' case */ -+ ret = __test_tls(tfm, enc, template, tcount, false); -+ if (ret) -+ return ret; -+ /* test 'dst != src' case */ -+ return __test_tls(tfm, enc, template, tcount, true); -+} -+ -+static int alg_test_tls(const struct alg_test_desc *desc, const char *driver, -+ u32 type, u32 mask) -+{ -+ struct crypto_aead *tfm; -+ int err = 0; -+ -+ tfm = crypto_alloc_aead(driver, type, mask); -+ if (IS_ERR(tfm)) { -+ pr_err("alg: aead: Failed to load transform for %s: %ld\n", -+ driver, PTR_ERR(tfm)); -+ return PTR_ERR(tfm); -+ } -+ -+ if (desc->suite.tls.enc.vecs) { -+ err = test_tls(tfm, ENCRYPT, desc->suite.tls.enc.vecs, -+ desc->suite.tls.enc.count); -+ if (err) -+ goto out; -+ } -+ -+ if (!err && desc->suite.tls.dec.vecs) -+ err = test_tls(tfm, DECRYPT, desc->suite.tls.dec.vecs, -+ desc->suite.tls.dec.count); -+ -+out: -+ crypto_free_aead(tfm); -+ return err; -+} -+ - static int test_cipher(struct crypto_cipher *tfm, int enc, -- struct cipher_testvec *template, unsigned int tcount) -+ const struct cipher_testvec *template, -+ unsigned int tcount) - { - const char *algo = crypto_tfm_alg_driver_name(crypto_cipher_tfm(tfm)); - unsigned int i, j, k; -@@ -1066,7 +1306,8 @@ out_nobuf: - } - - static int __test_skcipher(struct crypto_skcipher *tfm, int enc, -- struct cipher_testvec *template, unsigned int tcount, -+ const struct cipher_testvec *template, -+ unsigned int tcount, - const bool diff_dst, const int align_offset) - { - const char *algo = -@@ -1079,12 +1320,16 @@ static int __test_skcipher(struct crypto - const char *e, *d; - struct tcrypt_result result; - void *data; -- char iv[MAX_IVLEN]; -+ char *iv; - char *xbuf[XBUFSIZE]; - char *xoutbuf[XBUFSIZE]; - int ret = -ENOMEM; - unsigned int ivsize = crypto_skcipher_ivsize(tfm); - -+ iv = kmalloc(MAX_IVLEN, GFP_KERNEL); -+ if (!iv) -+ return ret; -+ - if (testmgr_alloc_buf(xbuf)) - goto out_nobuf; - -@@ -1325,12 +1570,14 @@ out: - testmgr_free_buf(xoutbuf); - out_nooutbuf: - testmgr_free_buf(xbuf); -+ kfree(iv); - out_nobuf: - return ret; - } - - static int test_skcipher(struct crypto_skcipher *tfm, int enc, -- struct cipher_testvec *template, unsigned int tcount) -+ const struct cipher_testvec *template, -+ unsigned int tcount) - { - unsigned int alignmask; - int ret; -@@ -1362,8 +1609,10 @@ static int test_skcipher(struct crypto_s - return 0; - } - --static int test_comp(struct crypto_comp *tfm, struct comp_testvec *ctemplate, -- struct comp_testvec *dtemplate, int ctcount, int dtcount) -+static int test_comp(struct crypto_comp *tfm, -+ const struct comp_testvec *ctemplate, -+ const struct comp_testvec *dtemplate, -+ int ctcount, int dtcount) - { - const char *algo = crypto_tfm_alg_driver_name(crypto_comp_tfm(tfm)); - unsigned int i; -@@ -1442,7 +1691,154 @@ out: - return ret; - } - --static int test_cprng(struct crypto_rng *tfm, struct cprng_testvec *template, -+static int test_acomp(struct crypto_acomp *tfm, -+ const struct comp_testvec *ctemplate, -+ const struct comp_testvec *dtemplate, -+ int ctcount, int dtcount) -+{ -+ const char *algo = crypto_tfm_alg_driver_name(crypto_acomp_tfm(tfm)); -+ unsigned int i; -+ char *output; -+ int ret; -+ struct scatterlist src, dst; -+ struct acomp_req *req; -+ struct tcrypt_result result; -+ -+ output = kmalloc(COMP_BUF_SIZE, GFP_KERNEL); -+ if (!output) -+ return -ENOMEM; -+ -+ for (i = 0; i < ctcount; i++) { -+ unsigned int dlen = COMP_BUF_SIZE; -+ int ilen = ctemplate[i].inlen; -+ void *input_vec; -+ -+ input_vec = kmemdup(ctemplate[i].input, ilen, GFP_KERNEL); -+ if (!input_vec) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ -+ memset(output, 0, dlen); -+ init_completion(&result.completion); -+ sg_init_one(&src, input_vec, ilen); -+ sg_init_one(&dst, output, dlen); -+ -+ req = acomp_request_alloc(tfm); -+ if (!req) { -+ pr_err("alg: acomp: request alloc failed for %s\n", -+ algo); -+ kfree(input_vec); -+ ret = -ENOMEM; -+ goto out; -+ } -+ -+ acomp_request_set_params(req, &src, &dst, ilen, dlen); -+ acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, -+ tcrypt_complete, &result); -+ -+ ret = wait_async_op(&result, crypto_acomp_compress(req)); -+ if (ret) { -+ pr_err("alg: acomp: compression failed on test %d for %s: ret=%d\n", -+ i + 1, algo, -ret); -+ kfree(input_vec); -+ acomp_request_free(req); -+ goto out; -+ } -+ -+ if (req->dlen != ctemplate[i].outlen) { -+ pr_err("alg: acomp: Compression test %d failed for %s: output len = %d\n", -+ i + 1, algo, req->dlen); -+ ret = -EINVAL; -+ kfree(input_vec); -+ acomp_request_free(req); -+ goto out; -+ } -+ -+ if (memcmp(output, ctemplate[i].output, req->dlen)) { -+ pr_err("alg: acomp: Compression test %d failed for %s\n", -+ i + 1, algo); -+ hexdump(output, req->dlen); -+ ret = -EINVAL; -+ kfree(input_vec); -+ acomp_request_free(req); -+ goto out; -+ } -+ -+ kfree(input_vec); -+ acomp_request_free(req); -+ } -+ -+ for (i = 0; i < dtcount; i++) { -+ unsigned int dlen = COMP_BUF_SIZE; -+ int ilen = dtemplate[i].inlen; -+ void *input_vec; -+ -+ input_vec = kmemdup(dtemplate[i].input, ilen, GFP_KERNEL); -+ if (!input_vec) { -+ ret = -ENOMEM; -+ goto out; -+ } -+ -+ memset(output, 0, dlen); -+ init_completion(&result.completion); -+ sg_init_one(&src, input_vec, ilen); -+ sg_init_one(&dst, output, dlen); -+ -+ req = acomp_request_alloc(tfm); -+ if (!req) { -+ pr_err("alg: acomp: request alloc failed for %s\n", -+ algo); -+ kfree(input_vec); -+ ret = -ENOMEM; -+ goto out; -+ } -+ -+ acomp_request_set_params(req, &src, &dst, ilen, dlen); -+ acomp_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, -+ tcrypt_complete, &result); -+ -+ ret = wait_async_op(&result, crypto_acomp_decompress(req)); -+ if (ret) { -+ pr_err("alg: acomp: decompression failed on test %d for %s: ret=%d\n", -+ i + 1, algo, -ret); -+ kfree(input_vec); -+ acomp_request_free(req); -+ goto out; -+ } -+ -+ if (req->dlen != dtemplate[i].outlen) { -+ pr_err("alg: acomp: Decompression test %d failed for %s: output len = %d\n", -+ i + 1, algo, req->dlen); -+ ret = -EINVAL; -+ kfree(input_vec); -+ acomp_request_free(req); -+ goto out; -+ } -+ -+ if (memcmp(output, dtemplate[i].output, req->dlen)) { -+ pr_err("alg: acomp: Decompression test %d failed for %s\n", -+ i + 1, algo); -+ hexdump(output, req->dlen); -+ ret = -EINVAL; -+ kfree(input_vec); -+ acomp_request_free(req); -+ goto out; -+ } -+ -+ kfree(input_vec); -+ acomp_request_free(req); -+ } -+ -+ ret = 0; -+ -+out: -+ kfree(output); -+ return ret; -+} -+ -+static int test_cprng(struct crypto_rng *tfm, -+ const struct cprng_testvec *template, - unsigned int tcount) - { - const char *algo = crypto_tfm_alg_driver_name(crypto_rng_tfm(tfm)); -@@ -1509,7 +1905,7 @@ static int alg_test_aead(const struct al - struct crypto_aead *tfm; - int err = 0; - -- tfm = crypto_alloc_aead(driver, type | CRYPTO_ALG_INTERNAL, mask); -+ tfm = crypto_alloc_aead(driver, type, mask); - if (IS_ERR(tfm)) { - printk(KERN_ERR "alg: aead: Failed to load transform for %s: " - "%ld\n", driver, PTR_ERR(tfm)); -@@ -1538,7 +1934,7 @@ static int alg_test_cipher(const struct - struct crypto_cipher *tfm; - int err = 0; - -- tfm = crypto_alloc_cipher(driver, type | CRYPTO_ALG_INTERNAL, mask); -+ tfm = crypto_alloc_cipher(driver, type, mask); - if (IS_ERR(tfm)) { - printk(KERN_ERR "alg: cipher: Failed to load transform for " - "%s: %ld\n", driver, PTR_ERR(tfm)); -@@ -1567,7 +1963,7 @@ static int alg_test_skcipher(const struc - struct crypto_skcipher *tfm; - int err = 0; - -- tfm = crypto_alloc_skcipher(driver, type | CRYPTO_ALG_INTERNAL, mask); -+ tfm = crypto_alloc_skcipher(driver, type, mask); - if (IS_ERR(tfm)) { - printk(KERN_ERR "alg: skcipher: Failed to load transform for " - "%s: %ld\n", driver, PTR_ERR(tfm)); -@@ -1593,22 +1989,38 @@ out: - static int alg_test_comp(const struct alg_test_desc *desc, const char *driver, - u32 type, u32 mask) - { -- struct crypto_comp *tfm; -+ struct crypto_comp *comp; -+ struct crypto_acomp *acomp; - int err; -+ u32 algo_type = type & CRYPTO_ALG_TYPE_ACOMPRESS_MASK; - -- tfm = crypto_alloc_comp(driver, type, mask); -- if (IS_ERR(tfm)) { -- printk(KERN_ERR "alg: comp: Failed to load transform for %s: " -- "%ld\n", driver, PTR_ERR(tfm)); -- return PTR_ERR(tfm); -- } -+ if (algo_type == CRYPTO_ALG_TYPE_ACOMPRESS) { -+ acomp = crypto_alloc_acomp(driver, type, mask); -+ if (IS_ERR(acomp)) { -+ pr_err("alg: acomp: Failed to load transform for %s: %ld\n", -+ driver, PTR_ERR(acomp)); -+ return PTR_ERR(acomp); -+ } -+ err = test_acomp(acomp, desc->suite.comp.comp.vecs, -+ desc->suite.comp.decomp.vecs, -+ desc->suite.comp.comp.count, -+ desc->suite.comp.decomp.count); -+ crypto_free_acomp(acomp); -+ } else { -+ comp = crypto_alloc_comp(driver, type, mask); -+ if (IS_ERR(comp)) { -+ pr_err("alg: comp: Failed to load transform for %s: %ld\n", -+ driver, PTR_ERR(comp)); -+ return PTR_ERR(comp); -+ } - -- err = test_comp(tfm, desc->suite.comp.comp.vecs, -- desc->suite.comp.decomp.vecs, -- desc->suite.comp.comp.count, -- desc->suite.comp.decomp.count); -+ err = test_comp(comp, desc->suite.comp.comp.vecs, -+ desc->suite.comp.decomp.vecs, -+ desc->suite.comp.comp.count, -+ desc->suite.comp.decomp.count); - -- crypto_free_comp(tfm); -+ crypto_free_comp(comp); -+ } - return err; - } - -@@ -1618,7 +2030,7 @@ static int alg_test_hash(const struct al - struct crypto_ahash *tfm; - int err; - -- tfm = crypto_alloc_ahash(driver, type | CRYPTO_ALG_INTERNAL, mask); -+ tfm = crypto_alloc_ahash(driver, type, mask); - if (IS_ERR(tfm)) { - printk(KERN_ERR "alg: hash: Failed to load transform for %s: " - "%ld\n", driver, PTR_ERR(tfm)); -@@ -1646,7 +2058,7 @@ static int alg_test_crc32c(const struct - if (err) - goto out; - -- tfm = crypto_alloc_shash(driver, type | CRYPTO_ALG_INTERNAL, mask); -+ tfm = crypto_alloc_shash(driver, type, mask); - if (IS_ERR(tfm)) { - printk(KERN_ERR "alg: crc32c: Failed to load transform for %s: " - "%ld\n", driver, PTR_ERR(tfm)); -@@ -1688,7 +2100,7 @@ static int alg_test_cprng(const struct a - struct crypto_rng *rng; - int err; - -- rng = crypto_alloc_rng(driver, type | CRYPTO_ALG_INTERNAL, mask); -+ rng = crypto_alloc_rng(driver, type, mask); - if (IS_ERR(rng)) { - printk(KERN_ERR "alg: cprng: Failed to load transform for %s: " - "%ld\n", driver, PTR_ERR(rng)); -@@ -1703,7 +2115,7 @@ static int alg_test_cprng(const struct a - } - - --static int drbg_cavs_test(struct drbg_testvec *test, int pr, -+static int drbg_cavs_test(const struct drbg_testvec *test, int pr, - const char *driver, u32 type, u32 mask) - { - int ret = -EAGAIN; -@@ -1715,7 +2127,7 @@ static int drbg_cavs_test(struct drbg_te - if (!buf) - return -ENOMEM; - -- drng = crypto_alloc_rng(driver, type | CRYPTO_ALG_INTERNAL, mask); -+ drng = crypto_alloc_rng(driver, type, mask); - if (IS_ERR(drng)) { - printk(KERN_ERR "alg: drbg: could not allocate DRNG handle for " - "%s\n", driver); -@@ -1777,7 +2189,7 @@ static int alg_test_drbg(const struct al - int err = 0; - int pr = 0; - int i = 0; -- struct drbg_testvec *template = desc->suite.drbg.vecs; -+ const struct drbg_testvec *template = desc->suite.drbg.vecs; - unsigned int tcount = desc->suite.drbg.count; - - if (0 == memcmp(driver, "drbg_pr_", 8)) -@@ -1796,7 +2208,7 @@ static int alg_test_drbg(const struct al - - } - --static int do_test_kpp(struct crypto_kpp *tfm, struct kpp_testvec *vec, -+static int do_test_kpp(struct crypto_kpp *tfm, const struct kpp_testvec *vec, - const char *alg) - { - struct kpp_request *req; -@@ -1888,7 +2300,7 @@ free_req: - } - - static int test_kpp(struct crypto_kpp *tfm, const char *alg, -- struct kpp_testvec *vecs, unsigned int tcount) -+ const struct kpp_testvec *vecs, unsigned int tcount) - { - int ret, i; - -@@ -1909,7 +2321,7 @@ static int alg_test_kpp(const struct alg - struct crypto_kpp *tfm; - int err = 0; - -- tfm = crypto_alloc_kpp(driver, type | CRYPTO_ALG_INTERNAL, mask); -+ tfm = crypto_alloc_kpp(driver, type, mask); - if (IS_ERR(tfm)) { - pr_err("alg: kpp: Failed to load tfm for %s: %ld\n", - driver, PTR_ERR(tfm)); -@@ -1924,7 +2336,7 @@ static int alg_test_kpp(const struct alg - } - - static int test_akcipher_one(struct crypto_akcipher *tfm, -- struct akcipher_testvec *vecs) -+ const struct akcipher_testvec *vecs) - { - char *xbuf[XBUFSIZE]; - struct akcipher_request *req; -@@ -2044,7 +2456,8 @@ free_xbuf: - } - - static int test_akcipher(struct crypto_akcipher *tfm, const char *alg, -- struct akcipher_testvec *vecs, unsigned int tcount) -+ const struct akcipher_testvec *vecs, -+ unsigned int tcount) - { - const char *algo = - crypto_tfm_alg_driver_name(crypto_akcipher_tfm(tfm)); -@@ -2068,7 +2481,7 @@ static int alg_test_akcipher(const struc - struct crypto_akcipher *tfm; - int err = 0; - -- tfm = crypto_alloc_akcipher(driver, type | CRYPTO_ALG_INTERNAL, mask); -+ tfm = crypto_alloc_akcipher(driver, type, mask); - if (IS_ERR(tfm)) { - pr_err("alg: akcipher: Failed to load tfm for %s: %ld\n", - driver, PTR_ERR(tfm)); -@@ -2088,112 +2501,23 @@ static int alg_test_null(const struct al - return 0; - } - -+#define __VECS(tv) { .vecs = tv, .count = ARRAY_SIZE(tv) } -+ - /* Please keep this list sorted by algorithm name. */ - static const struct alg_test_desc alg_test_descs[] = { - { -- .alg = "__cbc-cast5-avx", -- .test = alg_test_null, -- }, { -- .alg = "__cbc-cast6-avx", -- .test = alg_test_null, -- }, { -- .alg = "__cbc-serpent-avx", -- .test = alg_test_null, -- }, { -- .alg = "__cbc-serpent-avx2", -- .test = alg_test_null, -- }, { -- .alg = "__cbc-serpent-sse2", -- .test = alg_test_null, -- }, { -- .alg = "__cbc-twofish-avx", -- .test = alg_test_null, -- }, { -- .alg = "__driver-cbc-aes-aesni", -- .test = alg_test_null, -- .fips_allowed = 1, -- }, { -- .alg = "__driver-cbc-camellia-aesni", -- .test = alg_test_null, -- }, { -- .alg = "__driver-cbc-camellia-aesni-avx2", -- .test = alg_test_null, -- }, { -- .alg = "__driver-cbc-cast5-avx", -- .test = alg_test_null, -- }, { -- .alg = "__driver-cbc-cast6-avx", -- .test = alg_test_null, -- }, { -- .alg = "__driver-cbc-serpent-avx", -- .test = alg_test_null, -- }, { -- .alg = "__driver-cbc-serpent-avx2", -- .test = alg_test_null, -- }, { -- .alg = "__driver-cbc-serpent-sse2", -- .test = alg_test_null, -- }, { -- .alg = "__driver-cbc-twofish-avx", -- .test = alg_test_null, -- }, { -- .alg = "__driver-ecb-aes-aesni", -- .test = alg_test_null, -- .fips_allowed = 1, -- }, { -- .alg = "__driver-ecb-camellia-aesni", -- .test = alg_test_null, -- }, { -- .alg = "__driver-ecb-camellia-aesni-avx2", -- .test = alg_test_null, -- }, { -- .alg = "__driver-ecb-cast5-avx", -- .test = alg_test_null, -- }, { -- .alg = "__driver-ecb-cast6-avx", -- .test = alg_test_null, -- }, { -- .alg = "__driver-ecb-serpent-avx", -- .test = alg_test_null, -- }, { -- .alg = "__driver-ecb-serpent-avx2", -- .test = alg_test_null, -- }, { -- .alg = "__driver-ecb-serpent-sse2", -- .test = alg_test_null, -- }, { -- .alg = "__driver-ecb-twofish-avx", -- .test = alg_test_null, -- }, { -- .alg = "__driver-gcm-aes-aesni", -- .test = alg_test_null, -- .fips_allowed = 1, -- }, { -- .alg = "__ghash-pclmulqdqni", -- .test = alg_test_null, -- .fips_allowed = 1, -- }, { - .alg = "ansi_cprng", - .test = alg_test_cprng, - .suite = { -- .cprng = { -- .vecs = ansi_cprng_aes_tv_template, -- .count = ANSI_CPRNG_AES_TEST_VECTORS -- } -+ .cprng = __VECS(ansi_cprng_aes_tv_template) - } - }, { - .alg = "authenc(hmac(md5),ecb(cipher_null))", - .test = alg_test_aead, - .suite = { - .aead = { -- .enc = { -- .vecs = hmac_md5_ecb_cipher_null_enc_tv_template, -- .count = HMAC_MD5_ECB_CIPHER_NULL_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = hmac_md5_ecb_cipher_null_dec_tv_template, -- .count = HMAC_MD5_ECB_CIPHER_NULL_DEC_TEST_VECTORS -- } -+ .enc = __VECS(hmac_md5_ecb_cipher_null_enc_tv_template), -+ .dec = __VECS(hmac_md5_ecb_cipher_null_dec_tv_template) - } - } - }, { -@@ -2201,12 +2525,7 @@ static const struct alg_test_desc alg_te - .test = alg_test_aead, - .suite = { - .aead = { -- .enc = { -- .vecs = -- hmac_sha1_aes_cbc_enc_tv_temp, -- .count = -- HMAC_SHA1_AES_CBC_ENC_TEST_VEC -- } -+ .enc = __VECS(hmac_sha1_aes_cbc_enc_tv_temp) - } - } - }, { -@@ -2214,12 +2533,7 @@ static const struct alg_test_desc alg_te - .test = alg_test_aead, - .suite = { - .aead = { -- .enc = { -- .vecs = -- hmac_sha1_des_cbc_enc_tv_temp, -- .count = -- HMAC_SHA1_DES_CBC_ENC_TEST_VEC -- } -+ .enc = __VECS(hmac_sha1_des_cbc_enc_tv_temp) - } - } - }, { -@@ -2228,12 +2542,7 @@ static const struct alg_test_desc alg_te - .fips_allowed = 1, - .suite = { - .aead = { -- .enc = { -- .vecs = -- hmac_sha1_des3_ede_cbc_enc_tv_temp, -- .count = -- HMAC_SHA1_DES3_EDE_CBC_ENC_TEST_VEC -- } -+ .enc = __VECS(hmac_sha1_des3_ede_cbc_enc_tv_temp) - } - } - }, { -@@ -2245,18 +2554,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_aead, - .suite = { - .aead = { -- .enc = { -- .vecs = -- hmac_sha1_ecb_cipher_null_enc_tv_temp, -- .count = -- HMAC_SHA1_ECB_CIPHER_NULL_ENC_TEST_VEC -- }, -- .dec = { -- .vecs = -- hmac_sha1_ecb_cipher_null_dec_tv_temp, -- .count = -- HMAC_SHA1_ECB_CIPHER_NULL_DEC_TEST_VEC -- } -+ .enc = __VECS(hmac_sha1_ecb_cipher_null_enc_tv_temp), -+ .dec = __VECS(hmac_sha1_ecb_cipher_null_dec_tv_temp) - } - } - }, { -@@ -2268,12 +2567,7 @@ static const struct alg_test_desc alg_te - .test = alg_test_aead, - .suite = { - .aead = { -- .enc = { -- .vecs = -- hmac_sha224_des_cbc_enc_tv_temp, -- .count = -- HMAC_SHA224_DES_CBC_ENC_TEST_VEC -- } -+ .enc = __VECS(hmac_sha224_des_cbc_enc_tv_temp) - } - } - }, { -@@ -2282,12 +2576,7 @@ static const struct alg_test_desc alg_te - .fips_allowed = 1, - .suite = { - .aead = { -- .enc = { -- .vecs = -- hmac_sha224_des3_ede_cbc_enc_tv_temp, -- .count = -- HMAC_SHA224_DES3_EDE_CBC_ENC_TEST_VEC -- } -+ .enc = __VECS(hmac_sha224_des3_ede_cbc_enc_tv_temp) - } - } - }, { -@@ -2296,12 +2585,7 @@ static const struct alg_test_desc alg_te - .fips_allowed = 1, - .suite = { - .aead = { -- .enc = { -- .vecs = -- hmac_sha256_aes_cbc_enc_tv_temp, -- .count = -- HMAC_SHA256_AES_CBC_ENC_TEST_VEC -- } -+ .enc = __VECS(hmac_sha256_aes_cbc_enc_tv_temp) - } - } - }, { -@@ -2309,12 +2593,7 @@ static const struct alg_test_desc alg_te - .test = alg_test_aead, - .suite = { - .aead = { -- .enc = { -- .vecs = -- hmac_sha256_des_cbc_enc_tv_temp, -- .count = -- HMAC_SHA256_DES_CBC_ENC_TEST_VEC -- } -+ .enc = __VECS(hmac_sha256_des_cbc_enc_tv_temp) - } - } - }, { -@@ -2323,12 +2602,7 @@ static const struct alg_test_desc alg_te - .fips_allowed = 1, - .suite = { - .aead = { -- .enc = { -- .vecs = -- hmac_sha256_des3_ede_cbc_enc_tv_temp, -- .count = -- HMAC_SHA256_DES3_EDE_CBC_ENC_TEST_VEC -- } -+ .enc = __VECS(hmac_sha256_des3_ede_cbc_enc_tv_temp) - } - } - }, { -@@ -2344,12 +2618,7 @@ static const struct alg_test_desc alg_te - .test = alg_test_aead, - .suite = { - .aead = { -- .enc = { -- .vecs = -- hmac_sha384_des_cbc_enc_tv_temp, -- .count = -- HMAC_SHA384_DES_CBC_ENC_TEST_VEC -- } -+ .enc = __VECS(hmac_sha384_des_cbc_enc_tv_temp) - } - } - }, { -@@ -2358,12 +2627,7 @@ static const struct alg_test_desc alg_te - .fips_allowed = 1, - .suite = { - .aead = { -- .enc = { -- .vecs = -- hmac_sha384_des3_ede_cbc_enc_tv_temp, -- .count = -- HMAC_SHA384_DES3_EDE_CBC_ENC_TEST_VEC -- } -+ .enc = __VECS(hmac_sha384_des3_ede_cbc_enc_tv_temp) - } - } - }, { -@@ -2380,12 +2644,7 @@ static const struct alg_test_desc alg_te - .test = alg_test_aead, - .suite = { - .aead = { -- .enc = { -- .vecs = -- hmac_sha512_aes_cbc_enc_tv_temp, -- .count = -- HMAC_SHA512_AES_CBC_ENC_TEST_VEC -- } -+ .enc = __VECS(hmac_sha512_aes_cbc_enc_tv_temp) - } - } - }, { -@@ -2393,12 +2652,7 @@ static const struct alg_test_desc alg_te - .test = alg_test_aead, - .suite = { - .aead = { -- .enc = { -- .vecs = -- hmac_sha512_des_cbc_enc_tv_temp, -- .count = -- HMAC_SHA512_DES_CBC_ENC_TEST_VEC -- } -+ .enc = __VECS(hmac_sha512_des_cbc_enc_tv_temp) - } - } - }, { -@@ -2407,12 +2661,7 @@ static const struct alg_test_desc alg_te - .fips_allowed = 1, - .suite = { - .aead = { -- .enc = { -- .vecs = -- hmac_sha512_des3_ede_cbc_enc_tv_temp, -- .count = -- HMAC_SHA512_DES3_EDE_CBC_ENC_TEST_VEC -- } -+ .enc = __VECS(hmac_sha512_des3_ede_cbc_enc_tv_temp) - } - } - }, { -@@ -2429,14 +2678,8 @@ static const struct alg_test_desc alg_te - .fips_allowed = 1, - .suite = { - .cipher = { -- .enc = { -- .vecs = aes_cbc_enc_tv_template, -- .count = AES_CBC_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = aes_cbc_dec_tv_template, -- .count = AES_CBC_DEC_TEST_VECTORS -- } -+ .enc = __VECS(aes_cbc_enc_tv_template), -+ .dec = __VECS(aes_cbc_dec_tv_template) - } - } - }, { -@@ -2444,14 +2687,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = anubis_cbc_enc_tv_template, -- .count = ANUBIS_CBC_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = anubis_cbc_dec_tv_template, -- .count = ANUBIS_CBC_DEC_TEST_VECTORS -- } -+ .enc = __VECS(anubis_cbc_enc_tv_template), -+ .dec = __VECS(anubis_cbc_dec_tv_template) - } - } - }, { -@@ -2459,14 +2696,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = bf_cbc_enc_tv_template, -- .count = BF_CBC_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = bf_cbc_dec_tv_template, -- .count = BF_CBC_DEC_TEST_VECTORS -- } -+ .enc = __VECS(bf_cbc_enc_tv_template), -+ .dec = __VECS(bf_cbc_dec_tv_template) - } - } - }, { -@@ -2474,14 +2705,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = camellia_cbc_enc_tv_template, -- .count = CAMELLIA_CBC_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = camellia_cbc_dec_tv_template, -- .count = CAMELLIA_CBC_DEC_TEST_VECTORS -- } -+ .enc = __VECS(camellia_cbc_enc_tv_template), -+ .dec = __VECS(camellia_cbc_dec_tv_template) - } - } - }, { -@@ -2489,14 +2714,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = cast5_cbc_enc_tv_template, -- .count = CAST5_CBC_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = cast5_cbc_dec_tv_template, -- .count = CAST5_CBC_DEC_TEST_VECTORS -- } -+ .enc = __VECS(cast5_cbc_enc_tv_template), -+ .dec = __VECS(cast5_cbc_dec_tv_template) - } - } - }, { -@@ -2504,14 +2723,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = cast6_cbc_enc_tv_template, -- .count = CAST6_CBC_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = cast6_cbc_dec_tv_template, -- .count = CAST6_CBC_DEC_TEST_VECTORS -- } -+ .enc = __VECS(cast6_cbc_enc_tv_template), -+ .dec = __VECS(cast6_cbc_dec_tv_template) - } - } - }, { -@@ -2519,14 +2732,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = des_cbc_enc_tv_template, -- .count = DES_CBC_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = des_cbc_dec_tv_template, -- .count = DES_CBC_DEC_TEST_VECTORS -- } -+ .enc = __VECS(des_cbc_enc_tv_template), -+ .dec = __VECS(des_cbc_dec_tv_template) - } - } - }, { -@@ -2535,14 +2742,8 @@ static const struct alg_test_desc alg_te - .fips_allowed = 1, - .suite = { - .cipher = { -- .enc = { -- .vecs = des3_ede_cbc_enc_tv_template, -- .count = DES3_EDE_CBC_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = des3_ede_cbc_dec_tv_template, -- .count = DES3_EDE_CBC_DEC_TEST_VECTORS -- } -+ .enc = __VECS(des3_ede_cbc_enc_tv_template), -+ .dec = __VECS(des3_ede_cbc_dec_tv_template) - } - } - }, { -@@ -2550,14 +2751,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = serpent_cbc_enc_tv_template, -- .count = SERPENT_CBC_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = serpent_cbc_dec_tv_template, -- .count = SERPENT_CBC_DEC_TEST_VECTORS -- } -+ .enc = __VECS(serpent_cbc_enc_tv_template), -+ .dec = __VECS(serpent_cbc_dec_tv_template) - } - } - }, { -@@ -2565,30 +2760,25 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = tf_cbc_enc_tv_template, -- .count = TF_CBC_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = tf_cbc_dec_tv_template, -- .count = TF_CBC_DEC_TEST_VECTORS -- } -+ .enc = __VECS(tf_cbc_enc_tv_template), -+ .dec = __VECS(tf_cbc_dec_tv_template) - } - } - }, { -+ .alg = "cbcmac(aes)", -+ .fips_allowed = 1, -+ .test = alg_test_hash, -+ .suite = { -+ .hash = __VECS(aes_cbcmac_tv_template) -+ } -+ }, { - .alg = "ccm(aes)", - .test = alg_test_aead, - .fips_allowed = 1, - .suite = { - .aead = { -- .enc = { -- .vecs = aes_ccm_enc_tv_template, -- .count = AES_CCM_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = aes_ccm_dec_tv_template, -- .count = AES_CCM_DEC_TEST_VECTORS -- } -+ .enc = __VECS(aes_ccm_enc_tv_template), -+ .dec = __VECS(aes_ccm_dec_tv_template) - } - } - }, { -@@ -2596,14 +2786,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = chacha20_enc_tv_template, -- .count = CHACHA20_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = chacha20_enc_tv_template, -- .count = CHACHA20_ENC_TEST_VECTORS -- }, -+ .enc = __VECS(chacha20_enc_tv_template), -+ .dec = __VECS(chacha20_enc_tv_template), - } - } - }, { -@@ -2611,20 +2795,14 @@ static const struct alg_test_desc alg_te - .fips_allowed = 1, - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = aes_cmac128_tv_template, -- .count = CMAC_AES_TEST_VECTORS -- } -+ .hash = __VECS(aes_cmac128_tv_template) - } - }, { - .alg = "cmac(des3_ede)", - .fips_allowed = 1, - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = des3_ede_cmac64_tv_template, -- .count = CMAC_DES3_EDE_TEST_VECTORS -- } -+ .hash = __VECS(des3_ede_cmac64_tv_template) - } - }, { - .alg = "compress_null", -@@ -2633,94 +2811,30 @@ static const struct alg_test_desc alg_te - .alg = "crc32", - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = crc32_tv_template, -- .count = CRC32_TEST_VECTORS -- } -+ .hash = __VECS(crc32_tv_template) - } - }, { - .alg = "crc32c", - .test = alg_test_crc32c, - .fips_allowed = 1, - .suite = { -- .hash = { -- .vecs = crc32c_tv_template, -- .count = CRC32C_TEST_VECTORS -- } -+ .hash = __VECS(crc32c_tv_template) - } - }, { - .alg = "crct10dif", - .test = alg_test_hash, - .fips_allowed = 1, - .suite = { -- .hash = { -- .vecs = crct10dif_tv_template, -- .count = CRCT10DIF_TEST_VECTORS -- } -+ .hash = __VECS(crct10dif_tv_template) - } - }, { -- .alg = "cryptd(__driver-cbc-aes-aesni)", -- .test = alg_test_null, -- .fips_allowed = 1, -- }, { -- .alg = "cryptd(__driver-cbc-camellia-aesni)", -- .test = alg_test_null, -- }, { -- .alg = "cryptd(__driver-cbc-camellia-aesni-avx2)", -- .test = alg_test_null, -- }, { -- .alg = "cryptd(__driver-cbc-serpent-avx2)", -- .test = alg_test_null, -- }, { -- .alg = "cryptd(__driver-ecb-aes-aesni)", -- .test = alg_test_null, -- .fips_allowed = 1, -- }, { -- .alg = "cryptd(__driver-ecb-camellia-aesni)", -- .test = alg_test_null, -- }, { -- .alg = "cryptd(__driver-ecb-camellia-aesni-avx2)", -- .test = alg_test_null, -- }, { -- .alg = "cryptd(__driver-ecb-cast5-avx)", -- .test = alg_test_null, -- }, { -- .alg = "cryptd(__driver-ecb-cast6-avx)", -- .test = alg_test_null, -- }, { -- .alg = "cryptd(__driver-ecb-serpent-avx)", -- .test = alg_test_null, -- }, { -- .alg = "cryptd(__driver-ecb-serpent-avx2)", -- .test = alg_test_null, -- }, { -- .alg = "cryptd(__driver-ecb-serpent-sse2)", -- .test = alg_test_null, -- }, { -- .alg = "cryptd(__driver-ecb-twofish-avx)", -- .test = alg_test_null, -- }, { -- .alg = "cryptd(__driver-gcm-aes-aesni)", -- .test = alg_test_null, -- .fips_allowed = 1, -- }, { -- .alg = "cryptd(__ghash-pclmulqdqni)", -- .test = alg_test_null, -- .fips_allowed = 1, -- }, { - .alg = "ctr(aes)", - .test = alg_test_skcipher, - .fips_allowed = 1, - .suite = { - .cipher = { -- .enc = { -- .vecs = aes_ctr_enc_tv_template, -- .count = AES_CTR_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = aes_ctr_dec_tv_template, -- .count = AES_CTR_DEC_TEST_VECTORS -- } -+ .enc = __VECS(aes_ctr_enc_tv_template), -+ .dec = __VECS(aes_ctr_dec_tv_template) - } - } - }, { -@@ -2728,14 +2842,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = bf_ctr_enc_tv_template, -- .count = BF_CTR_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = bf_ctr_dec_tv_template, -- .count = BF_CTR_DEC_TEST_VECTORS -- } -+ .enc = __VECS(bf_ctr_enc_tv_template), -+ .dec = __VECS(bf_ctr_dec_tv_template) - } - } - }, { -@@ -2743,14 +2851,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = camellia_ctr_enc_tv_template, -- .count = CAMELLIA_CTR_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = camellia_ctr_dec_tv_template, -- .count = CAMELLIA_CTR_DEC_TEST_VECTORS -- } -+ .enc = __VECS(camellia_ctr_enc_tv_template), -+ .dec = __VECS(camellia_ctr_dec_tv_template) - } - } - }, { -@@ -2758,14 +2860,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = cast5_ctr_enc_tv_template, -- .count = CAST5_CTR_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = cast5_ctr_dec_tv_template, -- .count = CAST5_CTR_DEC_TEST_VECTORS -- } -+ .enc = __VECS(cast5_ctr_enc_tv_template), -+ .dec = __VECS(cast5_ctr_dec_tv_template) - } - } - }, { -@@ -2773,14 +2869,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = cast6_ctr_enc_tv_template, -- .count = CAST6_CTR_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = cast6_ctr_dec_tv_template, -- .count = CAST6_CTR_DEC_TEST_VECTORS -- } -+ .enc = __VECS(cast6_ctr_enc_tv_template), -+ .dec = __VECS(cast6_ctr_dec_tv_template) - } - } - }, { -@@ -2788,29 +2878,18 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = des_ctr_enc_tv_template, -- .count = DES_CTR_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = des_ctr_dec_tv_template, -- .count = DES_CTR_DEC_TEST_VECTORS -- } -+ .enc = __VECS(des_ctr_enc_tv_template), -+ .dec = __VECS(des_ctr_dec_tv_template) - } - } - }, { - .alg = "ctr(des3_ede)", - .test = alg_test_skcipher, -+ .fips_allowed = 1, - .suite = { - .cipher = { -- .enc = { -- .vecs = des3_ede_ctr_enc_tv_template, -- .count = DES3_EDE_CTR_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = des3_ede_ctr_dec_tv_template, -- .count = DES3_EDE_CTR_DEC_TEST_VECTORS -- } -+ .enc = __VECS(des3_ede_ctr_enc_tv_template), -+ .dec = __VECS(des3_ede_ctr_dec_tv_template) - } - } - }, { -@@ -2818,14 +2897,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = serpent_ctr_enc_tv_template, -- .count = SERPENT_CTR_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = serpent_ctr_dec_tv_template, -- .count = SERPENT_CTR_DEC_TEST_VECTORS -- } -+ .enc = __VECS(serpent_ctr_enc_tv_template), -+ .dec = __VECS(serpent_ctr_dec_tv_template) - } - } - }, { -@@ -2833,14 +2906,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = tf_ctr_enc_tv_template, -- .count = TF_CTR_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = tf_ctr_dec_tv_template, -- .count = TF_CTR_DEC_TEST_VECTORS -- } -+ .enc = __VECS(tf_ctr_enc_tv_template), -+ .dec = __VECS(tf_ctr_dec_tv_template) - } - } - }, { -@@ -2848,14 +2915,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = cts_mode_enc_tv_template, -- .count = CTS_MODE_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = cts_mode_dec_tv_template, -- .count = CTS_MODE_DEC_TEST_VECTORS -- } -+ .enc = __VECS(cts_mode_enc_tv_template), -+ .dec = __VECS(cts_mode_dec_tv_template) - } - } - }, { -@@ -2864,14 +2925,8 @@ static const struct alg_test_desc alg_te - .fips_allowed = 1, - .suite = { - .comp = { -- .comp = { -- .vecs = deflate_comp_tv_template, -- .count = DEFLATE_COMP_TEST_VECTORS -- }, -- .decomp = { -- .vecs = deflate_decomp_tv_template, -- .count = DEFLATE_DECOMP_TEST_VECTORS -- } -+ .comp = __VECS(deflate_comp_tv_template), -+ .decomp = __VECS(deflate_decomp_tv_template) - } - } - }, { -@@ -2879,10 +2934,7 @@ static const struct alg_test_desc alg_te - .test = alg_test_kpp, - .fips_allowed = 1, - .suite = { -- .kpp = { -- .vecs = dh_tv_template, -- .count = DH_TEST_VECTORS -- } -+ .kpp = __VECS(dh_tv_template) - } - }, { - .alg = "digest_null", -@@ -2892,30 +2944,21 @@ static const struct alg_test_desc alg_te - .test = alg_test_drbg, - .fips_allowed = 1, - .suite = { -- .drbg = { -- .vecs = drbg_nopr_ctr_aes128_tv_template, -- .count = ARRAY_SIZE(drbg_nopr_ctr_aes128_tv_template) -- } -+ .drbg = __VECS(drbg_nopr_ctr_aes128_tv_template) - } - }, { - .alg = "drbg_nopr_ctr_aes192", - .test = alg_test_drbg, - .fips_allowed = 1, - .suite = { -- .drbg = { -- .vecs = drbg_nopr_ctr_aes192_tv_template, -- .count = ARRAY_SIZE(drbg_nopr_ctr_aes192_tv_template) -- } -+ .drbg = __VECS(drbg_nopr_ctr_aes192_tv_template) - } - }, { - .alg = "drbg_nopr_ctr_aes256", - .test = alg_test_drbg, - .fips_allowed = 1, - .suite = { -- .drbg = { -- .vecs = drbg_nopr_ctr_aes256_tv_template, -- .count = ARRAY_SIZE(drbg_nopr_ctr_aes256_tv_template) -- } -+ .drbg = __VECS(drbg_nopr_ctr_aes256_tv_template) - } - }, { - /* -@@ -2930,11 +2973,7 @@ static const struct alg_test_desc alg_te - .test = alg_test_drbg, - .fips_allowed = 1, - .suite = { -- .drbg = { -- .vecs = drbg_nopr_hmac_sha256_tv_template, -- .count = -- ARRAY_SIZE(drbg_nopr_hmac_sha256_tv_template) -- } -+ .drbg = __VECS(drbg_nopr_hmac_sha256_tv_template) - } - }, { - /* covered by drbg_nopr_hmac_sha256 test */ -@@ -2954,10 +2993,7 @@ static const struct alg_test_desc alg_te - .test = alg_test_drbg, - .fips_allowed = 1, - .suite = { -- .drbg = { -- .vecs = drbg_nopr_sha256_tv_template, -- .count = ARRAY_SIZE(drbg_nopr_sha256_tv_template) -- } -+ .drbg = __VECS(drbg_nopr_sha256_tv_template) - } - }, { - /* covered by drbg_nopr_sha256 test */ -@@ -2973,10 +3009,7 @@ static const struct alg_test_desc alg_te - .test = alg_test_drbg, - .fips_allowed = 1, - .suite = { -- .drbg = { -- .vecs = drbg_pr_ctr_aes128_tv_template, -- .count = ARRAY_SIZE(drbg_pr_ctr_aes128_tv_template) -- } -+ .drbg = __VECS(drbg_pr_ctr_aes128_tv_template) - } - }, { - /* covered by drbg_pr_ctr_aes128 test */ -@@ -2996,10 +3029,7 @@ static const struct alg_test_desc alg_te - .test = alg_test_drbg, - .fips_allowed = 1, - .suite = { -- .drbg = { -- .vecs = drbg_pr_hmac_sha256_tv_template, -- .count = ARRAY_SIZE(drbg_pr_hmac_sha256_tv_template) -- } -+ .drbg = __VECS(drbg_pr_hmac_sha256_tv_template) - } - }, { - /* covered by drbg_pr_hmac_sha256 test */ -@@ -3019,10 +3049,7 @@ static const struct alg_test_desc alg_te - .test = alg_test_drbg, - .fips_allowed = 1, - .suite = { -- .drbg = { -- .vecs = drbg_pr_sha256_tv_template, -- .count = ARRAY_SIZE(drbg_pr_sha256_tv_template) -- } -+ .drbg = __VECS(drbg_pr_sha256_tv_template) - } - }, { - /* covered by drbg_pr_sha256 test */ -@@ -3034,23 +3061,13 @@ static const struct alg_test_desc alg_te - .fips_allowed = 1, - .test = alg_test_null, - }, { -- .alg = "ecb(__aes-aesni)", -- .test = alg_test_null, -- .fips_allowed = 1, -- }, { - .alg = "ecb(aes)", - .test = alg_test_skcipher, - .fips_allowed = 1, - .suite = { - .cipher = { -- .enc = { -- .vecs = aes_enc_tv_template, -- .count = AES_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = aes_dec_tv_template, -- .count = AES_DEC_TEST_VECTORS -- } -+ .enc = __VECS(aes_enc_tv_template), -+ .dec = __VECS(aes_dec_tv_template) - } - } - }, { -@@ -3058,14 +3075,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = anubis_enc_tv_template, -- .count = ANUBIS_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = anubis_dec_tv_template, -- .count = ANUBIS_DEC_TEST_VECTORS -- } -+ .enc = __VECS(anubis_enc_tv_template), -+ .dec = __VECS(anubis_dec_tv_template) - } - } - }, { -@@ -3073,14 +3084,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = arc4_enc_tv_template, -- .count = ARC4_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = arc4_dec_tv_template, -- .count = ARC4_DEC_TEST_VECTORS -- } -+ .enc = __VECS(arc4_enc_tv_template), -+ .dec = __VECS(arc4_dec_tv_template) - } - } - }, { -@@ -3088,14 +3093,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = bf_enc_tv_template, -- .count = BF_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = bf_dec_tv_template, -- .count = BF_DEC_TEST_VECTORS -- } -+ .enc = __VECS(bf_enc_tv_template), -+ .dec = __VECS(bf_dec_tv_template) - } - } - }, { -@@ -3103,14 +3102,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = camellia_enc_tv_template, -- .count = CAMELLIA_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = camellia_dec_tv_template, -- .count = CAMELLIA_DEC_TEST_VECTORS -- } -+ .enc = __VECS(camellia_enc_tv_template), -+ .dec = __VECS(camellia_dec_tv_template) - } - } - }, { -@@ -3118,14 +3111,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = cast5_enc_tv_template, -- .count = CAST5_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = cast5_dec_tv_template, -- .count = CAST5_DEC_TEST_VECTORS -- } -+ .enc = __VECS(cast5_enc_tv_template), -+ .dec = __VECS(cast5_dec_tv_template) - } - } - }, { -@@ -3133,14 +3120,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = cast6_enc_tv_template, -- .count = CAST6_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = cast6_dec_tv_template, -- .count = CAST6_DEC_TEST_VECTORS -- } -+ .enc = __VECS(cast6_enc_tv_template), -+ .dec = __VECS(cast6_dec_tv_template) - } - } - }, { -@@ -3151,14 +3132,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = des_enc_tv_template, -- .count = DES_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = des_dec_tv_template, -- .count = DES_DEC_TEST_VECTORS -- } -+ .enc = __VECS(des_enc_tv_template), -+ .dec = __VECS(des_dec_tv_template) - } - } - }, { -@@ -3167,14 +3142,8 @@ static const struct alg_test_desc alg_te - .fips_allowed = 1, - .suite = { - .cipher = { -- .enc = { -- .vecs = des3_ede_enc_tv_template, -- .count = DES3_EDE_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = des3_ede_dec_tv_template, -- .count = DES3_EDE_DEC_TEST_VECTORS -- } -+ .enc = __VECS(des3_ede_enc_tv_template), -+ .dec = __VECS(des3_ede_dec_tv_template) - } - } - }, { -@@ -3197,14 +3166,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = khazad_enc_tv_template, -- .count = KHAZAD_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = khazad_dec_tv_template, -- .count = KHAZAD_DEC_TEST_VECTORS -- } -+ .enc = __VECS(khazad_enc_tv_template), -+ .dec = __VECS(khazad_dec_tv_template) - } - } - }, { -@@ -3212,14 +3175,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = seed_enc_tv_template, -- .count = SEED_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = seed_dec_tv_template, -- .count = SEED_DEC_TEST_VECTORS -- } -+ .enc = __VECS(seed_enc_tv_template), -+ .dec = __VECS(seed_dec_tv_template) - } - } - }, { -@@ -3227,14 +3184,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = serpent_enc_tv_template, -- .count = SERPENT_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = serpent_dec_tv_template, -- .count = SERPENT_DEC_TEST_VECTORS -- } -+ .enc = __VECS(serpent_enc_tv_template), -+ .dec = __VECS(serpent_dec_tv_template) - } - } - }, { -@@ -3242,14 +3193,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = tea_enc_tv_template, -- .count = TEA_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = tea_dec_tv_template, -- .count = TEA_DEC_TEST_VECTORS -- } -+ .enc = __VECS(tea_enc_tv_template), -+ .dec = __VECS(tea_dec_tv_template) - } - } - }, { -@@ -3257,14 +3202,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = tnepres_enc_tv_template, -- .count = TNEPRES_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = tnepres_dec_tv_template, -- .count = TNEPRES_DEC_TEST_VECTORS -- } -+ .enc = __VECS(tnepres_enc_tv_template), -+ .dec = __VECS(tnepres_dec_tv_template) - } - } - }, { -@@ -3272,14 +3211,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = tf_enc_tv_template, -- .count = TF_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = tf_dec_tv_template, -- .count = TF_DEC_TEST_VECTORS -- } -+ .enc = __VECS(tf_enc_tv_template), -+ .dec = __VECS(tf_dec_tv_template) - } - } - }, { -@@ -3287,14 +3220,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = xeta_enc_tv_template, -- .count = XETA_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = xeta_dec_tv_template, -- .count = XETA_DEC_TEST_VECTORS -- } -+ .enc = __VECS(xeta_enc_tv_template), -+ .dec = __VECS(xeta_dec_tv_template) - } - } - }, { -@@ -3302,14 +3229,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = xtea_enc_tv_template, -- .count = XTEA_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = xtea_dec_tv_template, -- .count = XTEA_DEC_TEST_VECTORS -- } -+ .enc = __VECS(xtea_enc_tv_template), -+ .dec = __VECS(xtea_dec_tv_template) - } - } - }, { -@@ -3317,10 +3238,7 @@ static const struct alg_test_desc alg_te - .test = alg_test_kpp, - .fips_allowed = 1, - .suite = { -- .kpp = { -- .vecs = ecdh_tv_template, -- .count = ECDH_TEST_VECTORS -- } -+ .kpp = __VECS(ecdh_tv_template) - } - }, { - .alg = "gcm(aes)", -@@ -3328,14 +3246,8 @@ static const struct alg_test_desc alg_te - .fips_allowed = 1, - .suite = { - .aead = { -- .enc = { -- .vecs = aes_gcm_enc_tv_template, -- .count = AES_GCM_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = aes_gcm_dec_tv_template, -- .count = AES_GCM_DEC_TEST_VECTORS -- } -+ .enc = __VECS(aes_gcm_enc_tv_template), -+ .dec = __VECS(aes_gcm_dec_tv_template) - } - } - }, { -@@ -3343,136 +3255,94 @@ static const struct alg_test_desc alg_te - .test = alg_test_hash, - .fips_allowed = 1, - .suite = { -- .hash = { -- .vecs = ghash_tv_template, -- .count = GHASH_TEST_VECTORS -- } -+ .hash = __VECS(ghash_tv_template) - } - }, { - .alg = "hmac(crc32)", - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = bfin_crc_tv_template, -- .count = BFIN_CRC_TEST_VECTORS -- } -+ .hash = __VECS(bfin_crc_tv_template) - } - }, { - .alg = "hmac(md5)", - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = hmac_md5_tv_template, -- .count = HMAC_MD5_TEST_VECTORS -- } -+ .hash = __VECS(hmac_md5_tv_template) - } - }, { - .alg = "hmac(rmd128)", - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = hmac_rmd128_tv_template, -- .count = HMAC_RMD128_TEST_VECTORS -- } -+ .hash = __VECS(hmac_rmd128_tv_template) - } - }, { - .alg = "hmac(rmd160)", - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = hmac_rmd160_tv_template, -- .count = HMAC_RMD160_TEST_VECTORS -- } -+ .hash = __VECS(hmac_rmd160_tv_template) - } - }, { - .alg = "hmac(sha1)", - .test = alg_test_hash, - .fips_allowed = 1, - .suite = { -- .hash = { -- .vecs = hmac_sha1_tv_template, -- .count = HMAC_SHA1_TEST_VECTORS -- } -+ .hash = __VECS(hmac_sha1_tv_template) - } - }, { - .alg = "hmac(sha224)", - .test = alg_test_hash, - .fips_allowed = 1, - .suite = { -- .hash = { -- .vecs = hmac_sha224_tv_template, -- .count = HMAC_SHA224_TEST_VECTORS -- } -+ .hash = __VECS(hmac_sha224_tv_template) - } - }, { - .alg = "hmac(sha256)", - .test = alg_test_hash, - .fips_allowed = 1, - .suite = { -- .hash = { -- .vecs = hmac_sha256_tv_template, -- .count = HMAC_SHA256_TEST_VECTORS -- } -+ .hash = __VECS(hmac_sha256_tv_template) - } - }, { - .alg = "hmac(sha3-224)", - .test = alg_test_hash, - .fips_allowed = 1, - .suite = { -- .hash = { -- .vecs = hmac_sha3_224_tv_template, -- .count = HMAC_SHA3_224_TEST_VECTORS -- } -+ .hash = __VECS(hmac_sha3_224_tv_template) - } - }, { - .alg = "hmac(sha3-256)", - .test = alg_test_hash, - .fips_allowed = 1, - .suite = { -- .hash = { -- .vecs = hmac_sha3_256_tv_template, -- .count = HMAC_SHA3_256_TEST_VECTORS -- } -+ .hash = __VECS(hmac_sha3_256_tv_template) - } - }, { - .alg = "hmac(sha3-384)", - .test = alg_test_hash, - .fips_allowed = 1, - .suite = { -- .hash = { -- .vecs = hmac_sha3_384_tv_template, -- .count = HMAC_SHA3_384_TEST_VECTORS -- } -+ .hash = __VECS(hmac_sha3_384_tv_template) - } - }, { - .alg = "hmac(sha3-512)", - .test = alg_test_hash, - .fips_allowed = 1, - .suite = { -- .hash = { -- .vecs = hmac_sha3_512_tv_template, -- .count = HMAC_SHA3_512_TEST_VECTORS -- } -+ .hash = __VECS(hmac_sha3_512_tv_template) - } - }, { - .alg = "hmac(sha384)", - .test = alg_test_hash, - .fips_allowed = 1, - .suite = { -- .hash = { -- .vecs = hmac_sha384_tv_template, -- .count = HMAC_SHA384_TEST_VECTORS -- } -+ .hash = __VECS(hmac_sha384_tv_template) - } - }, { - .alg = "hmac(sha512)", - .test = alg_test_hash, - .fips_allowed = 1, - .suite = { -- .hash = { -- .vecs = hmac_sha512_tv_template, -- .count = HMAC_SHA512_TEST_VECTORS -- } -+ .hash = __VECS(hmac_sha512_tv_template) - } - }, { - .alg = "jitterentropy_rng", -@@ -3484,14 +3354,8 @@ static const struct alg_test_desc alg_te - .fips_allowed = 1, - .suite = { - .cipher = { -- .enc = { -- .vecs = aes_kw_enc_tv_template, -- .count = ARRAY_SIZE(aes_kw_enc_tv_template) -- }, -- .dec = { -- .vecs = aes_kw_dec_tv_template, -- .count = ARRAY_SIZE(aes_kw_dec_tv_template) -- } -+ .enc = __VECS(aes_kw_enc_tv_template), -+ .dec = __VECS(aes_kw_dec_tv_template) - } - } - }, { -@@ -3499,14 +3363,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = aes_lrw_enc_tv_template, -- .count = AES_LRW_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = aes_lrw_dec_tv_template, -- .count = AES_LRW_DEC_TEST_VECTORS -- } -+ .enc = __VECS(aes_lrw_enc_tv_template), -+ .dec = __VECS(aes_lrw_dec_tv_template) - } - } - }, { -@@ -3514,14 +3372,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = camellia_lrw_enc_tv_template, -- .count = CAMELLIA_LRW_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = camellia_lrw_dec_tv_template, -- .count = CAMELLIA_LRW_DEC_TEST_VECTORS -- } -+ .enc = __VECS(camellia_lrw_enc_tv_template), -+ .dec = __VECS(camellia_lrw_dec_tv_template) - } - } - }, { -@@ -3529,14 +3381,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = cast6_lrw_enc_tv_template, -- .count = CAST6_LRW_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = cast6_lrw_dec_tv_template, -- .count = CAST6_LRW_DEC_TEST_VECTORS -- } -+ .enc = __VECS(cast6_lrw_enc_tv_template), -+ .dec = __VECS(cast6_lrw_dec_tv_template) - } - } - }, { -@@ -3544,14 +3390,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = serpent_lrw_enc_tv_template, -- .count = SERPENT_LRW_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = serpent_lrw_dec_tv_template, -- .count = SERPENT_LRW_DEC_TEST_VECTORS -- } -+ .enc = __VECS(serpent_lrw_enc_tv_template), -+ .dec = __VECS(serpent_lrw_dec_tv_template) - } - } - }, { -@@ -3559,14 +3399,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = tf_lrw_enc_tv_template, -- .count = TF_LRW_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = tf_lrw_dec_tv_template, -- .count = TF_LRW_DEC_TEST_VECTORS -- } -+ .enc = __VECS(tf_lrw_enc_tv_template), -+ .dec = __VECS(tf_lrw_dec_tv_template) - } - } - }, { -@@ -3575,14 +3409,8 @@ static const struct alg_test_desc alg_te - .fips_allowed = 1, - .suite = { - .comp = { -- .comp = { -- .vecs = lz4_comp_tv_template, -- .count = LZ4_COMP_TEST_VECTORS -- }, -- .decomp = { -- .vecs = lz4_decomp_tv_template, -- .count = LZ4_DECOMP_TEST_VECTORS -- } -+ .comp = __VECS(lz4_comp_tv_template), -+ .decomp = __VECS(lz4_decomp_tv_template) - } - } - }, { -@@ -3591,14 +3419,8 @@ static const struct alg_test_desc alg_te - .fips_allowed = 1, - .suite = { - .comp = { -- .comp = { -- .vecs = lz4hc_comp_tv_template, -- .count = LZ4HC_COMP_TEST_VECTORS -- }, -- .decomp = { -- .vecs = lz4hc_decomp_tv_template, -- .count = LZ4HC_DECOMP_TEST_VECTORS -- } -+ .comp = __VECS(lz4hc_comp_tv_template), -+ .decomp = __VECS(lz4hc_decomp_tv_template) - } - } - }, { -@@ -3607,42 +3429,27 @@ static const struct alg_test_desc alg_te - .fips_allowed = 1, - .suite = { - .comp = { -- .comp = { -- .vecs = lzo_comp_tv_template, -- .count = LZO_COMP_TEST_VECTORS -- }, -- .decomp = { -- .vecs = lzo_decomp_tv_template, -- .count = LZO_DECOMP_TEST_VECTORS -- } -+ .comp = __VECS(lzo_comp_tv_template), -+ .decomp = __VECS(lzo_decomp_tv_template) - } - } - }, { - .alg = "md4", - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = md4_tv_template, -- .count = MD4_TEST_VECTORS -- } -+ .hash = __VECS(md4_tv_template) - } - }, { - .alg = "md5", - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = md5_tv_template, -- .count = MD5_TEST_VECTORS -- } -+ .hash = __VECS(md5_tv_template) - } - }, { - .alg = "michael_mic", - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = michael_mic_tv_template, -- .count = MICHAEL_MIC_TEST_VECTORS -- } -+ .hash = __VECS(michael_mic_tv_template) - } - }, { - .alg = "ofb(aes)", -@@ -3650,14 +3457,8 @@ static const struct alg_test_desc alg_te - .fips_allowed = 1, - .suite = { - .cipher = { -- .enc = { -- .vecs = aes_ofb_enc_tv_template, -- .count = AES_OFB_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = aes_ofb_dec_tv_template, -- .count = AES_OFB_DEC_TEST_VECTORS -- } -+ .enc = __VECS(aes_ofb_enc_tv_template), -+ .dec = __VECS(aes_ofb_dec_tv_template) - } - } - }, { -@@ -3665,24 +3466,15 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = fcrypt_pcbc_enc_tv_template, -- .count = FCRYPT_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = fcrypt_pcbc_dec_tv_template, -- .count = FCRYPT_DEC_TEST_VECTORS -- } -+ .enc = __VECS(fcrypt_pcbc_enc_tv_template), -+ .dec = __VECS(fcrypt_pcbc_dec_tv_template) - } - } - }, { - .alg = "poly1305", - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = poly1305_tv_template, -- .count = POLY1305_TEST_VECTORS -- } -+ .hash = __VECS(poly1305_tv_template) - } - }, { - .alg = "rfc3686(ctr(aes))", -@@ -3690,14 +3482,8 @@ static const struct alg_test_desc alg_te - .fips_allowed = 1, - .suite = { - .cipher = { -- .enc = { -- .vecs = aes_ctr_rfc3686_enc_tv_template, -- .count = AES_CTR_3686_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = aes_ctr_rfc3686_dec_tv_template, -- .count = AES_CTR_3686_DEC_TEST_VECTORS -- } -+ .enc = __VECS(aes_ctr_rfc3686_enc_tv_template), -+ .dec = __VECS(aes_ctr_rfc3686_dec_tv_template) - } - } - }, { -@@ -3706,14 +3492,8 @@ static const struct alg_test_desc alg_te - .fips_allowed = 1, - .suite = { - .aead = { -- .enc = { -- .vecs = aes_gcm_rfc4106_enc_tv_template, -- .count = AES_GCM_4106_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = aes_gcm_rfc4106_dec_tv_template, -- .count = AES_GCM_4106_DEC_TEST_VECTORS -- } -+ .enc = __VECS(aes_gcm_rfc4106_enc_tv_template), -+ .dec = __VECS(aes_gcm_rfc4106_dec_tv_template) - } - } - }, { -@@ -3722,14 +3502,8 @@ static const struct alg_test_desc alg_te - .fips_allowed = 1, - .suite = { - .aead = { -- .enc = { -- .vecs = aes_ccm_rfc4309_enc_tv_template, -- .count = AES_CCM_4309_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = aes_ccm_rfc4309_dec_tv_template, -- .count = AES_CCM_4309_DEC_TEST_VECTORS -- } -+ .enc = __VECS(aes_ccm_rfc4309_enc_tv_template), -+ .dec = __VECS(aes_ccm_rfc4309_dec_tv_template) - } - } - }, { -@@ -3737,14 +3511,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_aead, - .suite = { - .aead = { -- .enc = { -- .vecs = aes_gcm_rfc4543_enc_tv_template, -- .count = AES_GCM_4543_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = aes_gcm_rfc4543_dec_tv_template, -- .count = AES_GCM_4543_DEC_TEST_VECTORS -- }, -+ .enc = __VECS(aes_gcm_rfc4543_enc_tv_template), -+ .dec = __VECS(aes_gcm_rfc4543_dec_tv_template), - } - } - }, { -@@ -3752,14 +3520,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_aead, - .suite = { - .aead = { -- .enc = { -- .vecs = rfc7539_enc_tv_template, -- .count = RFC7539_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = rfc7539_dec_tv_template, -- .count = RFC7539_DEC_TEST_VECTORS -- }, -+ .enc = __VECS(rfc7539_enc_tv_template), -+ .dec = __VECS(rfc7539_dec_tv_template), - } - } - }, { -@@ -3767,71 +3529,47 @@ static const struct alg_test_desc alg_te - .test = alg_test_aead, - .suite = { - .aead = { -- .enc = { -- .vecs = rfc7539esp_enc_tv_template, -- .count = RFC7539ESP_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = rfc7539esp_dec_tv_template, -- .count = RFC7539ESP_DEC_TEST_VECTORS -- }, -+ .enc = __VECS(rfc7539esp_enc_tv_template), -+ .dec = __VECS(rfc7539esp_dec_tv_template), - } - } - }, { - .alg = "rmd128", - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = rmd128_tv_template, -- .count = RMD128_TEST_VECTORS -- } -+ .hash = __VECS(rmd128_tv_template) - } - }, { - .alg = "rmd160", - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = rmd160_tv_template, -- .count = RMD160_TEST_VECTORS -- } -+ .hash = __VECS(rmd160_tv_template) - } - }, { - .alg = "rmd256", - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = rmd256_tv_template, -- .count = RMD256_TEST_VECTORS -- } -+ .hash = __VECS(rmd256_tv_template) - } - }, { - .alg = "rmd320", - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = rmd320_tv_template, -- .count = RMD320_TEST_VECTORS -- } -+ .hash = __VECS(rmd320_tv_template) - } - }, { - .alg = "rsa", - .test = alg_test_akcipher, - .fips_allowed = 1, - .suite = { -- .akcipher = { -- .vecs = rsa_tv_template, -- .count = RSA_TEST_VECTORS -- } -+ .akcipher = __VECS(rsa_tv_template) - } - }, { - .alg = "salsa20", - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = salsa20_stream_enc_tv_template, -- .count = SALSA20_STREAM_ENC_TEST_VECTORS -- } -+ .enc = __VECS(salsa20_stream_enc_tv_template) - } - } - }, { -@@ -3839,162 +3577,120 @@ static const struct alg_test_desc alg_te - .test = alg_test_hash, - .fips_allowed = 1, - .suite = { -- .hash = { -- .vecs = sha1_tv_template, -- .count = SHA1_TEST_VECTORS -- } -+ .hash = __VECS(sha1_tv_template) - } - }, { - .alg = "sha224", - .test = alg_test_hash, - .fips_allowed = 1, - .suite = { -- .hash = { -- .vecs = sha224_tv_template, -- .count = SHA224_TEST_VECTORS -- } -+ .hash = __VECS(sha224_tv_template) - } - }, { - .alg = "sha256", - .test = alg_test_hash, - .fips_allowed = 1, - .suite = { -- .hash = { -- .vecs = sha256_tv_template, -- .count = SHA256_TEST_VECTORS -- } -+ .hash = __VECS(sha256_tv_template) - } - }, { - .alg = "sha3-224", - .test = alg_test_hash, - .fips_allowed = 1, - .suite = { -- .hash = { -- .vecs = sha3_224_tv_template, -- .count = SHA3_224_TEST_VECTORS -- } -+ .hash = __VECS(sha3_224_tv_template) - } - }, { - .alg = "sha3-256", - .test = alg_test_hash, - .fips_allowed = 1, - .suite = { -- .hash = { -- .vecs = sha3_256_tv_template, -- .count = SHA3_256_TEST_VECTORS -- } -+ .hash = __VECS(sha3_256_tv_template) - } - }, { - .alg = "sha3-384", - .test = alg_test_hash, - .fips_allowed = 1, - .suite = { -- .hash = { -- .vecs = sha3_384_tv_template, -- .count = SHA3_384_TEST_VECTORS -- } -+ .hash = __VECS(sha3_384_tv_template) - } - }, { - .alg = "sha3-512", - .test = alg_test_hash, - .fips_allowed = 1, - .suite = { -- .hash = { -- .vecs = sha3_512_tv_template, -- .count = SHA3_512_TEST_VECTORS -- } -+ .hash = __VECS(sha3_512_tv_template) - } - }, { - .alg = "sha384", - .test = alg_test_hash, - .fips_allowed = 1, - .suite = { -- .hash = { -- .vecs = sha384_tv_template, -- .count = SHA384_TEST_VECTORS -- } -+ .hash = __VECS(sha384_tv_template) - } - }, { - .alg = "sha512", - .test = alg_test_hash, - .fips_allowed = 1, - .suite = { -- .hash = { -- .vecs = sha512_tv_template, -- .count = SHA512_TEST_VECTORS -- } -+ .hash = __VECS(sha512_tv_template) - } - }, { - .alg = "tgr128", - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = tgr128_tv_template, -- .count = TGR128_TEST_VECTORS -- } -+ .hash = __VECS(tgr128_tv_template) - } - }, { - .alg = "tgr160", - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = tgr160_tv_template, -- .count = TGR160_TEST_VECTORS -- } -+ .hash = __VECS(tgr160_tv_template) - } - }, { - .alg = "tgr192", - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = tgr192_tv_template, -- .count = TGR192_TEST_VECTORS -+ .hash = __VECS(tgr192_tv_template) -+ } -+ }, { -+ .alg = "tls10(hmac(sha1),cbc(aes))", -+ .test = alg_test_tls, -+ .suite = { -+ .tls = { -+ .enc = __VECS(tls_enc_tv_template), -+ .dec = __VECS(tls_dec_tv_template) - } - } - }, { - .alg = "vmac(aes)", - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = aes_vmac128_tv_template, -- .count = VMAC_AES_TEST_VECTORS -- } -+ .hash = __VECS(aes_vmac128_tv_template) - } - }, { - .alg = "wp256", - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = wp256_tv_template, -- .count = WP256_TEST_VECTORS -- } -+ .hash = __VECS(wp256_tv_template) - } - }, { - .alg = "wp384", - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = wp384_tv_template, -- .count = WP384_TEST_VECTORS -- } -+ .hash = __VECS(wp384_tv_template) - } - }, { - .alg = "wp512", - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = wp512_tv_template, -- .count = WP512_TEST_VECTORS -- } -+ .hash = __VECS(wp512_tv_template) - } - }, { - .alg = "xcbc(aes)", - .test = alg_test_hash, - .suite = { -- .hash = { -- .vecs = aes_xcbc128_tv_template, -- .count = XCBC_AES_TEST_VECTORS -- } -+ .hash = __VECS(aes_xcbc128_tv_template) - } - }, { - .alg = "xts(aes)", -@@ -4002,14 +3698,8 @@ static const struct alg_test_desc alg_te - .fips_allowed = 1, - .suite = { - .cipher = { -- .enc = { -- .vecs = aes_xts_enc_tv_template, -- .count = AES_XTS_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = aes_xts_dec_tv_template, -- .count = AES_XTS_DEC_TEST_VECTORS -- } -+ .enc = __VECS(aes_xts_enc_tv_template), -+ .dec = __VECS(aes_xts_dec_tv_template) - } - } - }, { -@@ -4017,14 +3707,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = camellia_xts_enc_tv_template, -- .count = CAMELLIA_XTS_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = camellia_xts_dec_tv_template, -- .count = CAMELLIA_XTS_DEC_TEST_VECTORS -- } -+ .enc = __VECS(camellia_xts_enc_tv_template), -+ .dec = __VECS(camellia_xts_dec_tv_template) - } - } - }, { -@@ -4032,14 +3716,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = cast6_xts_enc_tv_template, -- .count = CAST6_XTS_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = cast6_xts_dec_tv_template, -- .count = CAST6_XTS_DEC_TEST_VECTORS -- } -+ .enc = __VECS(cast6_xts_enc_tv_template), -+ .dec = __VECS(cast6_xts_dec_tv_template) - } - } - }, { -@@ -4047,14 +3725,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = serpent_xts_enc_tv_template, -- .count = SERPENT_XTS_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = serpent_xts_dec_tv_template, -- .count = SERPENT_XTS_DEC_TEST_VECTORS -- } -+ .enc = __VECS(serpent_xts_enc_tv_template), -+ .dec = __VECS(serpent_xts_dec_tv_template) - } - } - }, { -@@ -4062,14 +3734,8 @@ static const struct alg_test_desc alg_te - .test = alg_test_skcipher, - .suite = { - .cipher = { -- .enc = { -- .vecs = tf_xts_enc_tv_template, -- .count = TF_XTS_ENC_TEST_VECTORS -- }, -- .dec = { -- .vecs = tf_xts_dec_tv_template, -- .count = TF_XTS_DEC_TEST_VECTORS -- } -+ .enc = __VECS(tf_xts_enc_tv_template), -+ .dec = __VECS(tf_xts_dec_tv_template) - } - } - } ---- a/crypto/testmgr.h -+++ b/crypto/testmgr.h -@@ -34,9 +34,9 @@ - - struct hash_testvec { - /* only used with keyed hash algorithms */ -- char *key; -- char *plaintext; -- char *digest; -+ const char *key; -+ const char *plaintext; -+ const char *digest; - unsigned char tap[MAX_TAP]; - unsigned short psize; - unsigned char np; -@@ -63,11 +63,11 @@ struct hash_testvec { - */ - - struct cipher_testvec { -- char *key; -- char *iv; -- char *iv_out; -- char *input; -- char *result; -+ const char *key; -+ const char *iv; -+ const char *iv_out; -+ const char *input; -+ const char *result; - unsigned short tap[MAX_TAP]; - int np; - unsigned char also_non_np; -@@ -80,11 +80,11 @@ struct cipher_testvec { - }; - - struct aead_testvec { -- char *key; -- char *iv; -- char *input; -- char *assoc; -- char *result; -+ const char *key; -+ const char *iv; -+ const char *input; -+ const char *assoc; -+ const char *result; - unsigned char tap[MAX_TAP]; - unsigned char atap[MAX_TAP]; - int np; -@@ -99,10 +99,10 @@ struct aead_testvec { - }; - - struct cprng_testvec { -- char *key; -- char *dt; -- char *v; -- char *result; -+ const char *key; -+ const char *dt; -+ const char *v; -+ const char *result; - unsigned char klen; - unsigned short dtlen; - unsigned short vlen; -@@ -111,24 +111,38 @@ struct cprng_testvec { - }; - - struct drbg_testvec { -- unsigned char *entropy; -+ const unsigned char *entropy; - size_t entropylen; -- unsigned char *entpra; -- unsigned char *entprb; -+ const unsigned char *entpra; -+ const unsigned char *entprb; - size_t entprlen; -- unsigned char *addtla; -- unsigned char *addtlb; -+ const unsigned char *addtla; -+ const unsigned char *addtlb; - size_t addtllen; -- unsigned char *pers; -+ const unsigned char *pers; - size_t perslen; -- unsigned char *expected; -+ const unsigned char *expected; - size_t expectedlen; - }; - -+struct tls_testvec { -+ char *key; /* wrapped keys for encryption and authentication */ -+ char *iv; /* initialization vector */ -+ char *input; /* input data */ -+ char *assoc; /* associated data: seq num, type, version, input len */ -+ char *result; /* result data */ -+ unsigned char fail; /* the test failure is expected */ -+ unsigned char novrfy; /* dec verification failure expected */ -+ unsigned char klen; /* key length */ -+ unsigned short ilen; /* input data length */ -+ unsigned short alen; /* associated data length */ -+ unsigned short rlen; /* result length */ -+}; -+ - struct akcipher_testvec { -- unsigned char *key; -- unsigned char *m; -- unsigned char *c; -+ const unsigned char *key; -+ const unsigned char *m; -+ const unsigned char *c; - unsigned int key_len; - unsigned int m_size; - unsigned int c_size; -@@ -136,27 +150,227 @@ struct akcipher_testvec { - }; - - struct kpp_testvec { -- unsigned char *secret; -- unsigned char *b_public; -- unsigned char *expected_a_public; -- unsigned char *expected_ss; -+ const unsigned char *secret; -+ const unsigned char *b_public; -+ const unsigned char *expected_a_public; -+ const unsigned char *expected_ss; - unsigned short secret_size; - unsigned short b_public_size; - unsigned short expected_a_public_size; - unsigned short expected_ss_size; - }; - --static char zeroed_string[48]; -+static const char zeroed_string[48]; - - /* -- * RSA test vectors. Borrowed from openSSL. -+ * TLS1.0 synthetic test vectors - */ --#ifdef CONFIG_CRYPTO_FIPS --#define RSA_TEST_VECTORS 2 -+static struct tls_testvec tls_enc_tv_template[] = { -+ { -+#ifdef __LITTLE_ENDIAN -+ .key = "\x08\x00" /* rta length */ -+ "\x01\x00" /* rta type */ -+#else -+ .key = "\x00\x08" /* rta length */ -+ "\x00\x01" /* rta type */ -+#endif -+ "\x00\x00\x00\x10" /* enc key length */ -+ "authenticationkey20benckeyis16_bytes", -+ .klen = 8 + 20 + 16, -+ .iv = "iv0123456789abcd", -+ .input = "Single block msg", -+ .ilen = 16, -+ .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07" -+ "\x00\x03\x01\x00\x10", -+ .alen = 13, -+ .result = "\xd5\xac\xb\xd2\xac\xad\x3f\xb1" -+ "\x59\x79\x1e\x91\x5f\x52\x14\x9c" -+ "\xc0\x75\xd8\x4c\x97\x0f\x07\x73" -+ "\xdc\x89\x47\x49\x49\xcb\x30\x6b" -+ "\x1b\x45\x23\xa1\xd0\x51\xcf\x02" -+ "\x2e\xa8\x5d\xa0\xfe\xca\x82\x61", -+ .rlen = 16 + 20 + 12, -+ }, { -+#ifdef __LITTLE_ENDIAN -+ .key = "\x08\x00" /* rta length */ -+ "\x01\x00" /* rta type */ -+#else -+ .key = "\x00\x08" /* rta length */ -+ "\x00\x01" /* rta type */ -+#endif -+ "\x00\x00\x00\x10" /* enc key length */ -+ "authenticationkey20benckeyis16_bytes", -+ .klen = 8 + 20 + 16, -+ .iv = "iv0123456789abcd", -+ .input = "", -+ .ilen = 0, -+ .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07" -+ "\x00\x03\x01\x00\x00", -+ .alen = 13, -+ .result = "\x58\x2a\x11\xc\x86\x8e\x4b\x67" -+ "\x2d\x16\x26\x1a\xac\x4b\xe2\x1a" -+ "\xe9\x6a\xcc\x4d\x6f\x79\x8a\x45" -+ "\x1f\x4e\x27\xf2\xa7\x59\xb4\x5a", -+ .rlen = 20 + 12, -+ }, { -+#ifdef __LITTLE_ENDIAN -+ .key = "\x08\x00" /* rta length */ -+ "\x01\x00" /* rta type */ -+#else -+ .key = "\x00\x08" /* rta length */ -+ "\x00\x01" /* rta type */ -+#endif -+ "\x00\x00\x00\x10" /* enc key length */ -+ "authenticationkey20benckeyis16_bytes", -+ .klen = 8 + 20 + 16, -+ .iv = "iv0123456789abcd", -+ .input = "285 bytes plaintext285 bytes plaintext285 bytes" -+ " plaintext285 bytes plaintext285 bytes plaintext285" -+ " bytes plaintext285 bytes plaintext285 bytes" -+ " plaintext285 bytes plaintext285 bytes plaintext285" -+ " bytes plaintext285 bytes plaintext285 bytes" -+ " plaintext285 bytes plaintext285 bytes plaintext285" -+ " bytes plaintext285 bytes plaintext", -+ .ilen = 285, -+ .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07" -+ "\x00\x03\x01\x01\x1d", -+ .alen = 13, -+ .result = "\x80\x23\x82\x44\x14\x2a\x1d\x94\xc\xc2\x1d\xd" -+ "\x3a\x32\x89\x4c\x57\x30\xa8\x89\x76\x46\xcc\x90" -+ "\x1d\x88\xb8\xa6\x1a\x58\xe\x2d\xeb\x2c\xc7\x3a" -+ "\x52\x4e\xdb\xb3\x1e\x83\x11\xf5\x3c\xce\x6e\x94" -+ "\xd3\x26\x6a\x9a\xd\xbd\xc7\x98\xb9\xb3\x3a\x51" -+ "\x1e\x4\x84\x8a\x8f\x54\x9a\x51\x69\x9c\xce\x31" -+ "\x8d\x5d\x8b\xee\x5f\x70\xc\xc9\xb8\x50\x54\xf8" -+ "\xb2\x4a\x7a\xcd\xeb\x7a\x82\x81\xc6\x41\xc8\x50" -+ "\x91\x8d\xc8\xed\xcd\x40\x8f\x55\xd1\xec\xc9\xac" -+ "\x15\x18\xf9\x20\xa0\xed\x18\xa1\xe3\x56\xe3\x14" -+ "\xe5\xe8\x66\x63\x20\xed\xe4\x62\x9d\xa3\xa4\x1d" -+ "\x81\x89\x18\xf2\x36\xae\xc8\x8a\x2b\xbc\xc3\xb8" -+ "\x80\xf\x97\x21\x36\x39\x8\x84\x23\x18\x9e\x9c" -+ "\x72\x32\x75\x2d\x2e\xf9\x60\xb\xe8\xcc\xd9\x74" -+ "\x4\x1b\x8e\x99\xc1\x94\xee\xd0\xac\x4e\xfc\x7e" -+ "\xf1\x96\xb3\xe7\x14\xb8\xf2\xc\x25\x97\x82\x6b" -+ "\xbd\x0\x65\xab\x5c\xe3\x16\xfb\x68\xef\xea\x9d" -+ "\xff\x44\x1d\x2a\x44\xf5\xc8\x56\x77\xb7\xbf\x13" -+ "\xc8\x54\xdb\x92\xfe\x16\x4c\xbe\x18\xe9\xb\x8d" -+ "\xb\xd4\x43\x58\x43\xaa\xf4\x3\x80\x97\x62\xd5" -+ "\xdf\x3c\x28\xaa\xee\x48\x4b\x55\x41\x1b\x31\x2" -+ "\xbe\xa0\x1c\xbd\xb7\x22\x2a\xe5\x53\x72\x73\x20" -+ "\x44\x4f\xe6\x1\x2b\x34\x33\x11\x7d\xfb\x10\xc1" -+ "\x66\x7c\xa6\xf4\x48\x36\x5e\x2\xda\x41\x4b\x3e" -+ "\xe7\x80\x17\x17\xce\xf1\x3e\x6a\x8e\x26\xf3\xb7" -+ "\x2b\x85\xd\x31\x8d\xba\x6c\x22\xb4\x28\x55\x7e" -+ "\x2a\x9e\x26\xf1\x3d\x21\xac\x65", -+ .rlen = 285 + 20 + 15, -+ } -+}; -+ -+static struct tls_testvec tls_dec_tv_template[] = { -+ { -+#ifdef __LITTLE_ENDIAN -+ .key = "\x08\x00" /* rta length */ -+ "\x01\x00" /* rta type */ -+#else -+ .key = "\x00\x08" /* rta length */ -+ "\x00\x01" /* rta type */ -+#endif -+ "\x00\x00\x00\x10" /* enc key length */ -+ "authenticationkey20benckeyis16_bytes", -+ .klen = 8 + 20 + 16, -+ .iv = "iv0123456789abcd", -+ .input = "\xd5\xac\xb\xd2\xac\xad\x3f\xb1" -+ "\x59\x79\x1e\x91\x5f\x52\x14\x9c" -+ "\xc0\x75\xd8\x4c\x97\x0f\x07\x73" -+ "\xdc\x89\x47\x49\x49\xcb\x30\x6b" -+ "\x1b\x45\x23\xa1\xd0\x51\xcf\x02" -+ "\x2e\xa8\x5d\xa0\xfe\xca\x82\x61", -+ .ilen = 16 + 20 + 12, -+ .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07" -+ "\x00\x03\x01\x00\x30", -+ .alen = 13, -+ .result = "Single block msg", -+ .rlen = 16, -+ }, { -+#ifdef __LITTLE_ENDIAN -+ .key = "\x08\x00" /* rta length */ -+ "\x01\x00" /* rta type */ - #else --#define RSA_TEST_VECTORS 5 -+ .key = "\x00\x08" /* rta length */ -+ "\x00\x01" /* rta type */ - #endif --static struct akcipher_testvec rsa_tv_template[] = { -+ "\x00\x00\x00\x10" /* enc key length */ -+ "authenticationkey20benckeyis16_bytes", -+ .klen = 8 + 20 + 16, -+ .iv = "iv0123456789abcd", -+ .input = "\x58\x2a\x11\xc\x86\x8e\x4b\x67" -+ "\x2d\x16\x26\x1a\xac\x4b\xe2\x1a" -+ "\xe9\x6a\xcc\x4d\x6f\x79\x8a\x45" -+ "\x1f\x4e\x27\xf2\xa7\x59\xb4\x5a", -+ .ilen = 20 + 12, -+ .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07" -+ "\x00\x03\x01\x00\x20", -+ .alen = 13, -+ .result = "", -+ .rlen = 0, -+ }, { -+#ifdef __LITTLE_ENDIAN -+ .key = "\x08\x00" /* rta length */ -+ "\x01\x00" /* rta type */ -+#else -+ .key = "\x00\x08" /* rta length */ -+ "\x00\x01" /* rta type */ -+#endif -+ "\x00\x00\x00\x10" /* enc key length */ -+ "authenticationkey20benckeyis16_bytes", -+ .klen = 8 + 20 + 16, -+ .iv = "iv0123456789abcd", -+ .input = "\x80\x23\x82\x44\x14\x2a\x1d\x94\xc\xc2\x1d\xd" -+ "\x3a\x32\x89\x4c\x57\x30\xa8\x89\x76\x46\xcc\x90" -+ "\x1d\x88\xb8\xa6\x1a\x58\xe\x2d\xeb\x2c\xc7\x3a" -+ "\x52\x4e\xdb\xb3\x1e\x83\x11\xf5\x3c\xce\x6e\x94" -+ "\xd3\x26\x6a\x9a\xd\xbd\xc7\x98\xb9\xb3\x3a\x51" -+ "\x1e\x4\x84\x8a\x8f\x54\x9a\x51\x69\x9c\xce\x31" -+ "\x8d\x5d\x8b\xee\x5f\x70\xc\xc9\xb8\x50\x54\xf8" -+ "\xb2\x4a\x7a\xcd\xeb\x7a\x82\x81\xc6\x41\xc8\x50" -+ "\x91\x8d\xc8\xed\xcd\x40\x8f\x55\xd1\xec\xc9\xac" -+ "\x15\x18\xf9\x20\xa0\xed\x18\xa1\xe3\x56\xe3\x14" -+ "\xe5\xe8\x66\x63\x20\xed\xe4\x62\x9d\xa3\xa4\x1d" -+ "\x81\x89\x18\xf2\x36\xae\xc8\x8a\x2b\xbc\xc3\xb8" -+ "\x80\xf\x97\x21\x36\x39\x8\x84\x23\x18\x9e\x9c" -+ "\x72\x32\x75\x2d\x2e\xf9\x60\xb\xe8\xcc\xd9\x74" -+ "\x4\x1b\x8e\x99\xc1\x94\xee\xd0\xac\x4e\xfc\x7e" -+ "\xf1\x96\xb3\xe7\x14\xb8\xf2\xc\x25\x97\x82\x6b" -+ "\xbd\x0\x65\xab\x5c\xe3\x16\xfb\x68\xef\xea\x9d" -+ "\xff\x44\x1d\x2a\x44\xf5\xc8\x56\x77\xb7\xbf\x13" -+ "\xc8\x54\xdb\x92\xfe\x16\x4c\xbe\x18\xe9\xb\x8d" -+ "\xb\xd4\x43\x58\x43\xaa\xf4\x3\x80\x97\x62\xd5" -+ "\xdf\x3c\x28\xaa\xee\x48\x4b\x55\x41\x1b\x31\x2" -+ "\xbe\xa0\x1c\xbd\xb7\x22\x2a\xe5\x53\x72\x73\x20" -+ "\x44\x4f\xe6\x1\x2b\x34\x33\x11\x7d\xfb\x10\xc1" -+ "\x66\x7c\xa6\xf4\x48\x36\x5e\x2\xda\x41\x4b\x3e" -+ "\xe7\x80\x17\x17\xce\xf1\x3e\x6a\x8e\x26\xf3\xb7" -+ "\x2b\x85\xd\x31\x8d\xba\x6c\x22\xb4\x28\x55\x7e" -+ "\x2a\x9e\x26\xf1\x3d\x21\xac\x65", -+ -+ .ilen = 285 + 20 + 15, -+ .assoc = "\x00\x01\x02\x03\x04\x05\x06\x07" -+ "\x00\x03\x01\x01\x40", -+ .alen = 13, -+ .result = "285 bytes plaintext285 bytes plaintext285 bytes" -+ " plaintext285 bytes plaintext285 bytes plaintext285" -+ " bytes plaintext285 bytes plaintext285 bytes" -+ " plaintext285 bytes plaintext285 bytes plaintext285" -+ " bytes plaintext285 bytes plaintext285 bytes" -+ " plaintext285 bytes plaintext285 bytes plaintext", -+ .rlen = 285, -+ } -+}; -+ -+/* -+ * RSA test vectors. Borrowed from openSSL. -+ */ -+static const struct akcipher_testvec rsa_tv_template[] = { - { - #ifndef CONFIG_CRYPTO_FIPS - .key = -@@ -340,6 +554,7 @@ static struct akcipher_testvec rsa_tv_te - .m_size = 8, - .c_size = 256, - .public_key_vec = true, -+#ifndef CONFIG_CRYPTO_FIPS - }, { - .key = - "\x30\x82\x09\x29" /* sequence of 2345 bytes */ -@@ -538,12 +753,11 @@ static struct akcipher_testvec rsa_tv_te - .key_len = 2349, - .m_size = 8, - .c_size = 512, -+#endif - } - }; - --#define DH_TEST_VECTORS 2 -- --struct kpp_testvec dh_tv_template[] = { -+static const struct kpp_testvec dh_tv_template[] = { - { - .secret = - #ifdef __LITTLE_ENDIAN -@@ -760,12 +974,7 @@ struct kpp_testvec dh_tv_template[] = { - } - }; - --#ifdef CONFIG_CRYPTO_FIPS --#define ECDH_TEST_VECTORS 1 --#else --#define ECDH_TEST_VECTORS 2 --#endif --struct kpp_testvec ecdh_tv_template[] = { -+static const struct kpp_testvec ecdh_tv_template[] = { - { - #ifndef CONFIG_CRYPTO_FIPS - .secret = -@@ -856,9 +1065,7 @@ struct kpp_testvec ecdh_tv_template[] = - /* - * MD4 test vectors from RFC1320 - */ --#define MD4_TEST_VECTORS 7 -- --static struct hash_testvec md4_tv_template [] = { -+static const struct hash_testvec md4_tv_template[] = { - { - .plaintext = "", - .digest = "\x31\xd6\xcf\xe0\xd1\x6a\xe9\x31" -@@ -899,8 +1106,7 @@ static struct hash_testvec md4_tv_templa - }, - }; - --#define SHA3_224_TEST_VECTORS 3 --static struct hash_testvec sha3_224_tv_template[] = { -+static const struct hash_testvec sha3_224_tv_template[] = { - { - .plaintext = "", - .digest = "\x6b\x4e\x03\x42\x36\x67\xdb\xb7" -@@ -925,8 +1131,7 @@ static struct hash_testvec sha3_224_tv_t - }, - }; - --#define SHA3_256_TEST_VECTORS 3 --static struct hash_testvec sha3_256_tv_template[] = { -+static const struct hash_testvec sha3_256_tv_template[] = { - { - .plaintext = "", - .digest = "\xa7\xff\xc6\xf8\xbf\x1e\xd7\x66" -@@ -952,8 +1157,7 @@ static struct hash_testvec sha3_256_tv_t - }; - - --#define SHA3_384_TEST_VECTORS 3 --static struct hash_testvec sha3_384_tv_template[] = { -+static const struct hash_testvec sha3_384_tv_template[] = { - { - .plaintext = "", - .digest = "\x0c\x63\xa7\x5b\x84\x5e\x4f\x7d" -@@ -985,8 +1189,7 @@ static struct hash_testvec sha3_384_tv_t - }; - - --#define SHA3_512_TEST_VECTORS 3 --static struct hash_testvec sha3_512_tv_template[] = { -+static const struct hash_testvec sha3_512_tv_template[] = { - { - .plaintext = "", - .digest = "\xa6\x9f\x73\xcc\xa2\x3a\x9a\xc5" -@@ -1027,9 +1230,7 @@ static struct hash_testvec sha3_512_tv_t - /* - * MD5 test vectors from RFC1321 - */ --#define MD5_TEST_VECTORS 7 -- --static struct hash_testvec md5_tv_template[] = { -+static const struct hash_testvec md5_tv_template[] = { - { - .digest = "\xd4\x1d\x8c\xd9\x8f\x00\xb2\x04" - "\xe9\x80\x09\x98\xec\xf8\x42\x7e", -@@ -1073,9 +1274,7 @@ static struct hash_testvec md5_tv_templa - /* - * RIPEMD-128 test vectors from ISO/IEC 10118-3:2004(E) - */ --#define RMD128_TEST_VECTORS 10 -- --static struct hash_testvec rmd128_tv_template[] = { -+static const struct hash_testvec rmd128_tv_template[] = { - { - .digest = "\xcd\xf2\x62\x13\xa1\x50\xdc\x3e" - "\xcb\x61\x0f\x18\xf6\xb3\x8b\x46", -@@ -1137,9 +1336,7 @@ static struct hash_testvec rmd128_tv_tem - /* - * RIPEMD-160 test vectors from ISO/IEC 10118-3:2004(E) - */ --#define RMD160_TEST_VECTORS 10 -- --static struct hash_testvec rmd160_tv_template[] = { -+static const struct hash_testvec rmd160_tv_template[] = { - { - .digest = "\x9c\x11\x85\xa5\xc5\xe9\xfc\x54\x61\x28" - "\x08\x97\x7e\xe8\xf5\x48\xb2\x25\x8d\x31", -@@ -1201,9 +1398,7 @@ static struct hash_testvec rmd160_tv_tem - /* - * RIPEMD-256 test vectors - */ --#define RMD256_TEST_VECTORS 8 -- --static struct hash_testvec rmd256_tv_template[] = { -+static const struct hash_testvec rmd256_tv_template[] = { - { - .digest = "\x02\xba\x4c\x4e\x5f\x8e\xcd\x18" - "\x77\xfc\x52\xd6\x4d\x30\xe3\x7a" -@@ -1269,9 +1464,7 @@ static struct hash_testvec rmd256_tv_tem - /* - * RIPEMD-320 test vectors - */ --#define RMD320_TEST_VECTORS 8 -- --static struct hash_testvec rmd320_tv_template[] = { -+static const struct hash_testvec rmd320_tv_template[] = { - { - .digest = "\x22\xd6\x5d\x56\x61\x53\x6c\xdc\x75\xc1" - "\xfd\xf5\xc6\xde\x7b\x41\xb9\xf2\x73\x25" -@@ -1334,36 +1527,49 @@ static struct hash_testvec rmd320_tv_tem - } - }; - --#define CRCT10DIF_TEST_VECTORS 3 --static struct hash_testvec crct10dif_tv_template[] = { -+static const struct hash_testvec crct10dif_tv_template[] = { - { -- .plaintext = "abc", -- .psize = 3, --#ifdef __LITTLE_ENDIAN -- .digest = "\x3b\x44", --#else -- .digest = "\x44\x3b", --#endif -- }, { -- .plaintext = "1234567890123456789012345678901234567890" -- "123456789012345678901234567890123456789", -- .psize = 79, --#ifdef __LITTLE_ENDIAN -- .digest = "\x70\x4b", --#else -- .digest = "\x4b\x70", --#endif -- }, { -- .plaintext = -- "abcddddddddddddddddddddddddddddddddddddddddddddddddddddd", -- .psize = 56, --#ifdef __LITTLE_ENDIAN -- .digest = "\xe3\x9c", --#else -- .digest = "\x9c\xe3", --#endif -- .np = 2, -- .tap = { 28, 28 } -+ .plaintext = "abc", -+ .psize = 3, -+ .digest = (u8 *)(u16 []){ 0x443b }, -+ }, { -+ .plaintext = "1234567890123456789012345678901234567890" -+ "123456789012345678901234567890123456789", -+ .psize = 79, -+ .digest = (u8 *)(u16 []){ 0x4b70 }, -+ .np = 2, -+ .tap = { 63, 16 }, -+ }, { -+ .plaintext = "abcdddddddddddddddddddddddddddddddddddddddd" -+ "ddddddddddddd", -+ .psize = 56, -+ .digest = (u8 *)(u16 []){ 0x9ce3 }, -+ .np = 8, -+ .tap = { 1, 2, 28, 7, 6, 5, 4, 3 }, -+ }, { -+ .plaintext = "1234567890123456789012345678901234567890" -+ "1234567890123456789012345678901234567890" -+ "1234567890123456789012345678901234567890" -+ "1234567890123456789012345678901234567890" -+ "1234567890123456789012345678901234567890" -+ "1234567890123456789012345678901234567890" -+ "1234567890123456789012345678901234567890" -+ "123456789012345678901234567890123456789", -+ .psize = 319, -+ .digest = (u8 *)(u16 []){ 0x44c6 }, -+ }, { -+ .plaintext = "1234567890123456789012345678901234567890" -+ "1234567890123456789012345678901234567890" -+ "1234567890123456789012345678901234567890" -+ "1234567890123456789012345678901234567890" -+ "1234567890123456789012345678901234567890" -+ "1234567890123456789012345678901234567890" -+ "1234567890123456789012345678901234567890" -+ "123456789012345678901234567890123456789", -+ .psize = 319, -+ .digest = (u8 *)(u16 []){ 0x44c6 }, -+ .np = 4, -+ .tap = { 1, 255, 57, 6 }, - } - }; - -@@ -1371,9 +1577,7 @@ static struct hash_testvec crct10dif_tv_ - * SHA1 test vectors from from FIPS PUB 180-1 - * Long vector from CAVS 5.0 - */ --#define SHA1_TEST_VECTORS 6 -- --static struct hash_testvec sha1_tv_template[] = { -+static const struct hash_testvec sha1_tv_template[] = { - { - .plaintext = "", - .psize = 0, -@@ -1563,9 +1767,7 @@ static struct hash_testvec sha1_tv_templ - /* - * SHA224 test vectors from from FIPS PUB 180-2 - */ --#define SHA224_TEST_VECTORS 5 -- --static struct hash_testvec sha224_tv_template[] = { -+static const struct hash_testvec sha224_tv_template[] = { - { - .plaintext = "", - .psize = 0, -@@ -1737,9 +1939,7 @@ static struct hash_testvec sha224_tv_tem - /* - * SHA256 test vectors from from NIST - */ --#define SHA256_TEST_VECTORS 5 -- --static struct hash_testvec sha256_tv_template[] = { -+static const struct hash_testvec sha256_tv_template[] = { - { - .plaintext = "", - .psize = 0, -@@ -1910,9 +2110,7 @@ static struct hash_testvec sha256_tv_tem - /* - * SHA384 test vectors from from NIST and kerneli - */ --#define SHA384_TEST_VECTORS 6 -- --static struct hash_testvec sha384_tv_template[] = { -+static const struct hash_testvec sha384_tv_template[] = { - { - .plaintext = "", - .psize = 0, -@@ -2104,9 +2302,7 @@ static struct hash_testvec sha384_tv_tem - /* - * SHA512 test vectors from from NIST and kerneli - */ --#define SHA512_TEST_VECTORS 6 -- --static struct hash_testvec sha512_tv_template[] = { -+static const struct hash_testvec sha512_tv_template[] = { - { - .plaintext = "", - .psize = 0, -@@ -2313,9 +2509,7 @@ static struct hash_testvec sha512_tv_tem - * by Vincent Rijmen and Paulo S. L. M. Barreto as part of the NESSIE - * submission - */ --#define WP512_TEST_VECTORS 8 -- --static struct hash_testvec wp512_tv_template[] = { -+static const struct hash_testvec wp512_tv_template[] = { - { - .plaintext = "", - .psize = 0, -@@ -2411,9 +2605,7 @@ static struct hash_testvec wp512_tv_temp - }, - }; - --#define WP384_TEST_VECTORS 8 -- --static struct hash_testvec wp384_tv_template[] = { -+static const struct hash_testvec wp384_tv_template[] = { - { - .plaintext = "", - .psize = 0, -@@ -2493,9 +2685,7 @@ static struct hash_testvec wp384_tv_temp - }, - }; - --#define WP256_TEST_VECTORS 8 -- --static struct hash_testvec wp256_tv_template[] = { -+static const struct hash_testvec wp256_tv_template[] = { - { - .plaintext = "", - .psize = 0, -@@ -2562,9 +2752,7 @@ static struct hash_testvec wp256_tv_temp - /* - * TIGER test vectors from Tiger website - */ --#define TGR192_TEST_VECTORS 6 -- --static struct hash_testvec tgr192_tv_template[] = { -+static const struct hash_testvec tgr192_tv_template[] = { - { - .plaintext = "", - .psize = 0, -@@ -2607,9 +2795,7 @@ static struct hash_testvec tgr192_tv_tem - }, - }; - --#define TGR160_TEST_VECTORS 6 -- --static struct hash_testvec tgr160_tv_template[] = { -+static const struct hash_testvec tgr160_tv_template[] = { - { - .plaintext = "", - .psize = 0, -@@ -2652,9 +2838,7 @@ static struct hash_testvec tgr160_tv_tem - }, - }; - --#define TGR128_TEST_VECTORS 6 -- --static struct hash_testvec tgr128_tv_template[] = { -+static const struct hash_testvec tgr128_tv_template[] = { - { - .plaintext = "", - .psize = 0, -@@ -2691,9 +2875,7 @@ static struct hash_testvec tgr128_tv_tem - }, - }; - --#define GHASH_TEST_VECTORS 6 -- --static struct hash_testvec ghash_tv_template[] = -+static const struct hash_testvec ghash_tv_template[] = - { - { - .key = "\xdf\xa6\xbf\x4d\xed\x81\xdb\x03" -@@ -2808,9 +2990,7 @@ static struct hash_testvec ghash_tv_temp - * HMAC-MD5 test vectors from RFC2202 - * (These need to be fixed to not use strlen). - */ --#define HMAC_MD5_TEST_VECTORS 7 -- --static struct hash_testvec hmac_md5_tv_template[] = -+static const struct hash_testvec hmac_md5_tv_template[] = - { - { - .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", -@@ -2890,9 +3070,7 @@ static struct hash_testvec hmac_md5_tv_t - /* - * HMAC-RIPEMD128 test vectors from RFC2286 - */ --#define HMAC_RMD128_TEST_VECTORS 7 -- --static struct hash_testvec hmac_rmd128_tv_template[] = { -+static const struct hash_testvec hmac_rmd128_tv_template[] = { - { - .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", - .ksize = 16, -@@ -2971,9 +3149,7 @@ static struct hash_testvec hmac_rmd128_t - /* - * HMAC-RIPEMD160 test vectors from RFC2286 - */ --#define HMAC_RMD160_TEST_VECTORS 7 -- --static struct hash_testvec hmac_rmd160_tv_template[] = { -+static const struct hash_testvec hmac_rmd160_tv_template[] = { - { - .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", - .ksize = 20, -@@ -3052,9 +3228,7 @@ static struct hash_testvec hmac_rmd160_t - /* - * HMAC-SHA1 test vectors from RFC2202 - */ --#define HMAC_SHA1_TEST_VECTORS 7 -- --static struct hash_testvec hmac_sha1_tv_template[] = { -+static const struct hash_testvec hmac_sha1_tv_template[] = { - { - .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b", - .ksize = 20, -@@ -3135,9 +3309,7 @@ static struct hash_testvec hmac_sha1_tv_ - /* - * SHA224 HMAC test vectors from RFC4231 - */ --#define HMAC_SHA224_TEST_VECTORS 4 -- --static struct hash_testvec hmac_sha224_tv_template[] = { -+static const struct hash_testvec hmac_sha224_tv_template[] = { - { - .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" - "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" -@@ -3250,9 +3422,7 @@ static struct hash_testvec hmac_sha224_t - * HMAC-SHA256 test vectors from - * draft-ietf-ipsec-ciph-sha-256-01.txt - */ --#define HMAC_SHA256_TEST_VECTORS 10 -- --static struct hash_testvec hmac_sha256_tv_template[] = { -+static const struct hash_testvec hmac_sha256_tv_template[] = { - { - .key = "\x01\x02\x03\x04\x05\x06\x07\x08" - "\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10" -@@ -3387,9 +3557,7 @@ static struct hash_testvec hmac_sha256_t - }, - }; - --#define CMAC_AES_TEST_VECTORS 6 -- --static struct hash_testvec aes_cmac128_tv_template[] = { -+static const struct hash_testvec aes_cmac128_tv_template[] = { - { /* From NIST Special Publication 800-38B, AES-128 */ - .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" - "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", -@@ -3464,9 +3632,67 @@ static struct hash_testvec aes_cmac128_t - } - }; - --#define CMAC_DES3_EDE_TEST_VECTORS 4 -+static const struct hash_testvec aes_cbcmac_tv_template[] = { -+ { -+ .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" -+ "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", -+ .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" -+ "\xe9\x3d\x7e\x11\x73\x93\x17\x2a", -+ .digest = "\x3a\xd7\x7b\xb4\x0d\x7a\x36\x60" -+ "\xa8\x9e\xca\xf3\x24\x66\xef\x97", -+ .psize = 16, -+ .ksize = 16, -+ }, { -+ .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" -+ "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", -+ .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" -+ "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" -+ "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" -+ "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51" -+ "\x30", -+ .digest = "\x9d\x0d\xd0\x63\xfb\xcb\x24\x43" -+ "\xf8\xf2\x76\x03\xac\x39\xb0\x9d", -+ .psize = 33, -+ .ksize = 16, -+ .np = 2, -+ .tap = { 7, 26 }, -+ }, { -+ .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" -+ "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", -+ .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" -+ "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" -+ "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" -+ "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51" -+ "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11" -+ "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef" -+ "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17" -+ "\xad\x2b\x41\x7b\xe6\x6c\x37", -+ .digest = "\xc0\x71\x73\xb8\xa0\x2c\x11\x7c" -+ "\xaf\xdc\xb2\xf8\x89\x32\xa3\x3a", -+ .psize = 63, -+ .ksize = 16, -+ }, { -+ .key = "\x60\x3d\xeb\x10\x15\xca\x71\xbe" -+ "\x2b\x73\xae\xf0\x85\x7d\x77\x81" -+ "\x1f\x35\x2c\x07\x3b\x61\x08\xd7" -+ "\x2d\x98\x10\xa3\x09\x14\xdf\xf4", -+ .plaintext = "\x6b\xc1\xbe\xe2\x2e\x40\x9f\x96" -+ "\xe9\x3d\x7e\x11\x73\x93\x17\x2a" -+ "\xae\x2d\x8a\x57\x1e\x03\xac\x9c" -+ "\x9e\xb7\x6f\xac\x45\xaf\x8e\x51" -+ "\x30\xc8\x1c\x46\xa3\x5c\xe4\x11" -+ "\xe5\xfb\xc1\x19\x1a\x0a\x52\xef" -+ "\xf6\x9f\x24\x45\xdf\x4f\x9b\x17" -+ "\xad\x2b\x41\x7b\xe6\x6c\x37\x10" -+ "\x1c", -+ .digest = "\x6a\x4e\xdb\x21\x47\x51\xdf\x4f" -+ "\xa8\x4d\x4c\x10\x3b\x72\x7d\xd6", -+ .psize = 65, -+ .ksize = 32, -+ } -+}; - --static struct hash_testvec des3_ede_cmac64_tv_template[] = { -+static const struct hash_testvec des3_ede_cmac64_tv_template[] = { - /* - * From NIST Special Publication 800-38B, Three Key TDEA - * Corrected test vectors from: -@@ -3512,9 +3738,7 @@ static struct hash_testvec des3_ede_cmac - } - }; - --#define XCBC_AES_TEST_VECTORS 6 -- --static struct hash_testvec aes_xcbc128_tv_template[] = { -+static const struct hash_testvec aes_xcbc128_tv_template[] = { - { - .key = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", -@@ -3580,36 +3804,35 @@ static struct hash_testvec aes_xcbc128_t - } - }; - --#define VMAC_AES_TEST_VECTORS 11 --static char vmac_string1[128] = {'\x01', '\x01', '\x01', '\x01', -- '\x02', '\x03', '\x02', '\x02', -- '\x02', '\x04', '\x01', '\x07', -- '\x04', '\x01', '\x04', '\x03',}; --static char vmac_string2[128] = {'a', 'b', 'c',}; --static char vmac_string3[128] = {'a', 'b', 'c', 'a', 'b', 'c', -- 'a', 'b', 'c', 'a', 'b', 'c', -- 'a', 'b', 'c', 'a', 'b', 'c', -- 'a', 'b', 'c', 'a', 'b', 'c', -- 'a', 'b', 'c', 'a', 'b', 'c', -- 'a', 'b', 'c', 'a', 'b', 'c', -- 'a', 'b', 'c', 'a', 'b', 'c', -- 'a', 'b', 'c', 'a', 'b', 'c', -- }; -- --static char vmac_string4[17] = {'b', 'c', 'e', 'f', -- 'i', 'j', 'l', 'm', -- 'o', 'p', 'r', 's', -- 't', 'u', 'w', 'x', 'z'}; -- --static char vmac_string5[127] = {'r', 'm', 'b', 't', 'c', -- 'o', 'l', 'k', ']', '%', -- '9', '2', '7', '!', 'A'}; -- --static char vmac_string6[129] = {'p', 't', '*', '7', 'l', -- 'i', '!', '#', 'w', '0', -- 'z', '/', '4', 'A', 'n'}; -+static const char vmac_string1[128] = {'\x01', '\x01', '\x01', '\x01', -+ '\x02', '\x03', '\x02', '\x02', -+ '\x02', '\x04', '\x01', '\x07', -+ '\x04', '\x01', '\x04', '\x03',}; -+static const char vmac_string2[128] = {'a', 'b', 'c',}; -+static const char vmac_string3[128] = {'a', 'b', 'c', 'a', 'b', 'c', -+ 'a', 'b', 'c', 'a', 'b', 'c', -+ 'a', 'b', 'c', 'a', 'b', 'c', -+ 'a', 'b', 'c', 'a', 'b', 'c', -+ 'a', 'b', 'c', 'a', 'b', 'c', -+ 'a', 'b', 'c', 'a', 'b', 'c', -+ 'a', 'b', 'c', 'a', 'b', 'c', -+ 'a', 'b', 'c', 'a', 'b', 'c', -+ }; -+ -+static const char vmac_string4[17] = {'b', 'c', 'e', 'f', -+ 'i', 'j', 'l', 'm', -+ 'o', 'p', 'r', 's', -+ 't', 'u', 'w', 'x', 'z'}; -+ -+static const char vmac_string5[127] = {'r', 'm', 'b', 't', 'c', -+ 'o', 'l', 'k', ']', '%', -+ '9', '2', '7', '!', 'A'}; -+ -+static const char vmac_string6[129] = {'p', 't', '*', '7', 'l', -+ 'i', '!', '#', 'w', '0', -+ 'z', '/', '4', 'A', 'n'}; - --static struct hash_testvec aes_vmac128_tv_template[] = { -+static const struct hash_testvec aes_vmac128_tv_template[] = { - { - .key = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", -@@ -3687,9 +3910,7 @@ static struct hash_testvec aes_vmac128_t - * SHA384 HMAC test vectors from RFC4231 - */ - --#define HMAC_SHA384_TEST_VECTORS 4 -- --static struct hash_testvec hmac_sha384_tv_template[] = { -+static const struct hash_testvec hmac_sha384_tv_template[] = { - { - .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" - "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" -@@ -3787,9 +4008,7 @@ static struct hash_testvec hmac_sha384_t - * SHA512 HMAC test vectors from RFC4231 - */ - --#define HMAC_SHA512_TEST_VECTORS 4 -- --static struct hash_testvec hmac_sha512_tv_template[] = { -+static const struct hash_testvec hmac_sha512_tv_template[] = { - { - .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" - "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" -@@ -3894,9 +4113,7 @@ static struct hash_testvec hmac_sha512_t - }, - }; - --#define HMAC_SHA3_224_TEST_VECTORS 4 -- --static struct hash_testvec hmac_sha3_224_tv_template[] = { -+static const struct hash_testvec hmac_sha3_224_tv_template[] = { - { - .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" - "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" -@@ -3985,9 +4202,7 @@ static struct hash_testvec hmac_sha3_224 - }, - }; - --#define HMAC_SHA3_256_TEST_VECTORS 4 -- --static struct hash_testvec hmac_sha3_256_tv_template[] = { -+static const struct hash_testvec hmac_sha3_256_tv_template[] = { - { - .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" - "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" -@@ -4076,9 +4291,7 @@ static struct hash_testvec hmac_sha3_256 - }, - }; - --#define HMAC_SHA3_384_TEST_VECTORS 4 -- --static struct hash_testvec hmac_sha3_384_tv_template[] = { -+static const struct hash_testvec hmac_sha3_384_tv_template[] = { - { - .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" - "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" -@@ -4175,9 +4388,7 @@ static struct hash_testvec hmac_sha3_384 - }, - }; - --#define HMAC_SHA3_512_TEST_VECTORS 4 -- --static struct hash_testvec hmac_sha3_512_tv_template[] = { -+static const struct hash_testvec hmac_sha3_512_tv_template[] = { - { - .key = "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" - "\x0b\x0b\x0b\x0b\x0b\x0b\x0b\x0b" -@@ -4286,9 +4497,7 @@ static struct hash_testvec hmac_sha3_512 - * Poly1305 test vectors from RFC7539 A.3. - */ - --#define POLY1305_TEST_VECTORS 11 -- --static struct hash_testvec poly1305_tv_template[] = { -+static const struct hash_testvec poly1305_tv_template[] = { - { /* Test Vector #1 */ - .plaintext = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" -@@ -4533,20 +4742,7 @@ static struct hash_testvec poly1305_tv_t - /* - * DES test vectors. - */ --#define DES_ENC_TEST_VECTORS 11 --#define DES_DEC_TEST_VECTORS 5 --#define DES_CBC_ENC_TEST_VECTORS 6 --#define DES_CBC_DEC_TEST_VECTORS 5 --#define DES_CTR_ENC_TEST_VECTORS 2 --#define DES_CTR_DEC_TEST_VECTORS 2 --#define DES3_EDE_ENC_TEST_VECTORS 4 --#define DES3_EDE_DEC_TEST_VECTORS 4 --#define DES3_EDE_CBC_ENC_TEST_VECTORS 2 --#define DES3_EDE_CBC_DEC_TEST_VECTORS 2 --#define DES3_EDE_CTR_ENC_TEST_VECTORS 2 --#define DES3_EDE_CTR_DEC_TEST_VECTORS 2 -- --static struct cipher_testvec des_enc_tv_template[] = { -+static const struct cipher_testvec des_enc_tv_template[] = { - { /* From Applied Cryptography */ - .key = "\x01\x23\x45\x67\x89\xab\xcd\xef", - .klen = 8, -@@ -4720,7 +4916,7 @@ static struct cipher_testvec des_enc_tv_ - }, - }; - --static struct cipher_testvec des_dec_tv_template[] = { -+static const struct cipher_testvec des_dec_tv_template[] = { - { /* From Applied Cryptography */ - .key = "\x01\x23\x45\x67\x89\xab\xcd\xef", - .klen = 8, -@@ -4830,7 +5026,7 @@ static struct cipher_testvec des_dec_tv_ - }, - }; - --static struct cipher_testvec des_cbc_enc_tv_template[] = { -+static const struct cipher_testvec des_cbc_enc_tv_template[] = { - { /* From OpenSSL */ - .key = "\x01\x23\x45\x67\x89\xab\xcd\xef", - .klen = 8, -@@ -4956,7 +5152,7 @@ static struct cipher_testvec des_cbc_enc - }, - }; - --static struct cipher_testvec des_cbc_dec_tv_template[] = { -+static const struct cipher_testvec des_cbc_dec_tv_template[] = { - { /* FIPS Pub 81 */ - .key = "\x01\x23\x45\x67\x89\xab\xcd\xef", - .klen = 8, -@@ -5065,7 +5261,7 @@ static struct cipher_testvec des_cbc_dec - }, - }; - --static struct cipher_testvec des_ctr_enc_tv_template[] = { -+static const struct cipher_testvec des_ctr_enc_tv_template[] = { - { /* Generated with Crypto++ */ - .key = "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55", - .klen = 8, -@@ -5211,7 +5407,7 @@ static struct cipher_testvec des_ctr_enc - }, - }; - --static struct cipher_testvec des_ctr_dec_tv_template[] = { -+static const struct cipher_testvec des_ctr_dec_tv_template[] = { - { /* Generated with Crypto++ */ - .key = "\xC9\x83\xA6\xC9\xEC\x0F\x32\x55", - .klen = 8, -@@ -5357,7 +5553,7 @@ static struct cipher_testvec des_ctr_dec - }, - }; - --static struct cipher_testvec des3_ede_enc_tv_template[] = { -+static const struct cipher_testvec des3_ede_enc_tv_template[] = { - { /* These are from openssl */ - .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" - "\x55\x55\x55\x55\x55\x55\x55\x55" -@@ -5522,7 +5718,7 @@ static struct cipher_testvec des3_ede_en - }, - }; - --static struct cipher_testvec des3_ede_dec_tv_template[] = { -+static const struct cipher_testvec des3_ede_dec_tv_template[] = { - { /* These are from openssl */ - .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" - "\x55\x55\x55\x55\x55\x55\x55\x55" -@@ -5687,7 +5883,7 @@ static struct cipher_testvec des3_ede_de - }, - }; - --static struct cipher_testvec des3_ede_cbc_enc_tv_template[] = { -+static const struct cipher_testvec des3_ede_cbc_enc_tv_template[] = { - { /* Generated from openssl */ - .key = "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24" - "\x44\x4D\x99\x5A\x12\xD6\x40\xC0" -@@ -5867,7 +6063,7 @@ static struct cipher_testvec des3_ede_cb - }, - }; - --static struct cipher_testvec des3_ede_cbc_dec_tv_template[] = { -+static const struct cipher_testvec des3_ede_cbc_dec_tv_template[] = { - { /* Generated from openssl */ - .key = "\xE9\xC0\xFF\x2E\x76\x0B\x64\x24" - "\x44\x4D\x99\x5A\x12\xD6\x40\xC0" -@@ -6047,7 +6243,7 @@ static struct cipher_testvec des3_ede_cb - }, - }; - --static struct cipher_testvec des3_ede_ctr_enc_tv_template[] = { -+static const struct cipher_testvec des3_ede_ctr_enc_tv_template[] = { - { /* Generated with Crypto++ */ - .key = "\x9C\xD6\xF3\x9C\xB9\x5A\x67\x00" - "\x5A\x67\x00\x2D\xCE\xEB\x2D\xCE" -@@ -6325,7 +6521,7 @@ static struct cipher_testvec des3_ede_ct - }, - }; - --static struct cipher_testvec des3_ede_ctr_dec_tv_template[] = { -+static const struct cipher_testvec des3_ede_ctr_dec_tv_template[] = { - { /* Generated with Crypto++ */ - .key = "\x9C\xD6\xF3\x9C\xB9\x5A\x67\x00" - "\x5A\x67\x00\x2D\xCE\xEB\x2D\xCE" -@@ -6606,14 +6802,7 @@ static struct cipher_testvec des3_ede_ct - /* - * Blowfish test vectors. - */ --#define BF_ENC_TEST_VECTORS 7 --#define BF_DEC_TEST_VECTORS 7 --#define BF_CBC_ENC_TEST_VECTORS 2 --#define BF_CBC_DEC_TEST_VECTORS 2 --#define BF_CTR_ENC_TEST_VECTORS 2 --#define BF_CTR_DEC_TEST_VECTORS 2 -- --static struct cipher_testvec bf_enc_tv_template[] = { -+static const struct cipher_testvec bf_enc_tv_template[] = { - { /* DES test vectors from OpenSSL */ - .key = "\x00\x00\x00\x00\x00\x00\x00\x00", - .klen = 8, -@@ -6805,7 +6994,7 @@ static struct cipher_testvec bf_enc_tv_t - }, - }; - --static struct cipher_testvec bf_dec_tv_template[] = { -+static const struct cipher_testvec bf_dec_tv_template[] = { - { /* DES test vectors from OpenSSL */ - .key = "\x00\x00\x00\x00\x00\x00\x00\x00", - .klen = 8, -@@ -6997,7 +7186,7 @@ static struct cipher_testvec bf_dec_tv_t - }, - }; - --static struct cipher_testvec bf_cbc_enc_tv_template[] = { -+static const struct cipher_testvec bf_cbc_enc_tv_template[] = { - { /* From OpenSSL */ - .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" - "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87", -@@ -7154,7 +7343,7 @@ static struct cipher_testvec bf_cbc_enc_ - }, - }; - --static struct cipher_testvec bf_cbc_dec_tv_template[] = { -+static const struct cipher_testvec bf_cbc_dec_tv_template[] = { - { /* From OpenSSL */ - .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" - "\xf0\xe1\xd2\xc3\xb4\xa5\x96\x87", -@@ -7311,7 +7500,7 @@ static struct cipher_testvec bf_cbc_dec_ - }, - }; - --static struct cipher_testvec bf_ctr_enc_tv_template[] = { -+static const struct cipher_testvec bf_ctr_enc_tv_template[] = { - { /* Generated with Crypto++ */ - .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" - "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" -@@ -7723,7 +7912,7 @@ static struct cipher_testvec bf_ctr_enc_ - }, - }; - --static struct cipher_testvec bf_ctr_dec_tv_template[] = { -+static const struct cipher_testvec bf_ctr_dec_tv_template[] = { - { /* Generated with Crypto++ */ - .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" - "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" -@@ -8138,18 +8327,7 @@ static struct cipher_testvec bf_ctr_dec_ - /* - * Twofish test vectors. - */ --#define TF_ENC_TEST_VECTORS 4 --#define TF_DEC_TEST_VECTORS 4 --#define TF_CBC_ENC_TEST_VECTORS 5 --#define TF_CBC_DEC_TEST_VECTORS 5 --#define TF_CTR_ENC_TEST_VECTORS 2 --#define TF_CTR_DEC_TEST_VECTORS 2 --#define TF_LRW_ENC_TEST_VECTORS 8 --#define TF_LRW_DEC_TEST_VECTORS 8 --#define TF_XTS_ENC_TEST_VECTORS 5 --#define TF_XTS_DEC_TEST_VECTORS 5 -- --static struct cipher_testvec tf_enc_tv_template[] = { -+static const struct cipher_testvec tf_enc_tv_template[] = { - { - .key = zeroed_string, - .klen = 16, -@@ -8317,7 +8495,7 @@ static struct cipher_testvec tf_enc_tv_t - }, - }; - --static struct cipher_testvec tf_dec_tv_template[] = { -+static const struct cipher_testvec tf_dec_tv_template[] = { - { - .key = zeroed_string, - .klen = 16, -@@ -8485,7 +8663,7 @@ static struct cipher_testvec tf_dec_tv_t - }, - }; - --static struct cipher_testvec tf_cbc_enc_tv_template[] = { -+static const struct cipher_testvec tf_cbc_enc_tv_template[] = { - { /* Generated with Nettle */ - .key = zeroed_string, - .klen = 16, -@@ -8668,7 +8846,7 @@ static struct cipher_testvec tf_cbc_enc_ - }, - }; - --static struct cipher_testvec tf_cbc_dec_tv_template[] = { -+static const struct cipher_testvec tf_cbc_dec_tv_template[] = { - { /* Reverse of the first four above */ - .key = zeroed_string, - .klen = 16, -@@ -8851,7 +9029,7 @@ static struct cipher_testvec tf_cbc_dec_ - }, - }; - --static struct cipher_testvec tf_ctr_enc_tv_template[] = { -+static const struct cipher_testvec tf_ctr_enc_tv_template[] = { - { /* Generated with Crypto++ */ - .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" - "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" -@@ -9262,7 +9440,7 @@ static struct cipher_testvec tf_ctr_enc_ - }, - }; - --static struct cipher_testvec tf_ctr_dec_tv_template[] = { -+static const struct cipher_testvec tf_ctr_dec_tv_template[] = { - { /* Generated with Crypto++ */ - .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" - "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" -@@ -9673,7 +9851,7 @@ static struct cipher_testvec tf_ctr_dec_ - }, - }; - --static struct cipher_testvec tf_lrw_enc_tv_template[] = { -+static const struct cipher_testvec tf_lrw_enc_tv_template[] = { - /* Generated from AES-LRW test vectors */ - { - .key = "\x45\x62\xac\x25\xf8\x28\x17\x6d" -@@ -9925,7 +10103,7 @@ static struct cipher_testvec tf_lrw_enc_ - }, - }; - --static struct cipher_testvec tf_lrw_dec_tv_template[] = { -+static const struct cipher_testvec tf_lrw_dec_tv_template[] = { - /* Generated from AES-LRW test vectors */ - /* same as enc vectors with input and result reversed */ - { -@@ -10178,7 +10356,7 @@ static struct cipher_testvec tf_lrw_dec_ - }, - }; - --static struct cipher_testvec tf_xts_enc_tv_template[] = { -+static const struct cipher_testvec tf_xts_enc_tv_template[] = { - /* Generated from AES-XTS test vectors */ - { - .key = "\x00\x00\x00\x00\x00\x00\x00\x00" -@@ -10520,7 +10698,7 @@ static struct cipher_testvec tf_xts_enc_ - }, - }; - --static struct cipher_testvec tf_xts_dec_tv_template[] = { -+static const struct cipher_testvec tf_xts_dec_tv_template[] = { - /* Generated from AES-XTS test vectors */ - /* same as enc vectors with input and result reversed */ - { -@@ -10867,25 +11045,7 @@ static struct cipher_testvec tf_xts_dec_ - * Serpent test vectors. These are backwards because Serpent writes - * octet sequences in right-to-left mode. - */ --#define SERPENT_ENC_TEST_VECTORS 5 --#define SERPENT_DEC_TEST_VECTORS 5 -- --#define TNEPRES_ENC_TEST_VECTORS 4 --#define TNEPRES_DEC_TEST_VECTORS 4 -- --#define SERPENT_CBC_ENC_TEST_VECTORS 1 --#define SERPENT_CBC_DEC_TEST_VECTORS 1 -- --#define SERPENT_CTR_ENC_TEST_VECTORS 2 --#define SERPENT_CTR_DEC_TEST_VECTORS 2 -- --#define SERPENT_LRW_ENC_TEST_VECTORS 8 --#define SERPENT_LRW_DEC_TEST_VECTORS 8 -- --#define SERPENT_XTS_ENC_TEST_VECTORS 5 --#define SERPENT_XTS_DEC_TEST_VECTORS 5 -- --static struct cipher_testvec serpent_enc_tv_template[] = { -+static const struct cipher_testvec serpent_enc_tv_template[] = { - { - .input = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", -@@ -11061,7 +11221,7 @@ static struct cipher_testvec serpent_enc - }, - }; - --static struct cipher_testvec tnepres_enc_tv_template[] = { -+static const struct cipher_testvec tnepres_enc_tv_template[] = { - { /* KeySize=128, PT=0, I=1 */ - .input = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", -@@ -11111,7 +11271,7 @@ static struct cipher_testvec tnepres_enc - }; - - --static struct cipher_testvec serpent_dec_tv_template[] = { -+static const struct cipher_testvec serpent_dec_tv_template[] = { - { - .input = "\x12\x07\xfc\xce\x9b\xd0\xd6\x47" - "\x6a\xe9\x8f\xbe\xd1\x43\xa0\xe2", -@@ -11287,7 +11447,7 @@ static struct cipher_testvec serpent_dec - }, - }; - --static struct cipher_testvec tnepres_dec_tv_template[] = { -+static const struct cipher_testvec tnepres_dec_tv_template[] = { - { - .input = "\x41\xcc\x6b\x31\x59\x31\x45\x97" - "\x6d\x6f\xbb\x38\x4b\x37\x21\x28", -@@ -11328,7 +11488,7 @@ static struct cipher_testvec tnepres_dec - }, - }; - --static struct cipher_testvec serpent_cbc_enc_tv_template[] = { -+static const struct cipher_testvec serpent_cbc_enc_tv_template[] = { - { /* Generated with Crypto++ */ - .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" - "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" -@@ -11469,7 +11629,7 @@ static struct cipher_testvec serpent_cbc - }, - }; - --static struct cipher_testvec serpent_cbc_dec_tv_template[] = { -+static const struct cipher_testvec serpent_cbc_dec_tv_template[] = { - { /* Generated with Crypto++ */ - .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" - "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" -@@ -11610,7 +11770,7 @@ static struct cipher_testvec serpent_cbc - }, - }; - --static struct cipher_testvec serpent_ctr_enc_tv_template[] = { -+static const struct cipher_testvec serpent_ctr_enc_tv_template[] = { - { /* Generated with Crypto++ */ - .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" - "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" -@@ -12021,7 +12181,7 @@ static struct cipher_testvec serpent_ctr - }, - }; - --static struct cipher_testvec serpent_ctr_dec_tv_template[] = { -+static const struct cipher_testvec serpent_ctr_dec_tv_template[] = { - { /* Generated with Crypto++ */ - .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" - "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" -@@ -12432,7 +12592,7 @@ static struct cipher_testvec serpent_ctr - }, - }; - --static struct cipher_testvec serpent_lrw_enc_tv_template[] = { -+static const struct cipher_testvec serpent_lrw_enc_tv_template[] = { - /* Generated from AES-LRW test vectors */ - { - .key = "\x45\x62\xac\x25\xf8\x28\x17\x6d" -@@ -12684,7 +12844,7 @@ static struct cipher_testvec serpent_lrw - }, - }; - --static struct cipher_testvec serpent_lrw_dec_tv_template[] = { -+static const struct cipher_testvec serpent_lrw_dec_tv_template[] = { - /* Generated from AES-LRW test vectors */ - /* same as enc vectors with input and result reversed */ - { -@@ -12937,7 +13097,7 @@ static struct cipher_testvec serpent_lrw - }, - }; - --static struct cipher_testvec serpent_xts_enc_tv_template[] = { -+static const struct cipher_testvec serpent_xts_enc_tv_template[] = { - /* Generated from AES-XTS test vectors */ - { - .key = "\x00\x00\x00\x00\x00\x00\x00\x00" -@@ -13279,7 +13439,7 @@ static struct cipher_testvec serpent_xts - }, - }; - --static struct cipher_testvec serpent_xts_dec_tv_template[] = { -+static const struct cipher_testvec serpent_xts_dec_tv_template[] = { - /* Generated from AES-XTS test vectors */ - /* same as enc vectors with input and result reversed */ - { -@@ -13623,18 +13783,7 @@ static struct cipher_testvec serpent_xts - }; - - /* Cast6 test vectors from RFC 2612 */ --#define CAST6_ENC_TEST_VECTORS 4 --#define CAST6_DEC_TEST_VECTORS 4 --#define CAST6_CBC_ENC_TEST_VECTORS 1 --#define CAST6_CBC_DEC_TEST_VECTORS 1 --#define CAST6_CTR_ENC_TEST_VECTORS 2 --#define CAST6_CTR_DEC_TEST_VECTORS 2 --#define CAST6_LRW_ENC_TEST_VECTORS 1 --#define CAST6_LRW_DEC_TEST_VECTORS 1 --#define CAST6_XTS_ENC_TEST_VECTORS 1 --#define CAST6_XTS_DEC_TEST_VECTORS 1 -- --static struct cipher_testvec cast6_enc_tv_template[] = { -+static const struct cipher_testvec cast6_enc_tv_template[] = { - { - .key = "\x23\x42\xbb\x9e\xfa\x38\x54\x2c" - "\x0a\xf7\x56\x47\xf2\x9f\x61\x5d", -@@ -13805,7 +13954,7 @@ static struct cipher_testvec cast6_enc_t - }, - }; - --static struct cipher_testvec cast6_dec_tv_template[] = { -+static const struct cipher_testvec cast6_dec_tv_template[] = { - { - .key = "\x23\x42\xbb\x9e\xfa\x38\x54\x2c" - "\x0a\xf7\x56\x47\xf2\x9f\x61\x5d", -@@ -13976,7 +14125,7 @@ static struct cipher_testvec cast6_dec_t - }, - }; - --static struct cipher_testvec cast6_cbc_enc_tv_template[] = { -+static const struct cipher_testvec cast6_cbc_enc_tv_template[] = { - { /* Generated from TF test vectors */ - .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" - "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" -@@ -14117,7 +14266,7 @@ static struct cipher_testvec cast6_cbc_e - }, - }; - --static struct cipher_testvec cast6_cbc_dec_tv_template[] = { -+static const struct cipher_testvec cast6_cbc_dec_tv_template[] = { - { /* Generated from TF test vectors */ - .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" - "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" -@@ -14258,7 +14407,7 @@ static struct cipher_testvec cast6_cbc_d - }, - }; - --static struct cipher_testvec cast6_ctr_enc_tv_template[] = { -+static const struct cipher_testvec cast6_ctr_enc_tv_template[] = { - { /* Generated from TF test vectors */ - .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" - "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" -@@ -14415,7 +14564,7 @@ static struct cipher_testvec cast6_ctr_e - }, - }; - --static struct cipher_testvec cast6_ctr_dec_tv_template[] = { -+static const struct cipher_testvec cast6_ctr_dec_tv_template[] = { - { /* Generated from TF test vectors */ - .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" - "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" -@@ -14572,7 +14721,7 @@ static struct cipher_testvec cast6_ctr_d - }, - }; - --static struct cipher_testvec cast6_lrw_enc_tv_template[] = { -+static const struct cipher_testvec cast6_lrw_enc_tv_template[] = { - { /* Generated from TF test vectors */ - .key = "\xf8\xd4\x76\xff\xd6\x46\xee\x6c" - "\x23\x84\xcb\x1c\x77\xd6\x19\x5d" -@@ -14719,7 +14868,7 @@ static struct cipher_testvec cast6_lrw_e - }, - }; - --static struct cipher_testvec cast6_lrw_dec_tv_template[] = { -+static const struct cipher_testvec cast6_lrw_dec_tv_template[] = { - { /* Generated from TF test vectors */ - .key = "\xf8\xd4\x76\xff\xd6\x46\xee\x6c" - "\x23\x84\xcb\x1c\x77\xd6\x19\x5d" -@@ -14866,7 +15015,7 @@ static struct cipher_testvec cast6_lrw_d - }, - }; - --static struct cipher_testvec cast6_xts_enc_tv_template[] = { -+static const struct cipher_testvec cast6_xts_enc_tv_template[] = { - { /* Generated from TF test vectors */ - .key = "\x27\x18\x28\x18\x28\x45\x90\x45" - "\x23\x53\x60\x28\x74\x71\x35\x26" -@@ -15015,7 +15164,7 @@ static struct cipher_testvec cast6_xts_e - }, - }; - --static struct cipher_testvec cast6_xts_dec_tv_template[] = { -+static const struct cipher_testvec cast6_xts_dec_tv_template[] = { - { /* Generated from TF test vectors */ - .key = "\x27\x18\x28\x18\x28\x45\x90\x45" - "\x23\x53\x60\x28\x74\x71\x35\x26" -@@ -15168,39 +15317,7 @@ static struct cipher_testvec cast6_xts_d - /* - * AES test vectors. - */ --#define AES_ENC_TEST_VECTORS 4 --#define AES_DEC_TEST_VECTORS 4 --#define AES_CBC_ENC_TEST_VECTORS 5 --#define AES_CBC_DEC_TEST_VECTORS 5 --#define HMAC_MD5_ECB_CIPHER_NULL_ENC_TEST_VECTORS 2 --#define HMAC_MD5_ECB_CIPHER_NULL_DEC_TEST_VECTORS 2 --#define HMAC_SHA1_ECB_CIPHER_NULL_ENC_TEST_VEC 2 --#define HMAC_SHA1_ECB_CIPHER_NULL_DEC_TEST_VEC 2 --#define HMAC_SHA1_AES_CBC_ENC_TEST_VEC 7 --#define HMAC_SHA256_AES_CBC_ENC_TEST_VEC 7 --#define HMAC_SHA512_AES_CBC_ENC_TEST_VEC 7 --#define AES_LRW_ENC_TEST_VECTORS 8 --#define AES_LRW_DEC_TEST_VECTORS 8 --#define AES_XTS_ENC_TEST_VECTORS 5 --#define AES_XTS_DEC_TEST_VECTORS 5 --#define AES_CTR_ENC_TEST_VECTORS 5 --#define AES_CTR_DEC_TEST_VECTORS 5 --#define AES_OFB_ENC_TEST_VECTORS 1 --#define AES_OFB_DEC_TEST_VECTORS 1 --#define AES_CTR_3686_ENC_TEST_VECTORS 7 --#define AES_CTR_3686_DEC_TEST_VECTORS 6 --#define AES_GCM_ENC_TEST_VECTORS 9 --#define AES_GCM_DEC_TEST_VECTORS 8 --#define AES_GCM_4106_ENC_TEST_VECTORS 23 --#define AES_GCM_4106_DEC_TEST_VECTORS 23 --#define AES_GCM_4543_ENC_TEST_VECTORS 1 --#define AES_GCM_4543_DEC_TEST_VECTORS 2 --#define AES_CCM_ENC_TEST_VECTORS 8 --#define AES_CCM_DEC_TEST_VECTORS 7 --#define AES_CCM_4309_ENC_TEST_VECTORS 7 --#define AES_CCM_4309_DEC_TEST_VECTORS 10 -- --static struct cipher_testvec aes_enc_tv_template[] = { -+static const struct cipher_testvec aes_enc_tv_template[] = { - { /* From FIPS-197 */ - .key = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", -@@ -15372,7 +15489,7 @@ static struct cipher_testvec aes_enc_tv_ - }, - }; - --static struct cipher_testvec aes_dec_tv_template[] = { -+static const struct cipher_testvec aes_dec_tv_template[] = { - { /* From FIPS-197 */ - .key = "\x00\x01\x02\x03\x04\x05\x06\x07" - "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", -@@ -15544,7 +15661,7 @@ static struct cipher_testvec aes_dec_tv_ - }, - }; - --static struct cipher_testvec aes_cbc_enc_tv_template[] = { -+static const struct cipher_testvec aes_cbc_enc_tv_template[] = { - { /* From RFC 3602 */ - .key = "\x06\xa9\x21\x40\x36\xb8\xa1\x5b" - "\x51\x2e\x03\xd5\x34\x12\x00\x06", -@@ -15766,7 +15883,7 @@ static struct cipher_testvec aes_cbc_enc - }, - }; - --static struct cipher_testvec aes_cbc_dec_tv_template[] = { -+static const struct cipher_testvec aes_cbc_dec_tv_template[] = { - { /* From RFC 3602 */ - .key = "\x06\xa9\x21\x40\x36\xb8\xa1\x5b" - "\x51\x2e\x03\xd5\x34\x12\x00\x06", -@@ -15988,7 +16105,7 @@ static struct cipher_testvec aes_cbc_dec - }, - }; - --static struct aead_testvec hmac_md5_ecb_cipher_null_enc_tv_template[] = { -+static const struct aead_testvec hmac_md5_ecb_cipher_null_enc_tv_template[] = { - { /* Input data from RFC 2410 Case 1 */ - #ifdef __LITTLE_ENDIAN - .key = "\x08\x00" /* rta length */ -@@ -16030,7 +16147,7 @@ static struct aead_testvec hmac_md5_ecb_ - }, - }; - --static struct aead_testvec hmac_md5_ecb_cipher_null_dec_tv_template[] = { -+static const struct aead_testvec hmac_md5_ecb_cipher_null_dec_tv_template[] = { - { - #ifdef __LITTLE_ENDIAN - .key = "\x08\x00" /* rta length */ -@@ -16072,7 +16189,7 @@ static struct aead_testvec hmac_md5_ecb_ - }, - }; - --static struct aead_testvec hmac_sha1_aes_cbc_enc_tv_temp[] = { -+static const struct aead_testvec hmac_sha1_aes_cbc_enc_tv_temp[] = { - { /* RFC 3602 Case 1 */ - #ifdef __LITTLE_ENDIAN - .key = "\x08\x00" /* rta length */ -@@ -16341,7 +16458,7 @@ static struct aead_testvec hmac_sha1_aes - }, - }; - --static struct aead_testvec hmac_sha1_ecb_cipher_null_enc_tv_temp[] = { -+static const struct aead_testvec hmac_sha1_ecb_cipher_null_enc_tv_temp[] = { - { /* Input data from RFC 2410 Case 1 */ - #ifdef __LITTLE_ENDIAN - .key = "\x08\x00" /* rta length */ -@@ -16387,7 +16504,7 @@ static struct aead_testvec hmac_sha1_ecb - }, - }; - --static struct aead_testvec hmac_sha1_ecb_cipher_null_dec_tv_temp[] = { -+static const struct aead_testvec hmac_sha1_ecb_cipher_null_dec_tv_temp[] = { - { - #ifdef __LITTLE_ENDIAN - .key = "\x08\x00" /* rta length */ -@@ -16433,7 +16550,7 @@ static struct aead_testvec hmac_sha1_ecb - }, - }; - --static struct aead_testvec hmac_sha256_aes_cbc_enc_tv_temp[] = { -+static const struct aead_testvec hmac_sha256_aes_cbc_enc_tv_temp[] = { - { /* RFC 3602 Case 1 */ - #ifdef __LITTLE_ENDIAN - .key = "\x08\x00" /* rta length */ -@@ -16716,7 +16833,7 @@ static struct aead_testvec hmac_sha256_a - }, - }; - --static struct aead_testvec hmac_sha512_aes_cbc_enc_tv_temp[] = { -+static const struct aead_testvec hmac_sha512_aes_cbc_enc_tv_temp[] = { - { /* RFC 3602 Case 1 */ - #ifdef __LITTLE_ENDIAN - .key = "\x08\x00" /* rta length */ -@@ -17055,9 +17172,7 @@ static struct aead_testvec hmac_sha512_a - }, - }; - --#define HMAC_SHA1_DES_CBC_ENC_TEST_VEC 1 -- --static struct aead_testvec hmac_sha1_des_cbc_enc_tv_temp[] = { -+static const struct aead_testvec hmac_sha1_des_cbc_enc_tv_temp[] = { - { /*Generated with cryptopp*/ - #ifdef __LITTLE_ENDIAN - .key = "\x08\x00" /* rta length */ -@@ -17116,9 +17231,7 @@ static struct aead_testvec hmac_sha1_des - }, - }; - --#define HMAC_SHA224_DES_CBC_ENC_TEST_VEC 1 -- --static struct aead_testvec hmac_sha224_des_cbc_enc_tv_temp[] = { -+static const struct aead_testvec hmac_sha224_des_cbc_enc_tv_temp[] = { - { /*Generated with cryptopp*/ - #ifdef __LITTLE_ENDIAN - .key = "\x08\x00" /* rta length */ -@@ -17177,9 +17290,7 @@ static struct aead_testvec hmac_sha224_d - }, - }; - --#define HMAC_SHA256_DES_CBC_ENC_TEST_VEC 1 -- --static struct aead_testvec hmac_sha256_des_cbc_enc_tv_temp[] = { -+static const struct aead_testvec hmac_sha256_des_cbc_enc_tv_temp[] = { - { /*Generated with cryptopp*/ - #ifdef __LITTLE_ENDIAN - .key = "\x08\x00" /* rta length */ -@@ -17240,9 +17351,7 @@ static struct aead_testvec hmac_sha256_d - }, - }; - --#define HMAC_SHA384_DES_CBC_ENC_TEST_VEC 1 -- --static struct aead_testvec hmac_sha384_des_cbc_enc_tv_temp[] = { -+static const struct aead_testvec hmac_sha384_des_cbc_enc_tv_temp[] = { - { /*Generated with cryptopp*/ - #ifdef __LITTLE_ENDIAN - .key = "\x08\x00" /* rta length */ -@@ -17307,9 +17416,7 @@ static struct aead_testvec hmac_sha384_d - }, - }; - --#define HMAC_SHA512_DES_CBC_ENC_TEST_VEC 1 -- --static struct aead_testvec hmac_sha512_des_cbc_enc_tv_temp[] = { -+static const struct aead_testvec hmac_sha512_des_cbc_enc_tv_temp[] = { - { /*Generated with cryptopp*/ - #ifdef __LITTLE_ENDIAN - .key = "\x08\x00" /* rta length */ -@@ -17378,9 +17485,7 @@ static struct aead_testvec hmac_sha512_d - }, - }; - --#define HMAC_SHA1_DES3_EDE_CBC_ENC_TEST_VEC 1 -- --static struct aead_testvec hmac_sha1_des3_ede_cbc_enc_tv_temp[] = { -+static const struct aead_testvec hmac_sha1_des3_ede_cbc_enc_tv_temp[] = { - { /*Generated with cryptopp*/ - #ifdef __LITTLE_ENDIAN - .key = "\x08\x00" /* rta length */ -@@ -17441,9 +17546,7 @@ static struct aead_testvec hmac_sha1_des - }, - }; - --#define HMAC_SHA224_DES3_EDE_CBC_ENC_TEST_VEC 1 -- --static struct aead_testvec hmac_sha224_des3_ede_cbc_enc_tv_temp[] = { -+static const struct aead_testvec hmac_sha224_des3_ede_cbc_enc_tv_temp[] = { - { /*Generated with cryptopp*/ - #ifdef __LITTLE_ENDIAN - .key = "\x08\x00" /* rta length */ -@@ -17504,9 +17607,7 @@ static struct aead_testvec hmac_sha224_d - }, - }; - --#define HMAC_SHA256_DES3_EDE_CBC_ENC_TEST_VEC 1 -- --static struct aead_testvec hmac_sha256_des3_ede_cbc_enc_tv_temp[] = { -+static const struct aead_testvec hmac_sha256_des3_ede_cbc_enc_tv_temp[] = { - { /*Generated with cryptopp*/ - #ifdef __LITTLE_ENDIAN - .key = "\x08\x00" /* rta length */ -@@ -17569,9 +17670,7 @@ static struct aead_testvec hmac_sha256_d - }, - }; - --#define HMAC_SHA384_DES3_EDE_CBC_ENC_TEST_VEC 1 -- --static struct aead_testvec hmac_sha384_des3_ede_cbc_enc_tv_temp[] = { -+static const struct aead_testvec hmac_sha384_des3_ede_cbc_enc_tv_temp[] = { - { /*Generated with cryptopp*/ - #ifdef __LITTLE_ENDIAN - .key = "\x08\x00" /* rta length */ -@@ -17638,9 +17737,7 @@ static struct aead_testvec hmac_sha384_d - }, - }; - --#define HMAC_SHA512_DES3_EDE_CBC_ENC_TEST_VEC 1 -- --static struct aead_testvec hmac_sha512_des3_ede_cbc_enc_tv_temp[] = { -+static const struct aead_testvec hmac_sha512_des3_ede_cbc_enc_tv_temp[] = { - { /*Generated with cryptopp*/ - #ifdef __LITTLE_ENDIAN - .key = "\x08\x00" /* rta length */ -@@ -17711,7 +17808,7 @@ static struct aead_testvec hmac_sha512_d - }, - }; - --static struct cipher_testvec aes_lrw_enc_tv_template[] = { -+static const struct cipher_testvec aes_lrw_enc_tv_template[] = { - /* from http://grouper.ieee.org/groups/1619/email/pdf00017.pdf */ - { /* LRW-32-AES 1 */ - .key = "\x45\x62\xac\x25\xf8\x28\x17\x6d" -@@ -17964,7 +18061,7 @@ static struct cipher_testvec aes_lrw_enc - } - }; - --static struct cipher_testvec aes_lrw_dec_tv_template[] = { -+static const struct cipher_testvec aes_lrw_dec_tv_template[] = { - /* from http://grouper.ieee.org/groups/1619/email/pdf00017.pdf */ - /* same as enc vectors with input and result reversed */ - { /* LRW-32-AES 1 */ -@@ -18218,7 +18315,7 @@ static struct cipher_testvec aes_lrw_dec - } - }; - --static struct cipher_testvec aes_xts_enc_tv_template[] = { -+static const struct cipher_testvec aes_xts_enc_tv_template[] = { - /* http://grouper.ieee.org/groups/1619/email/pdf00086.pdf */ - { /* XTS-AES 1 */ - .key = "\x00\x00\x00\x00\x00\x00\x00\x00" -@@ -18561,7 +18658,7 @@ static struct cipher_testvec aes_xts_enc - } - }; - --static struct cipher_testvec aes_xts_dec_tv_template[] = { -+static const struct cipher_testvec aes_xts_dec_tv_template[] = { - /* http://grouper.ieee.org/groups/1619/email/pdf00086.pdf */ - { /* XTS-AES 1 */ - .key = "\x00\x00\x00\x00\x00\x00\x00\x00" -@@ -18905,7 +19002,7 @@ static struct cipher_testvec aes_xts_dec - }; - - --static struct cipher_testvec aes_ctr_enc_tv_template[] = { -+static const struct cipher_testvec aes_ctr_enc_tv_template[] = { - { /* From NIST Special Publication 800-38A, Appendix F.5 */ - .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" - "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", -@@ -19260,7 +19357,7 @@ static struct cipher_testvec aes_ctr_enc - }, - }; - --static struct cipher_testvec aes_ctr_dec_tv_template[] = { -+static const struct cipher_testvec aes_ctr_dec_tv_template[] = { - { /* From NIST Special Publication 800-38A, Appendix F.5 */ - .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" - "\xab\xf7\x15\x88\x09\xcf\x4f\x3c", -@@ -19615,7 +19712,7 @@ static struct cipher_testvec aes_ctr_dec - }, - }; - --static struct cipher_testvec aes_ctr_rfc3686_enc_tv_template[] = { -+static const struct cipher_testvec aes_ctr_rfc3686_enc_tv_template[] = { - { /* From RFC 3686 */ - .key = "\xae\x68\x52\xf8\x12\x10\x67\xcc" - "\x4b\xf7\xa5\x76\x55\x77\xf3\x9e" -@@ -20747,7 +20844,7 @@ static struct cipher_testvec aes_ctr_rfc - }, - }; - --static struct cipher_testvec aes_ctr_rfc3686_dec_tv_template[] = { -+static const struct cipher_testvec aes_ctr_rfc3686_dec_tv_template[] = { - { /* From RFC 3686 */ - .key = "\xae\x68\x52\xf8\x12\x10\x67\xcc" - "\x4b\xf7\xa5\x76\x55\x77\xf3\x9e" -@@ -20838,7 +20935,7 @@ static struct cipher_testvec aes_ctr_rfc - }, - }; - --static struct cipher_testvec aes_ofb_enc_tv_template[] = { -+static const struct cipher_testvec aes_ofb_enc_tv_template[] = { - /* From NIST Special Publication 800-38A, Appendix F.5 */ - { - .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" -@@ -20867,7 +20964,7 @@ static struct cipher_testvec aes_ofb_enc - } - }; - --static struct cipher_testvec aes_ofb_dec_tv_template[] = { -+static const struct cipher_testvec aes_ofb_dec_tv_template[] = { - /* From NIST Special Publication 800-38A, Appendix F.5 */ - { - .key = "\x2b\x7e\x15\x16\x28\xae\xd2\xa6" -@@ -20896,7 +20993,7 @@ static struct cipher_testvec aes_ofb_dec - } - }; - --static struct aead_testvec aes_gcm_enc_tv_template[] = { -+static const struct aead_testvec aes_gcm_enc_tv_template[] = { - { /* From McGrew & Viega - http://citeseer.ist.psu.edu/656989.html */ - .key = zeroed_string, - .klen = 16, -@@ -21056,7 +21153,7 @@ static struct aead_testvec aes_gcm_enc_t - } - }; - --static struct aead_testvec aes_gcm_dec_tv_template[] = { -+static const struct aead_testvec aes_gcm_dec_tv_template[] = { - { /* From McGrew & Viega - http://citeseer.ist.psu.edu/656989.html */ - .key = zeroed_string, - .klen = 32, -@@ -21258,7 +21355,7 @@ static struct aead_testvec aes_gcm_dec_t - } - }; - --static struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { -+static const struct aead_testvec aes_gcm_rfc4106_enc_tv_template[] = { - { /* Generated using Crypto++ */ - .key = zeroed_string, - .klen = 20, -@@ -21871,7 +21968,7 @@ static struct aead_testvec aes_gcm_rfc41 - } - }; - --static struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { -+static const struct aead_testvec aes_gcm_rfc4106_dec_tv_template[] = { - { /* Generated using Crypto++ */ - .key = zeroed_string, - .klen = 20, -@@ -22485,7 +22582,7 @@ static struct aead_testvec aes_gcm_rfc41 - } - }; - --static struct aead_testvec aes_gcm_rfc4543_enc_tv_template[] = { -+static const struct aead_testvec aes_gcm_rfc4543_enc_tv_template[] = { - { /* From draft-mcgrew-gcm-test-01 */ - .key = "\x4c\x80\xcd\xef\xbb\x5d\x10\xda" - "\x90\x6a\xc7\x3c\x36\x13\xa6\x34" -@@ -22516,7 +22613,7 @@ static struct aead_testvec aes_gcm_rfc45 - } - }; - --static struct aead_testvec aes_gcm_rfc4543_dec_tv_template[] = { -+static const struct aead_testvec aes_gcm_rfc4543_dec_tv_template[] = { - { /* From draft-mcgrew-gcm-test-01 */ - .key = "\x4c\x80\xcd\xef\xbb\x5d\x10\xda" - "\x90\x6a\xc7\x3c\x36\x13\xa6\x34" -@@ -22575,7 +22672,7 @@ static struct aead_testvec aes_gcm_rfc45 - }, - }; - --static struct aead_testvec aes_ccm_enc_tv_template[] = { -+static const struct aead_testvec aes_ccm_enc_tv_template[] = { - { /* From RFC 3610 */ - .key = "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" - "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf", -@@ -22859,7 +22956,7 @@ static struct aead_testvec aes_ccm_enc_t - } - }; - --static struct aead_testvec aes_ccm_dec_tv_template[] = { -+static const struct aead_testvec aes_ccm_dec_tv_template[] = { - { /* From RFC 3610 */ - .key = "\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7" - "\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf", -@@ -23191,7 +23288,7 @@ static struct aead_testvec aes_ccm_dec_t - * These vectors are copied/generated from the ones for rfc4106 with - * the key truncated by one byte.. - */ --static struct aead_testvec aes_ccm_rfc4309_enc_tv_template[] = { -+static const struct aead_testvec aes_ccm_rfc4309_enc_tv_template[] = { - { /* Generated using Crypto++ */ - .key = zeroed_string, - .klen = 19, -@@ -23804,7 +23901,7 @@ static struct aead_testvec aes_ccm_rfc43 - } - }; - --static struct aead_testvec aes_ccm_rfc4309_dec_tv_template[] = { -+static const struct aead_testvec aes_ccm_rfc4309_dec_tv_template[] = { - { /* Generated using Crypto++ */ - .key = zeroed_string, - .klen = 19, -@@ -24420,9 +24517,7 @@ static struct aead_testvec aes_ccm_rfc43 - /* - * ChaCha20-Poly1305 AEAD test vectors from RFC7539 2.8.2./A.5. - */ --#define RFC7539_ENC_TEST_VECTORS 2 --#define RFC7539_DEC_TEST_VECTORS 2 --static struct aead_testvec rfc7539_enc_tv_template[] = { -+static const struct aead_testvec rfc7539_enc_tv_template[] = { - { - .key = "\x80\x81\x82\x83\x84\x85\x86\x87" - "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f" -@@ -24554,7 +24649,7 @@ static struct aead_testvec rfc7539_enc_t - }, - }; - --static struct aead_testvec rfc7539_dec_tv_template[] = { -+static const struct aead_testvec rfc7539_dec_tv_template[] = { - { - .key = "\x80\x81\x82\x83\x84\x85\x86\x87" - "\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f" -@@ -24689,9 +24784,7 @@ static struct aead_testvec rfc7539_dec_t - /* - * draft-irtf-cfrg-chacha20-poly1305 - */ --#define RFC7539ESP_DEC_TEST_VECTORS 1 --#define RFC7539ESP_ENC_TEST_VECTORS 1 --static struct aead_testvec rfc7539esp_enc_tv_template[] = { -+static const struct aead_testvec rfc7539esp_enc_tv_template[] = { - { - .key = "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a" - "\xf3\x33\x88\x86\x04\xf6\xb5\xf0" -@@ -24779,7 +24872,7 @@ static struct aead_testvec rfc7539esp_en - }, - }; - --static struct aead_testvec rfc7539esp_dec_tv_template[] = { -+static const struct aead_testvec rfc7539esp_dec_tv_template[] = { - { - .key = "\x1c\x92\x40\xa5\xeb\x55\xd3\x8a" - "\xf3\x33\x88\x86\x04\xf6\xb5\xf0" -@@ -24875,7 +24968,7 @@ static struct aead_testvec rfc7539esp_de - * semiblock of the ciphertext from the test vector. For decryption, iv is - * the first semiblock of the ciphertext. - */ --static struct cipher_testvec aes_kw_enc_tv_template[] = { -+static const struct cipher_testvec aes_kw_enc_tv_template[] = { - { - .key = "\x75\x75\xda\x3a\x93\x60\x7c\xc2" - "\xbf\xd8\xce\xc7\xaa\xdf\xd9\xa6", -@@ -24890,7 +24983,7 @@ static struct cipher_testvec aes_kw_enc_ - }, - }; - --static struct cipher_testvec aes_kw_dec_tv_template[] = { -+static const struct cipher_testvec aes_kw_dec_tv_template[] = { - { - .key = "\x80\xaa\x99\x73\x27\xa4\x80\x6b" - "\x6a\x7a\x41\xa5\x2b\x86\xc3\x71" -@@ -24913,9 +25006,7 @@ static struct cipher_testvec aes_kw_dec_ - * http://csrc.nist.gov/groups/STM/cavp/documents/rng/RNGVS.pdf - * Only AES-128 is supported at this time. - */ --#define ANSI_CPRNG_AES_TEST_VECTORS 6 -- --static struct cprng_testvec ansi_cprng_aes_tv_template[] = { -+static const struct cprng_testvec ansi_cprng_aes_tv_template[] = { - { - .key = "\xf3\xb1\x66\x6d\x13\x60\x72\x42" - "\xed\x06\x1c\xab\xb8\xd4\x62\x02", -@@ -25011,7 +25102,7 @@ static struct cprng_testvec ansi_cprng_a - * (Hash, HMAC, CTR) are tested with all permutations of use cases (w/ and - * w/o personalization string, w/ and w/o additional input string). - */ --static struct drbg_testvec drbg_pr_sha256_tv_template[] = { -+static const struct drbg_testvec drbg_pr_sha256_tv_template[] = { - { - .entropy = (unsigned char *) - "\x72\x88\x4c\xcd\x6c\x85\x57\x70\xf7\x0b\x8b\x86" -@@ -25169,7 +25260,7 @@ static struct drbg_testvec drbg_pr_sha25 - }, - }; - --static struct drbg_testvec drbg_pr_hmac_sha256_tv_template[] = { -+static const struct drbg_testvec drbg_pr_hmac_sha256_tv_template[] = { - { - .entropy = (unsigned char *) - "\x99\x69\xe5\x4b\x47\x03\xff\x31\x78\x5b\x87\x9a" -@@ -25327,7 +25418,7 @@ static struct drbg_testvec drbg_pr_hmac_ - }, - }; - --static struct drbg_testvec drbg_pr_ctr_aes128_tv_template[] = { -+static const struct drbg_testvec drbg_pr_ctr_aes128_tv_template[] = { - { - .entropy = (unsigned char *) - "\xd1\x44\xc6\x61\x81\x6d\xca\x9d\x15\x28\x8a\x42" -@@ -25451,7 +25542,7 @@ static struct drbg_testvec drbg_pr_ctr_a - * (Hash, HMAC, CTR) are tested with all permutations of use cases (w/ and - * w/o personalization string, w/ and w/o additional input string). - */ --static struct drbg_testvec drbg_nopr_sha256_tv_template[] = { -+static const struct drbg_testvec drbg_nopr_sha256_tv_template[] = { - { - .entropy = (unsigned char *) - "\xa6\x5a\xd0\xf3\x45\xdb\x4e\x0e\xff\xe8\x75\xc3" -@@ -25573,7 +25664,7 @@ static struct drbg_testvec drbg_nopr_sha - }, - }; - --static struct drbg_testvec drbg_nopr_hmac_sha256_tv_template[] = { -+static const struct drbg_testvec drbg_nopr_hmac_sha256_tv_template[] = { - { - .entropy = (unsigned char *) - "\xca\x85\x19\x11\x34\x93\x84\xbf\xfe\x89\xde\x1c" -@@ -25695,7 +25786,7 @@ static struct drbg_testvec drbg_nopr_hma - }, - }; - --static struct drbg_testvec drbg_nopr_ctr_aes192_tv_template[] = { -+static const struct drbg_testvec drbg_nopr_ctr_aes192_tv_template[] = { - { - .entropy = (unsigned char *) - "\xc3\x5c\x2f\xa2\xa8\x9d\x52\xa1\x1f\xa3\x2a\xa9" -@@ -25719,7 +25810,7 @@ static struct drbg_testvec drbg_nopr_ctr - }, - }; - --static struct drbg_testvec drbg_nopr_ctr_aes256_tv_template[] = { -+static const struct drbg_testvec drbg_nopr_ctr_aes256_tv_template[] = { - { - .entropy = (unsigned char *) - "\x36\x40\x19\x40\xfa\x8b\x1f\xba\x91\xa1\x66\x1f" -@@ -25743,7 +25834,7 @@ static struct drbg_testvec drbg_nopr_ctr - }, - }; - --static struct drbg_testvec drbg_nopr_ctr_aes128_tv_template[] = { -+static const struct drbg_testvec drbg_nopr_ctr_aes128_tv_template[] = { - { - .entropy = (unsigned char *) - "\x87\xe1\xc5\x32\x99\x7f\x57\xa3\x5c\x28\x6d\xe8" -@@ -25832,14 +25923,7 @@ static struct drbg_testvec drbg_nopr_ctr - }; - - /* Cast5 test vectors from RFC 2144 */ --#define CAST5_ENC_TEST_VECTORS 4 --#define CAST5_DEC_TEST_VECTORS 4 --#define CAST5_CBC_ENC_TEST_VECTORS 1 --#define CAST5_CBC_DEC_TEST_VECTORS 1 --#define CAST5_CTR_ENC_TEST_VECTORS 2 --#define CAST5_CTR_DEC_TEST_VECTORS 2 -- --static struct cipher_testvec cast5_enc_tv_template[] = { -+static const struct cipher_testvec cast5_enc_tv_template[] = { - { - .key = "\x01\x23\x45\x67\x12\x34\x56\x78" - "\x23\x45\x67\x89\x34\x56\x78\x9a", -@@ -26000,7 +26084,7 @@ static struct cipher_testvec cast5_enc_t - }, - }; - --static struct cipher_testvec cast5_dec_tv_template[] = { -+static const struct cipher_testvec cast5_dec_tv_template[] = { - { - .key = "\x01\x23\x45\x67\x12\x34\x56\x78" - "\x23\x45\x67\x89\x34\x56\x78\x9a", -@@ -26161,7 +26245,7 @@ static struct cipher_testvec cast5_dec_t - }, - }; - --static struct cipher_testvec cast5_cbc_enc_tv_template[] = { -+static const struct cipher_testvec cast5_cbc_enc_tv_template[] = { - { /* Generated from TF test vectors */ - .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" - "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A", -@@ -26299,7 +26383,7 @@ static struct cipher_testvec cast5_cbc_e - }, - }; - --static struct cipher_testvec cast5_cbc_dec_tv_template[] = { -+static const struct cipher_testvec cast5_cbc_dec_tv_template[] = { - { /* Generated from TF test vectors */ - .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" - "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A", -@@ -26437,7 +26521,7 @@ static struct cipher_testvec cast5_cbc_d - }, - }; - --static struct cipher_testvec cast5_ctr_enc_tv_template[] = { -+static const struct cipher_testvec cast5_ctr_enc_tv_template[] = { - { /* Generated from TF test vectors */ - .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" - "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A", -@@ -26588,7 +26672,7 @@ static struct cipher_testvec cast5_ctr_e - }, - }; - --static struct cipher_testvec cast5_ctr_dec_tv_template[] = { -+static const struct cipher_testvec cast5_ctr_dec_tv_template[] = { - { /* Generated from TF test vectors */ - .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" - "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A", -@@ -26742,10 +26826,7 @@ static struct cipher_testvec cast5_ctr_d - /* - * ARC4 test vectors from OpenSSL - */ --#define ARC4_ENC_TEST_VECTORS 7 --#define ARC4_DEC_TEST_VECTORS 7 -- --static struct cipher_testvec arc4_enc_tv_template[] = { -+static const struct cipher_testvec arc4_enc_tv_template[] = { - { - .key = "\x01\x23\x45\x67\x89\xab\xcd\xef", - .klen = 8, -@@ -26811,7 +26892,7 @@ static struct cipher_testvec arc4_enc_tv - }, - }; - --static struct cipher_testvec arc4_dec_tv_template[] = { -+static const struct cipher_testvec arc4_dec_tv_template[] = { - { - .key = "\x01\x23\x45\x67\x89\xab\xcd\xef", - .klen = 8, -@@ -26880,10 +26961,7 @@ static struct cipher_testvec arc4_dec_tv - /* - * TEA test vectors - */ --#define TEA_ENC_TEST_VECTORS 4 --#define TEA_DEC_TEST_VECTORS 4 -- --static struct cipher_testvec tea_enc_tv_template[] = { -+static const struct cipher_testvec tea_enc_tv_template[] = { - { - .key = zeroed_string, - .klen = 16, -@@ -26926,7 +27004,7 @@ static struct cipher_testvec tea_enc_tv_ - } - }; - --static struct cipher_testvec tea_dec_tv_template[] = { -+static const struct cipher_testvec tea_dec_tv_template[] = { - { - .key = zeroed_string, - .klen = 16, -@@ -26972,10 +27050,7 @@ static struct cipher_testvec tea_dec_tv_ - /* - * XTEA test vectors - */ --#define XTEA_ENC_TEST_VECTORS 4 --#define XTEA_DEC_TEST_VECTORS 4 -- --static struct cipher_testvec xtea_enc_tv_template[] = { -+static const struct cipher_testvec xtea_enc_tv_template[] = { - { - .key = zeroed_string, - .klen = 16, -@@ -27018,7 +27093,7 @@ static struct cipher_testvec xtea_enc_tv - } - }; - --static struct cipher_testvec xtea_dec_tv_template[] = { -+static const struct cipher_testvec xtea_dec_tv_template[] = { - { - .key = zeroed_string, - .klen = 16, -@@ -27064,10 +27139,7 @@ static struct cipher_testvec xtea_dec_tv - /* - * KHAZAD test vectors. - */ --#define KHAZAD_ENC_TEST_VECTORS 5 --#define KHAZAD_DEC_TEST_VECTORS 5 -- --static struct cipher_testvec khazad_enc_tv_template[] = { -+static const struct cipher_testvec khazad_enc_tv_template[] = { - { - .key = "\x80\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", -@@ -27113,7 +27185,7 @@ static struct cipher_testvec khazad_enc_ - }, - }; - --static struct cipher_testvec khazad_dec_tv_template[] = { -+static const struct cipher_testvec khazad_dec_tv_template[] = { - { - .key = "\x80\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00", -@@ -27163,12 +27235,7 @@ static struct cipher_testvec khazad_dec_ - * Anubis test vectors. - */ - --#define ANUBIS_ENC_TEST_VECTORS 5 --#define ANUBIS_DEC_TEST_VECTORS 5 --#define ANUBIS_CBC_ENC_TEST_VECTORS 2 --#define ANUBIS_CBC_DEC_TEST_VECTORS 2 -- --static struct cipher_testvec anubis_enc_tv_template[] = { -+static const struct cipher_testvec anubis_enc_tv_template[] = { - { - .key = "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe" - "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe", -@@ -27231,7 +27298,7 @@ static struct cipher_testvec anubis_enc_ - }, - }; - --static struct cipher_testvec anubis_dec_tv_template[] = { -+static const struct cipher_testvec anubis_dec_tv_template[] = { - { - .key = "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe" - "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe", -@@ -27294,7 +27361,7 @@ static struct cipher_testvec anubis_dec_ - }, - }; - --static struct cipher_testvec anubis_cbc_enc_tv_template[] = { -+static const struct cipher_testvec anubis_cbc_enc_tv_template[] = { - { - .key = "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe" - "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe", -@@ -27329,7 +27396,7 @@ static struct cipher_testvec anubis_cbc_ - }, - }; - --static struct cipher_testvec anubis_cbc_dec_tv_template[] = { -+static const struct cipher_testvec anubis_cbc_dec_tv_template[] = { - { - .key = "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe" - "\xfe\xfe\xfe\xfe\xfe\xfe\xfe\xfe", -@@ -27367,10 +27434,7 @@ static struct cipher_testvec anubis_cbc_ - /* - * XETA test vectors - */ --#define XETA_ENC_TEST_VECTORS 4 --#define XETA_DEC_TEST_VECTORS 4 -- --static struct cipher_testvec xeta_enc_tv_template[] = { -+static const struct cipher_testvec xeta_enc_tv_template[] = { - { - .key = zeroed_string, - .klen = 16, -@@ -27413,7 +27477,7 @@ static struct cipher_testvec xeta_enc_tv - } - }; - --static struct cipher_testvec xeta_dec_tv_template[] = { -+static const struct cipher_testvec xeta_dec_tv_template[] = { - { - .key = zeroed_string, - .klen = 16, -@@ -27459,10 +27523,7 @@ static struct cipher_testvec xeta_dec_tv - /* - * FCrypt test vectors - */ --#define FCRYPT_ENC_TEST_VECTORS ARRAY_SIZE(fcrypt_pcbc_enc_tv_template) --#define FCRYPT_DEC_TEST_VECTORS ARRAY_SIZE(fcrypt_pcbc_dec_tv_template) -- --static struct cipher_testvec fcrypt_pcbc_enc_tv_template[] = { -+static const struct cipher_testvec fcrypt_pcbc_enc_tv_template[] = { - { /* http://www.openafs.org/pipermail/openafs-devel/2000-December/005320.html */ - .key = "\x00\x00\x00\x00\x00\x00\x00\x00", - .klen = 8, -@@ -27523,7 +27584,7 @@ static struct cipher_testvec fcrypt_pcbc - } - }; - --static struct cipher_testvec fcrypt_pcbc_dec_tv_template[] = { -+static const struct cipher_testvec fcrypt_pcbc_dec_tv_template[] = { - { /* http://www.openafs.org/pipermail/openafs-devel/2000-December/005320.html */ - .key = "\x00\x00\x00\x00\x00\x00\x00\x00", - .klen = 8, -@@ -27587,18 +27648,7 @@ static struct cipher_testvec fcrypt_pcbc - /* - * CAMELLIA test vectors. - */ --#define CAMELLIA_ENC_TEST_VECTORS 4 --#define CAMELLIA_DEC_TEST_VECTORS 4 --#define CAMELLIA_CBC_ENC_TEST_VECTORS 3 --#define CAMELLIA_CBC_DEC_TEST_VECTORS 3 --#define CAMELLIA_CTR_ENC_TEST_VECTORS 2 --#define CAMELLIA_CTR_DEC_TEST_VECTORS 2 --#define CAMELLIA_LRW_ENC_TEST_VECTORS 8 --#define CAMELLIA_LRW_DEC_TEST_VECTORS 8 --#define CAMELLIA_XTS_ENC_TEST_VECTORS 5 --#define CAMELLIA_XTS_DEC_TEST_VECTORS 5 -- --static struct cipher_testvec camellia_enc_tv_template[] = { -+static const struct cipher_testvec camellia_enc_tv_template[] = { - { - .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" - "\xfe\xdc\xba\x98\x76\x54\x32\x10", -@@ -27898,7 +27948,7 @@ static struct cipher_testvec camellia_en - }, - }; - --static struct cipher_testvec camellia_dec_tv_template[] = { -+static const struct cipher_testvec camellia_dec_tv_template[] = { - { - .key = "\x01\x23\x45\x67\x89\xab\xcd\xef" - "\xfe\xdc\xba\x98\x76\x54\x32\x10", -@@ -28198,7 +28248,7 @@ static struct cipher_testvec camellia_de - }, - }; - --static struct cipher_testvec camellia_cbc_enc_tv_template[] = { -+static const struct cipher_testvec camellia_cbc_enc_tv_template[] = { - { - .key = "\x06\xa9\x21\x40\x36\xb8\xa1\x5b" - "\x51\x2e\x03\xd5\x34\x12\x00\x06", -@@ -28494,7 +28544,7 @@ static struct cipher_testvec camellia_cb - }, - }; - --static struct cipher_testvec camellia_cbc_dec_tv_template[] = { -+static const struct cipher_testvec camellia_cbc_dec_tv_template[] = { - { - .key = "\x06\xa9\x21\x40\x36\xb8\xa1\x5b" - "\x51\x2e\x03\xd5\x34\x12\x00\x06", -@@ -28790,7 +28840,7 @@ static struct cipher_testvec camellia_cb - }, - }; - --static struct cipher_testvec camellia_ctr_enc_tv_template[] = { -+static const struct cipher_testvec camellia_ctr_enc_tv_template[] = { - { /* Generated with Crypto++ */ - .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" - "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" -@@ -29457,7 +29507,7 @@ static struct cipher_testvec camellia_ct - }, - }; - --static struct cipher_testvec camellia_ctr_dec_tv_template[] = { -+static const struct cipher_testvec camellia_ctr_dec_tv_template[] = { - { /* Generated with Crypto++ */ - .key = "\x85\x62\x3F\x1C\xF9\xD6\x1C\xF9" - "\xD6\xB3\x90\x6D\x4A\x90\x6D\x4A" -@@ -30124,7 +30174,7 @@ static struct cipher_testvec camellia_ct - }, - }; - --static struct cipher_testvec camellia_lrw_enc_tv_template[] = { -+static const struct cipher_testvec camellia_lrw_enc_tv_template[] = { - /* Generated from AES-LRW test vectors */ - { - .key = "\x45\x62\xac\x25\xf8\x28\x17\x6d" -@@ -30376,7 +30426,7 @@ static struct cipher_testvec camellia_lr - }, - }; - --static struct cipher_testvec camellia_lrw_dec_tv_template[] = { -+static const struct cipher_testvec camellia_lrw_dec_tv_template[] = { - /* Generated from AES-LRW test vectors */ - /* same as enc vectors with input and result reversed */ - { -@@ -30629,7 +30679,7 @@ static struct cipher_testvec camellia_lr - }, - }; - --static struct cipher_testvec camellia_xts_enc_tv_template[] = { -+static const struct cipher_testvec camellia_xts_enc_tv_template[] = { - /* Generated from AES-XTS test vectors */ - { - .key = "\x00\x00\x00\x00\x00\x00\x00\x00" -@@ -30971,7 +31021,7 @@ static struct cipher_testvec camellia_xt - }, - }; - --static struct cipher_testvec camellia_xts_dec_tv_template[] = { -+static const struct cipher_testvec camellia_xts_dec_tv_template[] = { - /* Generated from AES-XTS test vectors */ - /* same as enc vectors with input and result reversed */ - { -@@ -31317,10 +31367,7 @@ static struct cipher_testvec camellia_xt - /* - * SEED test vectors - */ --#define SEED_ENC_TEST_VECTORS 4 --#define SEED_DEC_TEST_VECTORS 4 -- --static struct cipher_testvec seed_enc_tv_template[] = { -+static const struct cipher_testvec seed_enc_tv_template[] = { - { - .key = zeroed_string, - .klen = 16, -@@ -31362,7 +31409,7 @@ static struct cipher_testvec seed_enc_tv - } - }; - --static struct cipher_testvec seed_dec_tv_template[] = { -+static const struct cipher_testvec seed_dec_tv_template[] = { - { - .key = zeroed_string, - .klen = 16, -@@ -31404,8 +31451,7 @@ static struct cipher_testvec seed_dec_tv - } - }; - --#define SALSA20_STREAM_ENC_TEST_VECTORS 5 --static struct cipher_testvec salsa20_stream_enc_tv_template[] = { -+static const struct cipher_testvec salsa20_stream_enc_tv_template[] = { - /* - * Testvectors from verified.test-vectors submitted to ECRYPT. - * They are truncated to size 39, 64, 111, 129 to test a variety -@@ -32574,8 +32620,7 @@ static struct cipher_testvec salsa20_str - }, - }; - --#define CHACHA20_ENC_TEST_VECTORS 4 --static struct cipher_testvec chacha20_enc_tv_template[] = { -+static const struct cipher_testvec chacha20_enc_tv_template[] = { - { /* RFC7539 A.2. Test Vector #1 */ - .key = "\x00\x00\x00\x00\x00\x00\x00\x00" - "\x00\x00\x00\x00\x00\x00\x00\x00" -@@ -33086,9 +33131,7 @@ static struct cipher_testvec chacha20_en - /* - * CTS (Cipher Text Stealing) mode tests - */ --#define CTS_MODE_ENC_TEST_VECTORS 6 --#define CTS_MODE_DEC_TEST_VECTORS 6 --static struct cipher_testvec cts_mode_enc_tv_template[] = { -+static const struct cipher_testvec cts_mode_enc_tv_template[] = { - { /* from rfc3962 */ - .klen = 16, - .key = "\x63\x68\x69\x63\x6b\x65\x6e\x20" -@@ -33190,7 +33233,7 @@ static struct cipher_testvec cts_mode_en - } - }; - --static struct cipher_testvec cts_mode_dec_tv_template[] = { -+static const struct cipher_testvec cts_mode_dec_tv_template[] = { - { /* from rfc3962 */ - .klen = 16, - .key = "\x63\x68\x69\x63\x6b\x65\x6e\x20" -@@ -33308,10 +33351,7 @@ struct comp_testvec { - * Params: winbits=-11, Z_DEFAULT_COMPRESSION, MAX_MEM_LEVEL. - */ - --#define DEFLATE_COMP_TEST_VECTORS 2 --#define DEFLATE_DECOMP_TEST_VECTORS 2 -- --static struct comp_testvec deflate_comp_tv_template[] = { -+static const struct comp_testvec deflate_comp_tv_template[] = { - { - .inlen = 70, - .outlen = 38, -@@ -33347,7 +33387,7 @@ static struct comp_testvec deflate_comp_ - }, - }; - --static struct comp_testvec deflate_decomp_tv_template[] = { -+static const struct comp_testvec deflate_decomp_tv_template[] = { - { - .inlen = 122, - .outlen = 191, -@@ -33386,10 +33426,7 @@ static struct comp_testvec deflate_decom - /* - * LZO test vectors (null-terminated strings). - */ --#define LZO_COMP_TEST_VECTORS 2 --#define LZO_DECOMP_TEST_VECTORS 2 -- --static struct comp_testvec lzo_comp_tv_template[] = { -+static const struct comp_testvec lzo_comp_tv_template[] = { - { - .inlen = 70, - .outlen = 57, -@@ -33429,7 +33466,7 @@ static struct comp_testvec lzo_comp_tv_t - }, - }; - --static struct comp_testvec lzo_decomp_tv_template[] = { -+static const struct comp_testvec lzo_decomp_tv_template[] = { - { - .inlen = 133, - .outlen = 159, -@@ -33472,7 +33509,7 @@ static struct comp_testvec lzo_decomp_tv - */ - #define MICHAEL_MIC_TEST_VECTORS 6 - --static struct hash_testvec michael_mic_tv_template[] = { -+static const struct hash_testvec michael_mic_tv_template[] = { - { - .key = "\x00\x00\x00\x00\x00\x00\x00\x00", - .ksize = 8, -@@ -33520,9 +33557,7 @@ static struct hash_testvec michael_mic_t - /* - * CRC32 test vectors - */ --#define CRC32_TEST_VECTORS 14 -- --static struct hash_testvec crc32_tv_template[] = { -+static const struct hash_testvec crc32_tv_template[] = { - { - .key = "\x87\xa9\xcb\xed", - .ksize = 4, -@@ -33954,9 +33989,7 @@ static struct hash_testvec crc32_tv_temp - /* - * CRC32C test vectors - */ --#define CRC32C_TEST_VECTORS 15 -- --static struct hash_testvec crc32c_tv_template[] = { -+static const struct hash_testvec crc32c_tv_template[] = { - { - .psize = 0, - .digest = "\x00\x00\x00\x00", -@@ -34392,9 +34425,7 @@ static struct hash_testvec crc32c_tv_tem - /* - * Blakcifn CRC test vectors - */ --#define BFIN_CRC_TEST_VECTORS 6 -- --static struct hash_testvec bfin_crc_tv_template[] = { -+static const struct hash_testvec bfin_crc_tv_template[] = { - { - .psize = 0, - .digest = "\x00\x00\x00\x00", -@@ -34479,9 +34510,6 @@ static struct hash_testvec bfin_crc_tv_t - - }; - --#define LZ4_COMP_TEST_VECTORS 1 --#define LZ4_DECOMP_TEST_VECTORS 1 -- - static struct comp_testvec lz4_comp_tv_template[] = { - { - .inlen = 70, -@@ -34512,9 +34540,6 @@ static struct comp_testvec lz4_decomp_tv - }, - }; - --#define LZ4HC_COMP_TEST_VECTORS 1 --#define LZ4HC_DECOMP_TEST_VECTORS 1 -- - static struct comp_testvec lz4hc_comp_tv_template[] = { - { - .inlen = 70, ---- /dev/null -+++ b/crypto/tls.c -@@ -0,0 +1,607 @@ -+/* -+ * Copyright 2013 Freescale Semiconductor, Inc. -+ * Copyright 2017 NXP Semiconductor, Inc. -+ * -+ * This program is free software; you can redistribute it and/or modify it -+ * under the terms of the GNU General Public License as published by the Free -+ * Software Foundation; either version 2 of the License, or (at your option) -+ * any later version. -+ * -+ */ -+ -+#include <crypto/internal/aead.h> -+#include <crypto/internal/hash.h> -+#include <crypto/internal/skcipher.h> -+#include <crypto/authenc.h> -+#include <crypto/null.h> -+#include <crypto/scatterwalk.h> -+#include <linux/err.h> -+#include <linux/init.h> -+#include <linux/module.h> -+#include <linux/rtnetlink.h> -+ -+struct tls_instance_ctx { -+ struct crypto_ahash_spawn auth; -+ struct crypto_skcipher_spawn enc; -+}; -+ -+struct crypto_tls_ctx { -+ unsigned int reqoff; -+ struct crypto_ahash *auth; -+ struct crypto_skcipher *enc; -+ struct crypto_skcipher *null; -+}; -+ -+struct tls_request_ctx { -+ /* -+ * cryptlen holds the payload length in the case of encryption or -+ * payload_len + icv_len + padding_len in case of decryption -+ */ -+ unsigned int cryptlen; -+ /* working space for partial results */ -+ struct scatterlist tmp[2]; -+ struct scatterlist cipher[2]; -+ struct scatterlist dst[2]; -+ char tail[]; -+}; -+ -+struct async_op { -+ struct completion completion; -+ int err; -+}; -+ -+static void tls_async_op_done(struct crypto_async_request *req, int err) -+{ -+ struct async_op *areq = req->data; -+ -+ if (err == -EINPROGRESS) -+ return; -+ -+ areq->err = err; -+ complete(&areq->completion); -+} -+ -+static int crypto_tls_setkey(struct crypto_aead *tls, const u8 *key, -+ unsigned int keylen) -+{ -+ struct crypto_tls_ctx *ctx = crypto_aead_ctx(tls); -+ struct crypto_ahash *auth = ctx->auth; -+ struct crypto_skcipher *enc = ctx->enc; -+ struct crypto_authenc_keys keys; -+ int err = -EINVAL; -+ -+ if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) -+ goto badkey; -+ -+ crypto_ahash_clear_flags(auth, CRYPTO_TFM_REQ_MASK); -+ crypto_ahash_set_flags(auth, crypto_aead_get_flags(tls) & -+ CRYPTO_TFM_REQ_MASK); -+ err = crypto_ahash_setkey(auth, keys.authkey, keys.authkeylen); -+ crypto_aead_set_flags(tls, crypto_ahash_get_flags(auth) & -+ CRYPTO_TFM_RES_MASK); -+ -+ if (err) -+ goto out; -+ -+ crypto_skcipher_clear_flags(enc, CRYPTO_TFM_REQ_MASK); -+ crypto_skcipher_set_flags(enc, crypto_aead_get_flags(tls) & -+ CRYPTO_TFM_REQ_MASK); -+ err = crypto_skcipher_setkey(enc, keys.enckey, keys.enckeylen); -+ crypto_aead_set_flags(tls, crypto_skcipher_get_flags(enc) & -+ CRYPTO_TFM_RES_MASK); -+ -+out: -+ return err; -+ -+badkey: -+ crypto_aead_set_flags(tls, CRYPTO_TFM_RES_BAD_KEY_LEN); -+ goto out; -+} -+ -+/** -+ * crypto_tls_genicv - Calculate hmac digest for a TLS record -+ * @hash: (output) buffer to save the digest into -+ * @src: (input) scatterlist with the assoc and payload data -+ * @srclen: (input) size of the source buffer (assoclen + cryptlen) -+ * @req: (input) aead request -+ **/ -+static int crypto_tls_genicv(u8 *hash, struct scatterlist *src, -+ unsigned int srclen, struct aead_request *req) -+{ -+ struct crypto_aead *tls = crypto_aead_reqtfm(req); -+ struct crypto_tls_ctx *ctx = crypto_aead_ctx(tls); -+ struct tls_request_ctx *treq_ctx = aead_request_ctx(req); -+ struct async_op ahash_op; -+ struct ahash_request *ahreq = (void *)(treq_ctx->tail + ctx->reqoff); -+ unsigned int flags = CRYPTO_TFM_REQ_MAY_SLEEP; -+ int err = -EBADMSG; -+ -+ /* Bail out if the request assoc len is 0 */ -+ if (!req->assoclen) -+ return err; -+ -+ init_completion(&ahash_op.completion); -+ -+ /* the hash transform to be executed comes from the original request */ -+ ahash_request_set_tfm(ahreq, ctx->auth); -+ /* prepare the hash request with input data and result pointer */ -+ ahash_request_set_crypt(ahreq, src, hash, srclen); -+ /* set the notifier for when the async hash function returns */ -+ ahash_request_set_callback(ahreq, aead_request_flags(req) & flags, -+ tls_async_op_done, &ahash_op); -+ -+ /* Calculate the digest on the given data. The result is put in hash */ -+ err = crypto_ahash_digest(ahreq); -+ if (err == -EINPROGRESS) { -+ err = wait_for_completion_interruptible(&ahash_op.completion); -+ if (!err) -+ err = ahash_op.err; -+ } -+ -+ return err; -+} -+ -+/** -+ * crypto_tls_gen_padicv - Calculate and pad hmac digest for a TLS record -+ * @hash: (output) buffer to save the digest and padding into -+ * @phashlen: (output) the size of digest + padding -+ * @req: (input) aead request -+ **/ -+static int crypto_tls_gen_padicv(u8 *hash, unsigned int *phashlen, -+ struct aead_request *req) -+{ -+ struct crypto_aead *tls = crypto_aead_reqtfm(req); -+ unsigned int hash_size = crypto_aead_authsize(tls); -+ unsigned int block_size = crypto_aead_blocksize(tls); -+ unsigned int srclen = req->cryptlen + hash_size; -+ unsigned int icvlen = req->cryptlen + req->assoclen; -+ unsigned int padlen; -+ int err; -+ -+ err = crypto_tls_genicv(hash, req->src, icvlen, req); -+ if (err) -+ goto out; -+ -+ /* add padding after digest */ -+ padlen = block_size - (srclen % block_size); -+ memset(hash + hash_size, padlen - 1, padlen); -+ -+ *phashlen = hash_size + padlen; -+out: -+ return err; -+} -+ -+static int crypto_tls_copy_data(struct aead_request *req, -+ struct scatterlist *src, -+ struct scatterlist *dst, -+ unsigned int len) -+{ -+ struct crypto_aead *tls = crypto_aead_reqtfm(req); -+ struct crypto_tls_ctx *ctx = crypto_aead_ctx(tls); -+ SKCIPHER_REQUEST_ON_STACK(skreq, ctx->null); -+ -+ skcipher_request_set_tfm(skreq, ctx->null); -+ skcipher_request_set_callback(skreq, aead_request_flags(req), -+ NULL, NULL); -+ skcipher_request_set_crypt(skreq, src, dst, len, NULL); -+ -+ return crypto_skcipher_encrypt(skreq); -+} -+ -+static int crypto_tls_encrypt(struct aead_request *req) -+{ -+ struct crypto_aead *tls = crypto_aead_reqtfm(req); -+ struct crypto_tls_ctx *ctx = crypto_aead_ctx(tls); -+ struct tls_request_ctx *treq_ctx = aead_request_ctx(req); -+ struct skcipher_request *skreq; -+ struct scatterlist *cipher = treq_ctx->cipher; -+ struct scatterlist *tmp = treq_ctx->tmp; -+ struct scatterlist *sg, *src, *dst; -+ unsigned int cryptlen, phashlen; -+ u8 *hash = treq_ctx->tail; -+ int err; -+ -+ /* -+ * The hash result is saved at the beginning of the tls request ctx -+ * and is aligned as required by the hash transform. Enough space was -+ * allocated in crypto_tls_init_tfm to accommodate the difference. The -+ * requests themselves start later at treq_ctx->tail + ctx->reqoff so -+ * the result is not overwritten by the second (cipher) request. -+ */ -+ hash = (u8 *)ALIGN((unsigned long)hash + -+ crypto_ahash_alignmask(ctx->auth), -+ crypto_ahash_alignmask(ctx->auth) + 1); -+ -+ /* -+ * STEP 1: create ICV together with necessary padding -+ */ -+ err = crypto_tls_gen_padicv(hash, &phashlen, req); -+ if (err) -+ return err; -+ -+ /* -+ * STEP 2: Hash and padding are combined with the payload -+ * depending on the form it arrives. Scatter tables must have at least -+ * one page of data before chaining with another table and can't have -+ * an empty data page. The following code addresses these requirements. -+ * -+ * If the payload is empty, only the hash is encrypted, otherwise the -+ * payload scatterlist is merged with the hash. A special merging case -+ * is when the payload has only one page of data. In that case the -+ * payload page is moved to another scatterlist and prepared there for -+ * encryption. -+ */ -+ if (req->cryptlen) { -+ src = scatterwalk_ffwd(tmp, req->src, req->assoclen); -+ -+ sg_init_table(cipher, 2); -+ sg_set_buf(cipher + 1, hash, phashlen); -+ -+ if (sg_is_last(src)) { -+ sg_set_page(cipher, sg_page(src), req->cryptlen, -+ src->offset); -+ src = cipher; -+ } else { -+ unsigned int rem_len = req->cryptlen; -+ -+ for (sg = src; rem_len > sg->length; sg = sg_next(sg)) -+ rem_len -= min(rem_len, sg->length); -+ -+ sg_set_page(cipher, sg_page(sg), rem_len, sg->offset); -+ sg_chain(sg, 1, cipher); -+ } -+ } else { -+ sg_init_one(cipher, hash, phashlen); -+ src = cipher; -+ } -+ -+ /** -+ * If src != dst copy the associated data from source to destination. -+ * In both cases fast-forward passed the associated data in the dest. -+ */ -+ if (req->src != req->dst) { -+ err = crypto_tls_copy_data(req, req->src, req->dst, -+ req->assoclen); -+ if (err) -+ return err; -+ } -+ dst = scatterwalk_ffwd(treq_ctx->dst, req->dst, req->assoclen); -+ -+ /* -+ * STEP 3: encrypt the frame and return the result -+ */ -+ cryptlen = req->cryptlen + phashlen; -+ -+ /* -+ * The hash and the cipher are applied at different times and their -+ * requests can use the same memory space without interference -+ */ -+ skreq = (void *)(treq_ctx->tail + ctx->reqoff); -+ skcipher_request_set_tfm(skreq, ctx->enc); -+ skcipher_request_set_crypt(skreq, src, dst, cryptlen, req->iv); -+ skcipher_request_set_callback(skreq, aead_request_flags(req), -+ req->base.complete, req->base.data); -+ /* -+ * Apply the cipher transform. The result will be in req->dst when the -+ * asynchronuous call terminates -+ */ -+ err = crypto_skcipher_encrypt(skreq); -+ -+ return err; -+} -+ -+static int crypto_tls_decrypt(struct aead_request *req) -+{ -+ struct crypto_aead *tls = crypto_aead_reqtfm(req); -+ struct crypto_tls_ctx *ctx = crypto_aead_ctx(tls); -+ struct tls_request_ctx *treq_ctx = aead_request_ctx(req); -+ unsigned int cryptlen = req->cryptlen; -+ unsigned int hash_size = crypto_aead_authsize(tls); -+ unsigned int block_size = crypto_aead_blocksize(tls); -+ struct skcipher_request *skreq = (void *)(treq_ctx->tail + ctx->reqoff); -+ struct scatterlist *tmp = treq_ctx->tmp; -+ struct scatterlist *src, *dst; -+ -+ u8 padding[255]; /* padding can be 0-255 bytes */ -+ u8 pad_size; -+ u16 *len_field; -+ u8 *ihash, *hash = treq_ctx->tail; -+ -+ int paderr = 0; -+ int err = -EINVAL; -+ int i; -+ struct async_op ciph_op; -+ -+ /* -+ * Rule out bad packets. The input packet length must be at least one -+ * byte more than the hash_size -+ */ -+ if (cryptlen <= hash_size || cryptlen % block_size) -+ goto out; -+ -+ /* -+ * Step 1 - Decrypt the source. Fast-forward past the associated data -+ * to the encrypted data. The result will be overwritten in place so -+ * that the decrypted data will be adjacent to the associated data. The -+ * last step (computing the hash) will have it's input data already -+ * prepared and ready to be accessed at req->src. -+ */ -+ src = scatterwalk_ffwd(tmp, req->src, req->assoclen); -+ dst = src; -+ -+ init_completion(&ciph_op.completion); -+ skcipher_request_set_tfm(skreq, ctx->enc); -+ skcipher_request_set_callback(skreq, aead_request_flags(req), -+ tls_async_op_done, &ciph_op); -+ skcipher_request_set_crypt(skreq, src, dst, cryptlen, req->iv); -+ err = crypto_skcipher_decrypt(skreq); -+ if (err == -EINPROGRESS) { -+ err = wait_for_completion_interruptible(&ciph_op.completion); -+ if (!err) -+ err = ciph_op.err; -+ } -+ if (err) -+ goto out; -+ -+ /* -+ * Step 2 - Verify padding -+ * Retrieve the last byte of the payload; this is the padding size. -+ */ -+ cryptlen -= 1; -+ scatterwalk_map_and_copy(&pad_size, dst, cryptlen, 1, 0); -+ -+ /* RFC recommendation for invalid padding size. */ -+ if (cryptlen < pad_size + hash_size) { -+ pad_size = 0; -+ paderr = -EBADMSG; -+ } -+ cryptlen -= pad_size; -+ scatterwalk_map_and_copy(padding, dst, cryptlen, pad_size, 0); -+ -+ /* Padding content must be equal with pad_size. We verify it all */ -+ for (i = 0; i < pad_size; i++) -+ if (padding[i] != pad_size) -+ paderr = -EBADMSG; -+ -+ /* -+ * Step 3 - Verify hash -+ * Align the digest result as required by the hash transform. Enough -+ * space was allocated in crypto_tls_init_tfm -+ */ -+ hash = (u8 *)ALIGN((unsigned long)hash + -+ crypto_ahash_alignmask(ctx->auth), -+ crypto_ahash_alignmask(ctx->auth) + 1); -+ /* -+ * Two bytes at the end of the associated data make the length field. -+ * It must be updated with the length of the cleartext message before -+ * the hash is calculated. -+ */ -+ len_field = sg_virt(req->src) + req->assoclen - 2; -+ cryptlen -= hash_size; -+ *len_field = htons(cryptlen); -+ -+ /* This is the hash from the decrypted packet. Save it for later */ -+ ihash = hash + hash_size; -+ scatterwalk_map_and_copy(ihash, dst, cryptlen, hash_size, 0); -+ -+ /* Now compute and compare our ICV with the one from the packet */ -+ err = crypto_tls_genicv(hash, req->src, cryptlen + req->assoclen, req); -+ if (!err) -+ err = memcmp(hash, ihash, hash_size) ? -EBADMSG : 0; -+ -+ if (req->src != req->dst) { -+ err = crypto_tls_copy_data(req, req->src, req->dst, cryptlen + -+ req->assoclen); -+ if (err) -+ goto out; -+ } -+ -+ /* return the first found error */ -+ if (paderr) -+ err = paderr; -+ -+out: -+ aead_request_complete(req, err); -+ return err; -+} -+ -+static int crypto_tls_init_tfm(struct crypto_aead *tfm) -+{ -+ struct aead_instance *inst = aead_alg_instance(tfm); -+ struct tls_instance_ctx *ictx = aead_instance_ctx(inst); -+ struct crypto_tls_ctx *ctx = crypto_aead_ctx(tfm); -+ struct crypto_ahash *auth; -+ struct crypto_skcipher *enc; -+ struct crypto_skcipher *null; -+ int err; -+ -+ auth = crypto_spawn_ahash(&ictx->auth); -+ if (IS_ERR(auth)) -+ return PTR_ERR(auth); -+ -+ enc = crypto_spawn_skcipher(&ictx->enc); -+ err = PTR_ERR(enc); -+ if (IS_ERR(enc)) -+ goto err_free_ahash; -+ -+ null = crypto_get_default_null_skcipher2(); -+ err = PTR_ERR(null); -+ if (IS_ERR(null)) -+ goto err_free_skcipher; -+ -+ ctx->auth = auth; -+ ctx->enc = enc; -+ ctx->null = null; -+ -+ /* -+ * Allow enough space for two digests. The two digests will be compared -+ * during the decryption phase. One will come from the decrypted packet -+ * and the other will be calculated. For encryption, one digest is -+ * padded (up to a cipher blocksize) and chained with the payload -+ */ -+ ctx->reqoff = ALIGN(crypto_ahash_digestsize(auth) + -+ crypto_ahash_alignmask(auth), -+ crypto_ahash_alignmask(auth) + 1) + -+ max(crypto_ahash_digestsize(auth), -+ crypto_skcipher_blocksize(enc)); -+ -+ crypto_aead_set_reqsize(tfm, -+ sizeof(struct tls_request_ctx) + -+ ctx->reqoff + -+ max_t(unsigned int, -+ crypto_ahash_reqsize(auth) + -+ sizeof(struct ahash_request), -+ crypto_skcipher_reqsize(enc) + -+ sizeof(struct skcipher_request))); -+ -+ return 0; -+ -+err_free_skcipher: -+ crypto_free_skcipher(enc); -+err_free_ahash: -+ crypto_free_ahash(auth); -+ return err; -+} -+ -+static void crypto_tls_exit_tfm(struct crypto_aead *tfm) -+{ -+ struct crypto_tls_ctx *ctx = crypto_aead_ctx(tfm); -+ -+ crypto_free_ahash(ctx->auth); -+ crypto_free_skcipher(ctx->enc); -+ crypto_put_default_null_skcipher2(); -+} -+ -+static void crypto_tls_free(struct aead_instance *inst) -+{ -+ struct tls_instance_ctx *ctx = aead_instance_ctx(inst); -+ -+ crypto_drop_skcipher(&ctx->enc); -+ crypto_drop_ahash(&ctx->auth); -+ kfree(inst); -+} -+ -+static int crypto_tls_create(struct crypto_template *tmpl, struct rtattr **tb) -+{ -+ struct crypto_attr_type *algt; -+ struct aead_instance *inst; -+ struct hash_alg_common *auth; -+ struct crypto_alg *auth_base; -+ struct skcipher_alg *enc; -+ struct tls_instance_ctx *ctx; -+ const char *enc_name; -+ int err; -+ -+ algt = crypto_get_attr_type(tb); -+ if (IS_ERR(algt)) -+ return PTR_ERR(algt); -+ -+ if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) -+ return -EINVAL; -+ -+ auth = ahash_attr_alg(tb[1], CRYPTO_ALG_TYPE_HASH, -+ CRYPTO_ALG_TYPE_AHASH_MASK | -+ crypto_requires_sync(algt->type, algt->mask)); -+ if (IS_ERR(auth)) -+ return PTR_ERR(auth); -+ -+ auth_base = &auth->base; -+ -+ enc_name = crypto_attr_alg_name(tb[2]); -+ err = PTR_ERR(enc_name); -+ if (IS_ERR(enc_name)) -+ goto out_put_auth; -+ -+ inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); -+ err = -ENOMEM; -+ if (!inst) -+ goto out_put_auth; -+ -+ ctx = aead_instance_ctx(inst); -+ -+ err = crypto_init_ahash_spawn(&ctx->auth, auth, -+ aead_crypto_instance(inst)); -+ if (err) -+ goto err_free_inst; -+ -+ crypto_set_skcipher_spawn(&ctx->enc, aead_crypto_instance(inst)); -+ err = crypto_grab_skcipher(&ctx->enc, enc_name, 0, -+ crypto_requires_sync(algt->type, -+ algt->mask)); -+ if (err) -+ goto err_drop_auth; -+ -+ enc = crypto_spawn_skcipher_alg(&ctx->enc); -+ -+ err = -ENAMETOOLONG; -+ if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, -+ "tls10(%s,%s)", auth_base->cra_name, -+ enc->base.cra_name) >= CRYPTO_MAX_ALG_NAME) -+ goto err_drop_enc; -+ -+ if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, -+ "tls10(%s,%s)", auth_base->cra_driver_name, -+ enc->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME) -+ goto err_drop_enc; -+ -+ inst->alg.base.cra_flags = (auth_base->cra_flags | -+ enc->base.cra_flags) & CRYPTO_ALG_ASYNC; -+ inst->alg.base.cra_priority = enc->base.cra_priority * 10 + -+ auth_base->cra_priority; -+ inst->alg.base.cra_blocksize = enc->base.cra_blocksize; -+ inst->alg.base.cra_alignmask = auth_base->cra_alignmask | -+ enc->base.cra_alignmask; -+ inst->alg.base.cra_ctxsize = sizeof(struct crypto_tls_ctx); -+ -+ inst->alg.ivsize = crypto_skcipher_alg_ivsize(enc); -+ inst->alg.chunksize = crypto_skcipher_alg_chunksize(enc); -+ inst->alg.maxauthsize = auth->digestsize; -+ -+ inst->alg.init = crypto_tls_init_tfm; -+ inst->alg.exit = crypto_tls_exit_tfm; -+ -+ inst->alg.setkey = crypto_tls_setkey; -+ inst->alg.encrypt = crypto_tls_encrypt; -+ inst->alg.decrypt = crypto_tls_decrypt; -+ -+ inst->free = crypto_tls_free; -+ -+ err = aead_register_instance(tmpl, inst); -+ if (err) -+ goto err_drop_enc; -+ -+out: -+ crypto_mod_put(auth_base); -+ return err; -+ -+err_drop_enc: -+ crypto_drop_skcipher(&ctx->enc); -+err_drop_auth: -+ crypto_drop_ahash(&ctx->auth); -+err_free_inst: -+ kfree(inst); -+out_put_auth: -+ goto out; -+} -+ -+static struct crypto_template crypto_tls_tmpl = { -+ .name = "tls10", -+ .create = crypto_tls_create, -+ .module = THIS_MODULE, -+}; -+ -+static int __init crypto_tls_module_init(void) -+{ -+ return crypto_register_template(&crypto_tls_tmpl); -+} -+ -+static void __exit crypto_tls_module_exit(void) -+{ -+ crypto_unregister_template(&crypto_tls_tmpl); -+} -+ -+module_init(crypto_tls_module_init); -+module_exit(crypto_tls_module_exit); -+ -+MODULE_LICENSE("GPL"); -+MODULE_DESCRIPTION("TLS 1.0 record encryption"); ---- a/drivers/crypto/caam/Kconfig -+++ b/drivers/crypto/caam/Kconfig -@@ -1,6 +1,11 @@ -+config CRYPTO_DEV_FSL_CAAM_COMMON -+ tristate -+ - config CRYPTO_DEV_FSL_CAAM -- tristate "Freescale CAAM-Multicore driver backend" -+ tristate "Freescale CAAM-Multicore platform driver backend" - depends on FSL_SOC || ARCH_MXC || ARCH_LAYERSCAPE -+ select CRYPTO_DEV_FSL_CAAM_COMMON -+ select SOC_BUS - help - Enables the driver module for Freescale's Cryptographic Accelerator - and Assurance Module (CAAM), also known as the SEC version 4 (SEC4). -@@ -11,9 +16,16 @@ config CRYPTO_DEV_FSL_CAAM - To compile this driver as a module, choose M here: the module - will be called caam. - -+if CRYPTO_DEV_FSL_CAAM -+ -+config CRYPTO_DEV_FSL_CAAM_DEBUG -+ bool "Enable debug output in CAAM driver" -+ help -+ Selecting this will enable printing of various debug -+ information in the CAAM driver. -+ - config CRYPTO_DEV_FSL_CAAM_JR - tristate "Freescale CAAM Job Ring driver backend" -- depends on CRYPTO_DEV_FSL_CAAM - default y - help - Enables the driver module for Job Rings which are part of -@@ -24,9 +36,10 @@ config CRYPTO_DEV_FSL_CAAM_JR - To compile this driver as a module, choose M here: the module - will be called caam_jr. - -+if CRYPTO_DEV_FSL_CAAM_JR -+ - config CRYPTO_DEV_FSL_CAAM_RINGSIZE - int "Job Ring size" -- depends on CRYPTO_DEV_FSL_CAAM_JR - range 2 9 - default "9" - help -@@ -44,7 +57,6 @@ config CRYPTO_DEV_FSL_CAAM_RINGSIZE - - config CRYPTO_DEV_FSL_CAAM_INTC - bool "Job Ring interrupt coalescing" -- depends on CRYPTO_DEV_FSL_CAAM_JR - help - Enable the Job Ring's interrupt coalescing feature. - -@@ -74,7 +86,6 @@ config CRYPTO_DEV_FSL_CAAM_INTC_TIME_THL - - config CRYPTO_DEV_FSL_CAAM_CRYPTO_API - tristate "Register algorithm implementations with the Crypto API" -- depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR - default y - select CRYPTO_AEAD - select CRYPTO_AUTHENC -@@ -87,9 +98,25 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API - To compile this as a module, choose M here: the module - will be called caamalg. - -+config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI -+ tristate "Queue Interface as Crypto API backend" -+ depends on FSL_SDK_DPA && NET -+ default y -+ select CRYPTO_AUTHENC -+ select CRYPTO_BLKCIPHER -+ help -+ Selecting this will use CAAM Queue Interface (QI) for sending -+ & receiving crypto jobs to/from CAAM. This gives better performance -+ than job ring interface when the number of cores are more than the -+ number of job rings assigned to the kernel. The number of portals -+ assigned to the kernel should also be more than the number of -+ job rings. -+ -+ To compile this as a module, choose M here: the module -+ will be called caamalg_qi. -+ - config CRYPTO_DEV_FSL_CAAM_AHASH_API - tristate "Register hash algorithm implementations with Crypto API" -- depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR - default y - select CRYPTO_HASH - help -@@ -101,7 +128,6 @@ config CRYPTO_DEV_FSL_CAAM_AHASH_API - - config CRYPTO_DEV_FSL_CAAM_PKC_API - tristate "Register public key cryptography implementations with Crypto API" -- depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR - default y - select CRYPTO_RSA - help -@@ -113,7 +139,6 @@ config CRYPTO_DEV_FSL_CAAM_PKC_API - - config CRYPTO_DEV_FSL_CAAM_RNG_API - tristate "Register caam device for hwrng API" -- depends on CRYPTO_DEV_FSL_CAAM && CRYPTO_DEV_FSL_CAAM_JR - default y - select CRYPTO_RNG - select HW_RANDOM -@@ -124,13 +149,31 @@ config CRYPTO_DEV_FSL_CAAM_RNG_API - To compile this as a module, choose M here: the module - will be called caamrng. - --config CRYPTO_DEV_FSL_CAAM_IMX -- def_bool SOC_IMX6 || SOC_IMX7D -- depends on CRYPTO_DEV_FSL_CAAM -+endif # CRYPTO_DEV_FSL_CAAM_JR - --config CRYPTO_DEV_FSL_CAAM_DEBUG -- bool "Enable debug output in CAAM driver" -- depends on CRYPTO_DEV_FSL_CAAM -- help -- Selecting this will enable printing of various debug -- information in the CAAM driver. -+endif # CRYPTO_DEV_FSL_CAAM -+ -+config CRYPTO_DEV_FSL_DPAA2_CAAM -+ tristate "QorIQ DPAA2 CAAM (DPSECI) driver" -+ depends on FSL_MC_DPIO -+ select CRYPTO_DEV_FSL_CAAM_COMMON -+ select CRYPTO_BLKCIPHER -+ select CRYPTO_AUTHENC -+ select CRYPTO_AEAD -+ select CRYPTO_HASH -+ ---help--- -+ CAAM driver for QorIQ Data Path Acceleration Architecture 2. -+ It handles DPSECI DPAA2 objects that sit on the Management Complex -+ (MC) fsl-mc bus. -+ -+ To compile this as a module, choose M here: the module -+ will be called dpaa2_caam. -+ -+config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC -+ def_tristate (CRYPTO_DEV_FSL_CAAM_CRYPTO_API || \ -+ CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI || \ -+ CRYPTO_DEV_FSL_DPAA2_CAAM) -+ -+config CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC -+ def_tristate (CRYPTO_DEV_FSL_CAAM_AHASH_API || \ -+ CRYPTO_DEV_FSL_DPAA2_CAAM) ---- a/drivers/crypto/caam/Makefile -+++ b/drivers/crypto/caam/Makefile -@@ -5,13 +5,27 @@ ifeq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_DEBUG - ccflags-y := -DDEBUG - endif - -+ccflags-y += -DVERSION=\"\" -+ -+obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_COMMON) += error.o - obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o - obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_JR) += caam_jr.o - obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o -+obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += caamalg_qi.o -+obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC) += caamalg_desc.o - obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o -+obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC) += caamhash_desc.o - obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o - obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caam_pkc.o - - caam-objs := ctrl.o --caam_jr-objs := jr.o key_gen.o error.o -+caam_jr-objs := jr.o key_gen.o - caam_pkc-y := caampkc.o pkc_desc.o -+ifneq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI),) -+ ccflags-y += -DCONFIG_CAAM_QI -+ caam-objs += qi.o -+endif -+ -+obj-$(CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM) += dpaa2_caam.o -+ -+dpaa2_caam-y := caamalg_qi2.o dpseci.o ---- a/drivers/crypto/caam/caamalg.c -+++ b/drivers/crypto/caam/caamalg.c -@@ -2,6 +2,7 @@ - * caam - Freescale FSL CAAM support for crypto API - * - * Copyright 2008-2011 Freescale Semiconductor, Inc. -+ * Copyright 2016 NXP - * - * Based on talitos crypto API driver. - * -@@ -53,6 +54,7 @@ - #include "error.h" - #include "sg_sw_sec4.h" - #include "key_gen.h" -+#include "caamalg_desc.h" - - /* - * crypto alg -@@ -62,8 +64,6 @@ - #define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ - CTR_RFC3686_NONCE_SIZE + \ - SHA512_DIGEST_SIZE * 2) --/* max IV is max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ --#define CAAM_MAX_IV_LENGTH 16 - - #define AEAD_DESC_JOB_IO_LEN (DESC_JOB_IO_LEN + CAAM_CMD_SZ * 2) - #define GCM_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ -@@ -71,37 +71,6 @@ - #define AUTHENC_DESC_JOB_IO_LEN (AEAD_DESC_JOB_IO_LEN + \ - CAAM_CMD_SZ * 5) - --/* length of descriptors text */ --#define DESC_AEAD_BASE (4 * CAAM_CMD_SZ) --#define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 11 * CAAM_CMD_SZ) --#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ) --#define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 9 * CAAM_CMD_SZ) -- --/* Note: Nonce is counted in enckeylen */ --#define DESC_AEAD_CTR_RFC3686_LEN (4 * CAAM_CMD_SZ) -- --#define DESC_AEAD_NULL_BASE (3 * CAAM_CMD_SZ) --#define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 11 * CAAM_CMD_SZ) --#define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 13 * CAAM_CMD_SZ) -- --#define DESC_GCM_BASE (3 * CAAM_CMD_SZ) --#define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 16 * CAAM_CMD_SZ) --#define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 12 * CAAM_CMD_SZ) -- --#define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ) --#define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ) --#define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ) -- --#define DESC_RFC4543_BASE (3 * CAAM_CMD_SZ) --#define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 11 * CAAM_CMD_SZ) --#define DESC_RFC4543_DEC_LEN (DESC_RFC4543_BASE + 12 * CAAM_CMD_SZ) -- --#define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ) --#define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \ -- 20 * CAAM_CMD_SZ) --#define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \ -- 15 * CAAM_CMD_SZ) -- - #define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) - #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) - -@@ -112,47 +81,11 @@ - #define debug(format, arg...) - #endif - --#ifdef DEBUG --#include <linux/highmem.h> -- --static void dbg_dump_sg(const char *level, const char *prefix_str, -- int prefix_type, int rowsize, int groupsize, -- struct scatterlist *sg, size_t tlen, bool ascii, -- bool may_sleep) --{ -- struct scatterlist *it; -- void *it_page; -- size_t len; -- void *buf; -- -- for (it = sg; it != NULL && tlen > 0 ; it = sg_next(sg)) { -- /* -- * make sure the scatterlist's page -- * has a valid virtual memory mapping -- */ -- it_page = kmap_atomic(sg_page(it)); -- if (unlikely(!it_page)) { -- printk(KERN_ERR "dbg_dump_sg: kmap failed\n"); -- return; -- } -- -- buf = it_page + it->offset; -- len = min_t(size_t, tlen, it->length); -- print_hex_dump(level, prefix_str, prefix_type, rowsize, -- groupsize, buf, len, ascii); -- tlen -= len; -- -- kunmap_atomic(it_page); -- } --} --#endif -- - static struct list_head alg_list; - - struct caam_alg_entry { - int class1_alg_type; - int class2_alg_type; -- int alg_op; - bool rfc3686; - bool geniv; - }; -@@ -163,302 +96,71 @@ struct caam_aead_alg { - bool registered; - }; - --/* Set DK bit in class 1 operation if shared */ --static inline void append_dec_op1(u32 *desc, u32 type) --{ -- u32 *jump_cmd, *uncond_jump_cmd; -- -- /* DK bit is valid only for AES */ -- if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) { -- append_operation(desc, type | OP_ALG_AS_INITFINAL | -- OP_ALG_DECRYPT); -- return; -- } -- -- jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); -- append_operation(desc, type | OP_ALG_AS_INITFINAL | -- OP_ALG_DECRYPT); -- uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL); -- set_jump_tgt_here(desc, jump_cmd); -- append_operation(desc, type | OP_ALG_AS_INITFINAL | -- OP_ALG_DECRYPT | OP_ALG_AAI_DK); -- set_jump_tgt_here(desc, uncond_jump_cmd); --} -- --/* -- * For aead functions, read payload and write payload, -- * both of which are specified in req->src and req->dst -- */ --static inline void aead_append_src_dst(u32 *desc, u32 msg_type) --{ -- append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | -- KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH); --} -- --/* -- * For ablkcipher encrypt and decrypt, read from req->src and -- * write to req->dst -- */ --static inline void ablkcipher_append_src_dst(u32 *desc) --{ -- append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -- append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | -- KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); -- append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); --} -- - /* - * per-session context - */ - struct caam_ctx { -- struct device *jrdev; - u32 sh_desc_enc[DESC_MAX_USED_LEN]; - u32 sh_desc_dec[DESC_MAX_USED_LEN]; - u32 sh_desc_givenc[DESC_MAX_USED_LEN]; -+ u8 key[CAAM_MAX_KEY_SIZE]; - dma_addr_t sh_desc_enc_dma; - dma_addr_t sh_desc_dec_dma; - dma_addr_t sh_desc_givenc_dma; -- u32 class1_alg_type; -- u32 class2_alg_type; -- u32 alg_op; -- u8 key[CAAM_MAX_KEY_SIZE]; - dma_addr_t key_dma; -- unsigned int enckeylen; -- unsigned int split_key_len; -- unsigned int split_key_pad_len; -+ enum dma_data_direction dir; -+ struct device *jrdev; -+ struct alginfo adata; -+ struct alginfo cdata; - unsigned int authsize; - }; - --static void append_key_aead(u32 *desc, struct caam_ctx *ctx, -- int keys_fit_inline, bool is_rfc3686) --{ -- u32 *nonce; -- unsigned int enckeylen = ctx->enckeylen; -- -- /* -- * RFC3686 specific: -- * | ctx->key = {AUTH_KEY, ENC_KEY, NONCE} -- * | enckeylen = encryption key size + nonce size -- */ -- if (is_rfc3686) -- enckeylen -= CTR_RFC3686_NONCE_SIZE; -- -- if (keys_fit_inline) { -- append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, -- ctx->split_key_len, CLASS_2 | -- KEY_DEST_MDHA_SPLIT | KEY_ENC); -- append_key_as_imm(desc, (void *)ctx->key + -- ctx->split_key_pad_len, enckeylen, -- enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); -- } else { -- append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | -- KEY_DEST_MDHA_SPLIT | KEY_ENC); -- append_key(desc, ctx->key_dma + ctx->split_key_pad_len, -- enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); -- } -- -- /* Load Counter into CONTEXT1 reg */ -- if (is_rfc3686) { -- nonce = (u32 *)((void *)ctx->key + ctx->split_key_pad_len + -- enckeylen); -- append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, -- LDST_CLASS_IND_CCB | -- LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); -- append_move(desc, -- MOVE_SRC_OUTFIFO | -- MOVE_DEST_CLASS1CTX | -- (16 << MOVE_OFFSET_SHIFT) | -- (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); -- } --} -- --static void init_sh_desc_key_aead(u32 *desc, struct caam_ctx *ctx, -- int keys_fit_inline, bool is_rfc3686) --{ -- u32 *key_jump_cmd; -- -- /* Note: Context registers are saved. */ -- init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); -- -- /* Skip if already shared */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -- JUMP_COND_SHRD); -- -- append_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); -- -- set_jump_tgt_here(desc, key_jump_cmd); --} -- - static int aead_null_set_sh_desc(struct crypto_aead *aead) - { - struct caam_ctx *ctx = crypto_aead_ctx(aead); - struct device *jrdev = ctx->jrdev; -- bool keys_fit_inline = false; -- u32 *key_jump_cmd, *jump_cmd, *read_move_cmd, *write_move_cmd; -+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); - u32 *desc; -+ int rem_bytes = CAAM_DESC_BYTES_MAX - AEAD_DESC_JOB_IO_LEN - -+ ctx->adata.keylen_pad; - - /* - * Job Descriptor and Shared Descriptors - * must all fit into the 64-word Descriptor h/w Buffer - */ -- if (DESC_AEAD_NULL_ENC_LEN + AEAD_DESC_JOB_IO_LEN + -- ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX) -- keys_fit_inline = true; -+ if (rem_bytes >= DESC_AEAD_NULL_ENC_LEN) { -+ ctx->adata.key_inline = true; -+ ctx->adata.key_virt = ctx->key; -+ } else { -+ ctx->adata.key_inline = false; -+ ctx->adata.key_dma = ctx->key_dma; -+ } - - /* aead_encrypt shared descriptor */ - desc = ctx->sh_desc_enc; -- -- init_sh_desc(desc, HDR_SHARE_SERIAL); -- -- /* Skip if already shared */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -- JUMP_COND_SHRD); -- if (keys_fit_inline) -- append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, -- ctx->split_key_len, CLASS_2 | -- KEY_DEST_MDHA_SPLIT | KEY_ENC); -- else -- append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | -- KEY_DEST_MDHA_SPLIT | KEY_ENC); -- set_jump_tgt_here(desc, key_jump_cmd); -- -- /* assoclen + cryptlen = seqinlen */ -- append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); -- -- /* Prepare to read and write cryptlen + assoclen bytes */ -- append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); -- append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); -- -- /* -- * MOVE_LEN opcode is not available in all SEC HW revisions, -- * thus need to do some magic, i.e. self-patch the descriptor -- * buffer. -- */ -- read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | -- MOVE_DEST_MATH3 | -- (0x6 << MOVE_LEN_SHIFT)); -- write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | -- MOVE_DEST_DESCBUF | -- MOVE_WAITCOMP | -- (0x8 << MOVE_LEN_SHIFT)); -- -- /* Class 2 operation */ -- append_operation(desc, ctx->class2_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); -- -- /* Read and write cryptlen bytes */ -- aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); -- -- set_move_tgt_here(desc, read_move_cmd); -- set_move_tgt_here(desc, write_move_cmd); -- append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); -- append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | -- MOVE_AUX_LS); -- -- /* Write ICV */ -- append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | -- LDST_SRCDST_BYTE_CONTEXT); -- -- ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, -- "aead null enc shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -+ cnstr_shdsc_aead_null_encap(desc, &ctx->adata, ctx->authsize, -+ ctrlpriv->era); -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, -+ desc_bytes(desc), ctx->dir); - - /* - * Job Descriptor and Shared Descriptors - * must all fit into the 64-word Descriptor h/w Buffer - */ -- keys_fit_inline = false; -- if (DESC_AEAD_NULL_DEC_LEN + DESC_JOB_IO_LEN + -- ctx->split_key_pad_len <= CAAM_DESC_BYTES_MAX) -- keys_fit_inline = true; -- -- desc = ctx->sh_desc_dec; -+ if (rem_bytes >= DESC_AEAD_NULL_DEC_LEN) { -+ ctx->adata.key_inline = true; -+ ctx->adata.key_virt = ctx->key; -+ } else { -+ ctx->adata.key_inline = false; -+ ctx->adata.key_dma = ctx->key_dma; -+ } - - /* aead_decrypt shared descriptor */ -- init_sh_desc(desc, HDR_SHARE_SERIAL); -- -- /* Skip if already shared */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -- JUMP_COND_SHRD); -- if (keys_fit_inline) -- append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, -- ctx->split_key_len, CLASS_2 | -- KEY_DEST_MDHA_SPLIT | KEY_ENC); -- else -- append_key(desc, ctx->key_dma, ctx->split_key_len, CLASS_2 | -- KEY_DEST_MDHA_SPLIT | KEY_ENC); -- set_jump_tgt_here(desc, key_jump_cmd); -- -- /* Class 2 operation */ -- append_operation(desc, ctx->class2_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); -- -- /* assoclen + cryptlen = seqoutlen */ -- append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); -- -- /* Prepare to read and write cryptlen + assoclen bytes */ -- append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); -- append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); -- -- /* -- * MOVE_LEN opcode is not available in all SEC HW revisions, -- * thus need to do some magic, i.e. self-patch the descriptor -- * buffer. -- */ -- read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | -- MOVE_DEST_MATH2 | -- (0x6 << MOVE_LEN_SHIFT)); -- write_move_cmd = append_move(desc, MOVE_SRC_MATH2 | -- MOVE_DEST_DESCBUF | -- MOVE_WAITCOMP | -- (0x8 << MOVE_LEN_SHIFT)); -- -- /* Read and write cryptlen bytes */ -- aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); -- -- /* -- * Insert a NOP here, since we need at least 4 instructions between -- * code patching the descriptor buffer and the location being patched. -- */ -- jump_cmd = append_jump(desc, JUMP_TEST_ALL); -- set_jump_tgt_here(desc, jump_cmd); -- -- set_move_tgt_here(desc, read_move_cmd); -- set_move_tgt_here(desc, write_move_cmd); -- append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); -- append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | -- MOVE_AUX_LS); -- append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); -- -- /* Load ICV */ -- append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 | -- FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); -- -- ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, -- "aead null dec shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -+ desc = ctx->sh_desc_dec; -+ cnstr_shdsc_aead_null_decap(desc, &ctx->adata, ctx->authsize, -+ ctrlpriv->era); -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, -+ desc_bytes(desc), ctx->dir); - - return 0; - } -@@ -470,11 +172,12 @@ static int aead_set_sh_desc(struct crypt - unsigned int ivsize = crypto_aead_ivsize(aead); - struct caam_ctx *ctx = crypto_aead_ctx(aead); - struct device *jrdev = ctx->jrdev; -- bool keys_fit_inline; -- u32 geniv, moveiv; -+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); - u32 ctx1_iv_off = 0; -- u32 *desc; -- const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == -+ u32 *desc, *nonce = NULL; -+ u32 inl_mask; -+ unsigned int data_len[2]; -+ const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == - OP_ALG_AAI_CTR_MOD128); - const bool is_rfc3686 = alg->caam.rfc3686; - -@@ -482,7 +185,7 @@ static int aead_set_sh_desc(struct crypt - return 0; - - /* NULL encryption / decryption */ -- if (!ctx->enckeylen) -+ if (!ctx->cdata.keylen) - return aead_null_set_sh_desc(aead); - - /* -@@ -497,8 +200,14 @@ static int aead_set_sh_desc(struct crypt - * RFC3686 specific: - * CONTEXT1[255:128] = {NONCE, IV, COUNTER} - */ -- if (is_rfc3686) -+ if (is_rfc3686) { - ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; -+ nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + -+ ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); -+ } -+ -+ data_len[0] = ctx->adata.keylen_pad; -+ data_len[1] = ctx->cdata.keylen; - - if (alg->caam.geniv) - goto skip_enc; -@@ -507,146 +216,64 @@ static int aead_set_sh_desc(struct crypt - * Job Descriptor and Shared Descriptors - * must all fit into the 64-word Descriptor h/w Buffer - */ -- keys_fit_inline = false; -- if (DESC_AEAD_ENC_LEN + AUTHENC_DESC_JOB_IO_LEN + -- ctx->split_key_pad_len + ctx->enckeylen + -- (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= -- CAAM_DESC_BYTES_MAX) -- keys_fit_inline = true; -- -- /* aead_encrypt shared descriptor */ -- desc = ctx->sh_desc_enc; -- -- /* Note: Context registers are saved. */ -- init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); -- -- /* Class 2 operation */ -- append_operation(desc, ctx->class2_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); -- -- /* Read and write assoclen bytes */ -- append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); -- append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); -+ if (desc_inline_query(DESC_AEAD_ENC_LEN + -+ (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), -+ AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, -+ ARRAY_SIZE(data_len)) < 0) -+ return -EINVAL; - -- /* Skip assoc data */ -- append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); -+ if (inl_mask & 1) -+ ctx->adata.key_virt = ctx->key; -+ else -+ ctx->adata.key_dma = ctx->key_dma; - -- /* read assoc before reading payload */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | -- FIFOLDST_VLF); -+ if (inl_mask & 2) -+ ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; -+ else -+ ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; - -- /* Load Counter into CONTEXT1 reg */ -- if (is_rfc3686) -- append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | -- LDST_SRCDST_BYTE_CONTEXT | -- ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << -- LDST_OFFSET_SHIFT)); -- -- /* Class 1 operation */ -- append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); -- -- /* Read and write cryptlen bytes */ -- append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -- append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -- aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); -- -- /* Write ICV */ -- append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | -- LDST_SRCDST_BYTE_CONTEXT); -+ ctx->adata.key_inline = !!(inl_mask & 1); -+ ctx->cdata.key_inline = !!(inl_mask & 2); - -- ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, "aead enc shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -+ /* aead_encrypt shared descriptor */ -+ desc = ctx->sh_desc_enc; -+ cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, ivsize, -+ ctx->authsize, is_rfc3686, nonce, ctx1_iv_off, -+ false, ctrlpriv->era); -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, -+ desc_bytes(desc), ctx->dir); - - skip_enc: - /* - * Job Descriptor and Shared Descriptors - * must all fit into the 64-word Descriptor h/w Buffer - */ -- keys_fit_inline = false; -- if (DESC_AEAD_DEC_LEN + AUTHENC_DESC_JOB_IO_LEN + -- ctx->split_key_pad_len + ctx->enckeylen + -- (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= -- CAAM_DESC_BYTES_MAX) -- keys_fit_inline = true; -- -- /* aead_decrypt shared descriptor */ -- desc = ctx->sh_desc_dec; -- -- /* Note: Context registers are saved. */ -- init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); -- -- /* Class 2 operation */ -- append_operation(desc, ctx->class2_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); -+ if (desc_inline_query(DESC_AEAD_DEC_LEN + -+ (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), -+ AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, -+ ARRAY_SIZE(data_len)) < 0) -+ return -EINVAL; - -- /* Read and write assoclen bytes */ -- append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); -- if (alg->caam.geniv) -- append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, ivsize); -+ if (inl_mask & 1) -+ ctx->adata.key_virt = ctx->key; - else -- append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); -- -- /* Skip assoc data */ -- append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); -- -- /* read assoc before reading payload */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | -- KEY_VLF); -- -- if (alg->caam.geniv) { -- append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | -- LDST_SRCDST_BYTE_CONTEXT | -- (ctx1_iv_off << LDST_OFFSET_SHIFT)); -- append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | -- (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize); -- } -- -- /* Load Counter into CONTEXT1 reg */ -- if (is_rfc3686) -- append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | -- LDST_SRCDST_BYTE_CONTEXT | -- ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << -- LDST_OFFSET_SHIFT)); -+ ctx->adata.key_dma = ctx->key_dma; - -- /* Choose operation */ -- if (ctr_mode) -- append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); -+ if (inl_mask & 2) -+ ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; - else -- append_dec_op1(desc, ctx->class1_alg_type); -+ ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; - -- /* Read and write cryptlen bytes */ -- append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); -- append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); -- aead_append_src_dst(desc, FIFOLD_TYPE_MSG); -- -- /* Load ICV */ -- append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS2 | -- FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); -+ ctx->adata.key_inline = !!(inl_mask & 1); -+ ctx->cdata.key_inline = !!(inl_mask & 2); - -- ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, "aead dec shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -+ /* aead_decrypt shared descriptor */ -+ desc = ctx->sh_desc_dec; -+ cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, ivsize, -+ ctx->authsize, alg->caam.geniv, is_rfc3686, -+ nonce, ctx1_iv_off, false, ctrlpriv->era); -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, -+ desc_bytes(desc), ctx->dir); - - if (!alg->caam.geniv) - goto skip_givenc; -@@ -655,107 +282,32 @@ skip_enc: - * Job Descriptor and Shared Descriptors - * must all fit into the 64-word Descriptor h/w Buffer - */ -- keys_fit_inline = false; -- if (DESC_AEAD_GIVENC_LEN + AUTHENC_DESC_JOB_IO_LEN + -- ctx->split_key_pad_len + ctx->enckeylen + -- (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0) <= -- CAAM_DESC_BYTES_MAX) -- keys_fit_inline = true; -- -- /* aead_givencrypt shared descriptor */ -- desc = ctx->sh_desc_enc; -- -- /* Note: Context registers are saved. */ -- init_sh_desc_key_aead(desc, ctx, keys_fit_inline, is_rfc3686); -+ if (desc_inline_query(DESC_AEAD_GIVENC_LEN + -+ (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), -+ AUTHENC_DESC_JOB_IO_LEN, data_len, &inl_mask, -+ ARRAY_SIZE(data_len)) < 0) -+ return -EINVAL; - -- if (is_rfc3686) -- goto copy_iv; -+ if (inl_mask & 1) -+ ctx->adata.key_virt = ctx->key; -+ else -+ ctx->adata.key_dma = ctx->key_dma; - -- /* Generate IV */ -- geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | -- NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | -- NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT); -- append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | -- LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); -- append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); -- append_move(desc, MOVE_WAITCOMP | -- MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | -- (ctx1_iv_off << MOVE_OFFSET_SHIFT) | -- (ivsize << MOVE_LEN_SHIFT)); -- append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); -- --copy_iv: -- /* Copy IV to class 1 context */ -- append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | -- (ctx1_iv_off << MOVE_OFFSET_SHIFT) | -- (ivsize << MOVE_LEN_SHIFT)); -- -- /* Return to encryption */ -- append_operation(desc, ctx->class2_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); -- -- /* Read and write assoclen bytes */ -- append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); -- append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); -- -- /* ivsize + cryptlen = seqoutlen - authsize */ -- append_math_sub_imm_u32(desc, REG3, SEQOUTLEN, IMM, ctx->authsize); -- -- /* Skip assoc data */ -- append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); -- -- /* read assoc before reading payload */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | -- KEY_VLF); -- -- /* Copy iv from outfifo to class 2 fifo */ -- moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | -- NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT); -- append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | -- LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); -- append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB | -- LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); -+ if (inl_mask & 2) -+ ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; -+ else -+ ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; - -- /* Load Counter into CONTEXT1 reg */ -- if (is_rfc3686) -- append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | -- LDST_SRCDST_BYTE_CONTEXT | -- ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << -- LDST_OFFSET_SHIFT)); -- -- /* Class 1 operation */ -- append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); -- -- /* Will write ivsize + cryptlen */ -- append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -- -- /* Not need to reload iv */ -- append_seq_fifo_load(desc, ivsize, -- FIFOLD_CLASS_SKIP); -- -- /* Will read cryptlen */ -- append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF | -- FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH); -- append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); -- -- /* Write ICV */ -- append_seq_store(desc, ctx->authsize, LDST_CLASS_2_CCB | -- LDST_SRCDST_BYTE_CONTEXT); -+ ctx->adata.key_inline = !!(inl_mask & 1); -+ ctx->cdata.key_inline = !!(inl_mask & 2); - -- ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, "aead givenc shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -+ /* aead_givencrypt shared descriptor */ -+ desc = ctx->sh_desc_enc; -+ cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, ivsize, -+ ctx->authsize, is_rfc3686, nonce, -+ ctx1_iv_off, false, ctrlpriv->era); -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, -+ desc_bytes(desc), ctx->dir); - - skip_givenc: - return 0; -@@ -776,12 +328,12 @@ static int gcm_set_sh_desc(struct crypto - { - struct caam_ctx *ctx = crypto_aead_ctx(aead); - struct device *jrdev = ctx->jrdev; -- bool keys_fit_inline = false; -- u32 *key_jump_cmd, *zero_payload_jump_cmd, -- *zero_assoc_jump_cmd1, *zero_assoc_jump_cmd2; -+ unsigned int ivsize = crypto_aead_ivsize(aead); - u32 *desc; -+ int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - -+ ctx->cdata.keylen; - -- if (!ctx->enckeylen || !ctx->authsize) -+ if (!ctx->cdata.keylen || !ctx->authsize) - return 0; - - /* -@@ -789,175 +341,35 @@ static int gcm_set_sh_desc(struct crypto - * Job Descriptor and Shared Descriptor - * must fit into the 64-word Descriptor h/w Buffer - */ -- if (DESC_GCM_ENC_LEN + GCM_DESC_JOB_IO_LEN + -- ctx->enckeylen <= CAAM_DESC_BYTES_MAX) -- keys_fit_inline = true; -+ if (rem_bytes >= DESC_GCM_ENC_LEN) { -+ ctx->cdata.key_inline = true; -+ ctx->cdata.key_virt = ctx->key; -+ } else { -+ ctx->cdata.key_inline = false; -+ ctx->cdata.key_dma = ctx->key_dma; -+ } - - desc = ctx->sh_desc_enc; -- -- init_sh_desc(desc, HDR_SHARE_SERIAL); -- -- /* skip key loading if they are loaded due to sharing */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -- JUMP_COND_SHRD | JUMP_COND_SELF); -- if (keys_fit_inline) -- append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, -- ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); -- else -- append_key(desc, ctx->key_dma, ctx->enckeylen, -- CLASS_1 | KEY_DEST_CLASS_REG); -- set_jump_tgt_here(desc, key_jump_cmd); -- -- /* class 1 operation */ -- append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); -- -- /* if assoclen + cryptlen is ZERO, skip to ICV write */ -- append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -- zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | -- JUMP_COND_MATH_Z); -- -- /* if assoclen is ZERO, skip reading the assoc data */ -- append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); -- zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | -- JUMP_COND_MATH_Z); -- -- append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); -- -- /* skip assoc data */ -- append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); -- -- /* cryptlen = seqinlen - assoclen */ -- append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ); -- -- /* if cryptlen is ZERO jump to zero-payload commands */ -- zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | -- JUMP_COND_MATH_Z); -- -- /* read assoc data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); -- set_jump_tgt_here(desc, zero_assoc_jump_cmd1); -- -- append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -- -- /* write encrypted data */ -- append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); -- -- /* read payload data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); -- -- /* jump the zero-payload commands */ -- append_jump(desc, JUMP_TEST_ALL | 2); -- -- /* zero-payload commands */ -- set_jump_tgt_here(desc, zero_payload_jump_cmd); -- -- /* read assoc data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1); -- -- /* There is no input data */ -- set_jump_tgt_here(desc, zero_assoc_jump_cmd2); -- -- /* write ICV */ -- append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | -- LDST_SRCDST_BYTE_CONTEXT); -- -- ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, "gcm enc shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -+ cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, false); -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, -+ desc_bytes(desc), ctx->dir); - - /* - * Job Descriptor and Shared Descriptors - * must all fit into the 64-word Descriptor h/w Buffer - */ -- keys_fit_inline = false; -- if (DESC_GCM_DEC_LEN + GCM_DESC_JOB_IO_LEN + -- ctx->enckeylen <= CAAM_DESC_BYTES_MAX) -- keys_fit_inline = true; -+ if (rem_bytes >= DESC_GCM_DEC_LEN) { -+ ctx->cdata.key_inline = true; -+ ctx->cdata.key_virt = ctx->key; -+ } else { -+ ctx->cdata.key_inline = false; -+ ctx->cdata.key_dma = ctx->key_dma; -+ } - - desc = ctx->sh_desc_dec; -- -- init_sh_desc(desc, HDR_SHARE_SERIAL); -- -- /* skip key loading if they are loaded due to sharing */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | -- JUMP_TEST_ALL | JUMP_COND_SHRD | -- JUMP_COND_SELF); -- if (keys_fit_inline) -- append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, -- ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); -- else -- append_key(desc, ctx->key_dma, ctx->enckeylen, -- CLASS_1 | KEY_DEST_CLASS_REG); -- set_jump_tgt_here(desc, key_jump_cmd); -- -- /* class 1 operation */ -- append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); -- -- /* if assoclen is ZERO, skip reading the assoc data */ -- append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); -- zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | -- JUMP_COND_MATH_Z); -- -- append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); -- -- /* skip assoc data */ -- append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); -- -- /* read assoc data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); -- -- set_jump_tgt_here(desc, zero_assoc_jump_cmd1); -- -- /* cryptlen = seqoutlen - assoclen */ -- append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); -- -- /* jump to zero-payload command if cryptlen is zero */ -- zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | -- JUMP_COND_MATH_Z); -- -- append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); -- -- /* store encrypted data */ -- append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); -- -- /* read payload data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); -- -- /* zero-payload command */ -- set_jump_tgt_here(desc, zero_payload_jump_cmd); -- -- /* read ICV */ -- append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | -- FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); -- -- ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, "gcm dec shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -+ cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, false); -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, -+ desc_bytes(desc), ctx->dir); - - return 0; - } -@@ -976,11 +388,12 @@ static int rfc4106_set_sh_desc(struct cr - { - struct caam_ctx *ctx = crypto_aead_ctx(aead); - struct device *jrdev = ctx->jrdev; -- bool keys_fit_inline = false; -- u32 *key_jump_cmd; -+ unsigned int ivsize = crypto_aead_ivsize(aead); - u32 *desc; -+ int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - -+ ctx->cdata.keylen; - -- if (!ctx->enckeylen || !ctx->authsize) -+ if (!ctx->cdata.keylen || !ctx->authsize) - return 0; - - /* -@@ -988,148 +401,37 @@ static int rfc4106_set_sh_desc(struct cr - * Job Descriptor and Shared Descriptor - * must fit into the 64-word Descriptor h/w Buffer - */ -- if (DESC_RFC4106_ENC_LEN + GCM_DESC_JOB_IO_LEN + -- ctx->enckeylen <= CAAM_DESC_BYTES_MAX) -- keys_fit_inline = true; -+ if (rem_bytes >= DESC_RFC4106_ENC_LEN) { -+ ctx->cdata.key_inline = true; -+ ctx->cdata.key_virt = ctx->key; -+ } else { -+ ctx->cdata.key_inline = false; -+ ctx->cdata.key_dma = ctx->key_dma; -+ } - - desc = ctx->sh_desc_enc; -- -- init_sh_desc(desc, HDR_SHARE_SERIAL); -- -- /* Skip key loading if it is loaded due to sharing */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -- JUMP_COND_SHRD); -- if (keys_fit_inline) -- append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, -- ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); -- else -- append_key(desc, ctx->key_dma, ctx->enckeylen, -- CLASS_1 | KEY_DEST_CLASS_REG); -- set_jump_tgt_here(desc, key_jump_cmd); -- -- /* Class 1 operation */ -- append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); -- -- append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8); -- append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); -- -- /* Read assoc data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); -- -- /* Skip IV */ -- append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); -- -- /* Will read cryptlen bytes */ -- append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -- -- /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); -- -- /* Skip assoc data */ -- append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); -- -- /* cryptlen = seqoutlen - assoclen */ -- append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ); -- -- /* Write encrypted data */ -- append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); -- -- /* Read payload data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); -- -- /* Write ICV */ -- append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | -- LDST_SRCDST_BYTE_CONTEXT); -- -- ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, "rfc4106 enc shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -+ cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize, -+ false); -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, -+ desc_bytes(desc), ctx->dir); - - /* - * Job Descriptor and Shared Descriptors - * must all fit into the 64-word Descriptor h/w Buffer - */ -- keys_fit_inline = false; -- if (DESC_RFC4106_DEC_LEN + DESC_JOB_IO_LEN + -- ctx->enckeylen <= CAAM_DESC_BYTES_MAX) -- keys_fit_inline = true; -+ if (rem_bytes >= DESC_RFC4106_DEC_LEN) { -+ ctx->cdata.key_inline = true; -+ ctx->cdata.key_virt = ctx->key; -+ } else { -+ ctx->cdata.key_inline = false; -+ ctx->cdata.key_dma = ctx->key_dma; -+ } - - desc = ctx->sh_desc_dec; -- -- init_sh_desc(desc, HDR_SHARE_SERIAL); -- -- /* Skip key loading if it is loaded due to sharing */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | -- JUMP_TEST_ALL | JUMP_COND_SHRD); -- if (keys_fit_inline) -- append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, -- ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); -- else -- append_key(desc, ctx->key_dma, ctx->enckeylen, -- CLASS_1 | KEY_DEST_CLASS_REG); -- set_jump_tgt_here(desc, key_jump_cmd); -- -- /* Class 1 operation */ -- append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); -- -- append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, 8); -- append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); -- -- /* Read assoc data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); -- -- /* Skip IV */ -- append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); -- -- /* Will read cryptlen bytes */ -- append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ); -- -- /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); -- -- /* Skip assoc data */ -- append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); -- -- /* Will write cryptlen bytes */ -- append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); -- -- /* Store payload data */ -- append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); -- -- /* Read encrypted data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -- FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); -- -- /* Read ICV */ -- append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | -- FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); -- -- ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, "rfc4106 dec shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -+ cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize, -+ false); -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, -+ desc_bytes(desc), ctx->dir); - - return 0; - } -@@ -1149,12 +451,12 @@ static int rfc4543_set_sh_desc(struct cr - { - struct caam_ctx *ctx = crypto_aead_ctx(aead); - struct device *jrdev = ctx->jrdev; -- bool keys_fit_inline = false; -- u32 *key_jump_cmd; -- u32 *read_move_cmd, *write_move_cmd; -+ unsigned int ivsize = crypto_aead_ivsize(aead); - u32 *desc; -+ int rem_bytes = CAAM_DESC_BYTES_MAX - GCM_DESC_JOB_IO_LEN - -+ ctx->cdata.keylen; - -- if (!ctx->enckeylen || !ctx->authsize) -+ if (!ctx->cdata.keylen || !ctx->authsize) - return 0; - - /* -@@ -1162,151 +464,37 @@ static int rfc4543_set_sh_desc(struct cr - * Job Descriptor and Shared Descriptor - * must fit into the 64-word Descriptor h/w Buffer - */ -- if (DESC_RFC4543_ENC_LEN + GCM_DESC_JOB_IO_LEN + -- ctx->enckeylen <= CAAM_DESC_BYTES_MAX) -- keys_fit_inline = true; -+ if (rem_bytes >= DESC_RFC4543_ENC_LEN) { -+ ctx->cdata.key_inline = true; -+ ctx->cdata.key_virt = ctx->key; -+ } else { -+ ctx->cdata.key_inline = false; -+ ctx->cdata.key_dma = ctx->key_dma; -+ } - - desc = ctx->sh_desc_enc; -- -- init_sh_desc(desc, HDR_SHARE_SERIAL); -- -- /* Skip key loading if it is loaded due to sharing */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -- JUMP_COND_SHRD); -- if (keys_fit_inline) -- append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, -- ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); -- else -- append_key(desc, ctx->key_dma, ctx->enckeylen, -- CLASS_1 | KEY_DEST_CLASS_REG); -- set_jump_tgt_here(desc, key_jump_cmd); -- -- /* Class 1 operation */ -- append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); -- -- /* assoclen + cryptlen = seqinlen */ -- append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); -- -- /* -- * MOVE_LEN opcode is not available in all SEC HW revisions, -- * thus need to do some magic, i.e. self-patch the descriptor -- * buffer. -- */ -- read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | -- (0x6 << MOVE_LEN_SHIFT)); -- write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | -- (0x8 << MOVE_LEN_SHIFT)); -- -- /* Will read assoclen + cryptlen bytes */ -- append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -- -- /* Will write assoclen + cryptlen bytes */ -- append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -- -- /* Read and write assoclen + cryptlen bytes */ -- aead_append_src_dst(desc, FIFOLD_TYPE_AAD); -- -- set_move_tgt_here(desc, read_move_cmd); -- set_move_tgt_here(desc, write_move_cmd); -- append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); -- /* Move payload data to OFIFO */ -- append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); -- -- /* Write ICV */ -- append_seq_store(desc, ctx->authsize, LDST_CLASS_1_CCB | -- LDST_SRCDST_BYTE_CONTEXT); -- -- ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, "rfc4543 enc shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -+ cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize, -+ false); -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, -+ desc_bytes(desc), ctx->dir); - - /* - * Job Descriptor and Shared Descriptors - * must all fit into the 64-word Descriptor h/w Buffer - */ -- keys_fit_inline = false; -- if (DESC_RFC4543_DEC_LEN + GCM_DESC_JOB_IO_LEN + -- ctx->enckeylen <= CAAM_DESC_BYTES_MAX) -- keys_fit_inline = true; -+ if (rem_bytes >= DESC_RFC4543_DEC_LEN) { -+ ctx->cdata.key_inline = true; -+ ctx->cdata.key_virt = ctx->key; -+ } else { -+ ctx->cdata.key_inline = false; -+ ctx->cdata.key_dma = ctx->key_dma; -+ } - - desc = ctx->sh_desc_dec; -- -- init_sh_desc(desc, HDR_SHARE_SERIAL); -- -- /* Skip key loading if it is loaded due to sharing */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | -- JUMP_TEST_ALL | JUMP_COND_SHRD); -- if (keys_fit_inline) -- append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, -- ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); -- else -- append_key(desc, ctx->key_dma, ctx->enckeylen, -- CLASS_1 | KEY_DEST_CLASS_REG); -- set_jump_tgt_here(desc, key_jump_cmd); -- -- /* Class 1 operation */ -- append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT | OP_ALG_ICV_ON); -- -- /* assoclen + cryptlen = seqoutlen */ -- append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ); -- -- /* -- * MOVE_LEN opcode is not available in all SEC HW revisions, -- * thus need to do some magic, i.e. self-patch the descriptor -- * buffer. -- */ -- read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | -- (0x6 << MOVE_LEN_SHIFT)); -- write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | -- (0x8 << MOVE_LEN_SHIFT)); -- -- /* Will read assoclen + cryptlen bytes */ -- append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); -- -- /* Will write assoclen + cryptlen bytes */ -- append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); -- -- /* Store payload data */ -- append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); -- -- /* In-snoop assoclen + cryptlen data */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF | -- FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1); -- -- set_move_tgt_here(desc, read_move_cmd); -- set_move_tgt_here(desc, write_move_cmd); -- append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); -- /* Move payload data to OFIFO */ -- append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); -- append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); -- -- /* Read ICV */ -- append_seq_fifo_load(desc, ctx->authsize, FIFOLD_CLASS_CLASS1 | -- FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); -- -- ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, "rfc4543 dec shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -+ cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize, -+ false); -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, -+ desc_bytes(desc), ctx->dir); - - return 0; - } -@@ -1322,74 +510,67 @@ static int rfc4543_setauthsize(struct cr - return 0; - } - --static u32 gen_split_aead_key(struct caam_ctx *ctx, const u8 *key_in, -- u32 authkeylen) --{ -- return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len, -- ctx->split_key_pad_len, key_in, authkeylen, -- ctx->alg_op); --} -- - static int aead_setkey(struct crypto_aead *aead, - const u8 *key, unsigned int keylen) - { -- /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ -- static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; - struct caam_ctx *ctx = crypto_aead_ctx(aead); - struct device *jrdev = ctx->jrdev; -+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); - struct crypto_authenc_keys keys; - int ret = 0; - - if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) - goto badkey; - -- /* Pick class 2 key length from algorithm submask */ -- ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >> -- OP_ALG_ALGSEL_SHIFT] * 2; -- ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16); -- -- if (ctx->split_key_pad_len + keys.enckeylen > CAAM_MAX_KEY_SIZE) -- goto badkey; -- - #ifdef DEBUG - printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n", - keys.authkeylen + keys.enckeylen, keys.enckeylen, - keys.authkeylen); -- printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n", -- ctx->split_key_len, ctx->split_key_pad_len); - print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); - #endif - -- ret = gen_split_aead_key(ctx, keys.authkey, keys.authkeylen); -+ /* -+ * If DKP is supported, use it in the shared descriptor to generate -+ * the split key. -+ */ -+ if (ctrlpriv->era >= 6) { -+ ctx->adata.keylen = keys.authkeylen; -+ ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & -+ OP_ALG_ALGSEL_MASK); -+ -+ if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) -+ goto badkey; -+ -+ memcpy(ctx->key, keys.authkey, keys.authkeylen); -+ memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, -+ keys.enckeylen); -+ dma_sync_single_for_device(jrdev, ctx->key_dma, -+ ctx->adata.keylen_pad + -+ keys.enckeylen, ctx->dir); -+ goto skip_split_key; -+ } -+ -+ ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, keys.authkey, -+ keys.authkeylen, CAAM_MAX_KEY_SIZE - -+ keys.enckeylen); - if (ret) { - goto badkey; - } - - /* postpend encryption key to auth split key */ -- memcpy(ctx->key + ctx->split_key_pad_len, keys.enckey, keys.enckeylen); -- -- ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len + -- keys.enckeylen, DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->key_dma)) { -- dev_err(jrdev, "unable to map key i/o memory\n"); -- return -ENOMEM; -- } -+ memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); -+ dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + -+ keys.enckeylen, ctx->dir); - #ifdef DEBUG - print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, -- ctx->split_key_pad_len + keys.enckeylen, 1); -+ ctx->adata.keylen_pad + keys.enckeylen, 1); - #endif - -- ctx->enckeylen = keys.enckeylen; -- -- ret = aead_set_sh_desc(aead); -- if (ret) { -- dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len + -- keys.enckeylen, DMA_TO_DEVICE); -- } -- -- return ret; -+skip_split_key: -+ ctx->cdata.keylen = keys.enckeylen; -+ return aead_set_sh_desc(aead); - badkey: - crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); - return -EINVAL; -@@ -1400,7 +581,6 @@ static int gcm_setkey(struct crypto_aead - { - struct caam_ctx *ctx = crypto_aead_ctx(aead); - struct device *jrdev = ctx->jrdev; -- int ret = 0; - - #ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", -@@ -1408,21 +588,10 @@ static int gcm_setkey(struct crypto_aead - #endif - - memcpy(ctx->key, key, keylen); -- ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->key_dma)) { -- dev_err(jrdev, "unable to map key i/o memory\n"); -- return -ENOMEM; -- } -- ctx->enckeylen = keylen; -+ dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir); -+ ctx->cdata.keylen = keylen; - -- ret = gcm_set_sh_desc(aead); -- if (ret) { -- dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, -- DMA_TO_DEVICE); -- } -- -- return ret; -+ return gcm_set_sh_desc(aead); - } - - static int rfc4106_setkey(struct crypto_aead *aead, -@@ -1430,7 +599,6 @@ static int rfc4106_setkey(struct crypto_ - { - struct caam_ctx *ctx = crypto_aead_ctx(aead); - struct device *jrdev = ctx->jrdev; -- int ret = 0; - - if (keylen < 4) - return -EINVAL; -@@ -1446,22 +614,10 @@ static int rfc4106_setkey(struct crypto_ - * The last four bytes of the key material are used as the salt value - * in the nonce. Update the AES key length. - */ -- ctx->enckeylen = keylen - 4; -- -- ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen, -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->key_dma)) { -- dev_err(jrdev, "unable to map key i/o memory\n"); -- return -ENOMEM; -- } -- -- ret = rfc4106_set_sh_desc(aead); -- if (ret) { -- dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, -- DMA_TO_DEVICE); -- } -- -- return ret; -+ ctx->cdata.keylen = keylen - 4; -+ dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, -+ ctx->dir); -+ return rfc4106_set_sh_desc(aead); - } - - static int rfc4543_setkey(struct crypto_aead *aead, -@@ -1469,7 +625,6 @@ static int rfc4543_setkey(struct crypto_ - { - struct caam_ctx *ctx = crypto_aead_ctx(aead); - struct device *jrdev = ctx->jrdev; -- int ret = 0; - - if (keylen < 4) - return -EINVAL; -@@ -1485,43 +640,28 @@ static int rfc4543_setkey(struct crypto_ - * The last four bytes of the key material are used as the salt value - * in the nonce. Update the AES key length. - */ -- ctx->enckeylen = keylen - 4; -- -- ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->enckeylen, -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->key_dma)) { -- dev_err(jrdev, "unable to map key i/o memory\n"); -- return -ENOMEM; -- } -- -- ret = rfc4543_set_sh_desc(aead); -- if (ret) { -- dma_unmap_single(jrdev, ctx->key_dma, ctx->enckeylen, -- DMA_TO_DEVICE); -- } -- -- return ret; -+ ctx->cdata.keylen = keylen - 4; -+ dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, -+ ctx->dir); -+ return rfc4543_set_sh_desc(aead); - } - - static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, - const u8 *key, unsigned int keylen) - { - struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); -- struct ablkcipher_tfm *crt = &ablkcipher->base.crt_ablkcipher; - struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher); - const char *alg_name = crypto_tfm_alg_name(tfm); - struct device *jrdev = ctx->jrdev; -- int ret = 0; -- u32 *key_jump_cmd; -+ unsigned int ivsize = crypto_ablkcipher_ivsize(ablkcipher); - u32 *desc; -- u8 *nonce; -- u32 geniv; - u32 ctx1_iv_off = 0; -- const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == -+ const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == - OP_ALG_AAI_CTR_MOD128); - const bool is_rfc3686 = (ctr_mode && - (strstr(alg_name, "rfc3686") != NULL)); - -+ memcpy(ctx->key, key, keylen); - #ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -@@ -1544,215 +684,33 @@ static int ablkcipher_setkey(struct cryp - keylen -= CTR_RFC3686_NONCE_SIZE; - } - -- memcpy(ctx->key, key, keylen); -- ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->key_dma)) { -- dev_err(jrdev, "unable to map key i/o memory\n"); -- return -ENOMEM; -- } -- ctx->enckeylen = keylen; -+ dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); -+ ctx->cdata.keylen = keylen; -+ ctx->cdata.key_virt = ctx->key; -+ ctx->cdata.key_inline = true; - - /* ablkcipher_encrypt shared descriptor */ - desc = ctx->sh_desc_enc; -- init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); -- /* Skip if already shared */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -- JUMP_COND_SHRD); -- -- /* Load class1 key only */ -- append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, -- ctx->enckeylen, CLASS_1 | -- KEY_DEST_CLASS_REG); -- -- /* Load nonce into CONTEXT1 reg */ -- if (is_rfc3686) { -- nonce = (u8 *)key + keylen; -- append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, -- LDST_CLASS_IND_CCB | -- LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); -- append_move(desc, MOVE_WAITCOMP | -- MOVE_SRC_OUTFIFO | -- MOVE_DEST_CLASS1CTX | -- (16 << MOVE_OFFSET_SHIFT) | -- (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); -- } -- -- set_jump_tgt_here(desc, key_jump_cmd); -- -- /* Load iv */ -- append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT | -- LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); -- -- /* Load counter into CONTEXT1 reg */ -- if (is_rfc3686) -- append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | -- LDST_SRCDST_BYTE_CONTEXT | -- ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << -- LDST_OFFSET_SHIFT)); -- -- /* Load operation */ -- append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); -- -- /* Perform operation */ -- ablkcipher_append_src_dst(desc); -+ cnstr_shdsc_ablkcipher_encap(desc, &ctx->cdata, ivsize, is_rfc3686, -+ ctx1_iv_off); -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, -+ desc_bytes(desc), ctx->dir); - -- ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, -- "ablkcipher enc shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif - /* ablkcipher_decrypt shared descriptor */ - desc = ctx->sh_desc_dec; -+ cnstr_shdsc_ablkcipher_decap(desc, &ctx->cdata, ivsize, is_rfc3686, -+ ctx1_iv_off); -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, -+ desc_bytes(desc), ctx->dir); - -- init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); -- /* Skip if already shared */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -- JUMP_COND_SHRD); -- -- /* Load class1 key only */ -- append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, -- ctx->enckeylen, CLASS_1 | -- KEY_DEST_CLASS_REG); -- -- /* Load nonce into CONTEXT1 reg */ -- if (is_rfc3686) { -- nonce = (u8 *)key + keylen; -- append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, -- LDST_CLASS_IND_CCB | -- LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); -- append_move(desc, MOVE_WAITCOMP | -- MOVE_SRC_OUTFIFO | -- MOVE_DEST_CLASS1CTX | -- (16 << MOVE_OFFSET_SHIFT) | -- (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); -- } -- -- set_jump_tgt_here(desc, key_jump_cmd); -- -- /* load IV */ -- append_seq_load(desc, crt->ivsize, LDST_SRCDST_BYTE_CONTEXT | -- LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); -- -- /* Load counter into CONTEXT1 reg */ -- if (is_rfc3686) -- append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | -- LDST_SRCDST_BYTE_CONTEXT | -- ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << -- LDST_OFFSET_SHIFT)); -- -- /* Choose operation */ -- if (ctr_mode) -- append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); -- else -- append_dec_op1(desc, ctx->class1_alg_type); -- -- /* Perform operation */ -- ablkcipher_append_src_dst(desc); -- -- ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } -- --#ifdef DEBUG -- print_hex_dump(KERN_ERR, -- "ablkcipher dec shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif - /* ablkcipher_givencrypt shared descriptor */ - desc = ctx->sh_desc_givenc; -+ cnstr_shdsc_ablkcipher_givencap(desc, &ctx->cdata, ivsize, is_rfc3686, -+ ctx1_iv_off); -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_givenc_dma, -+ desc_bytes(desc), ctx->dir); - -- init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); -- /* Skip if already shared */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -- JUMP_COND_SHRD); -- -- /* Load class1 key only */ -- append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, -- ctx->enckeylen, CLASS_1 | -- KEY_DEST_CLASS_REG); -- -- /* Load Nonce into CONTEXT1 reg */ -- if (is_rfc3686) { -- nonce = (u8 *)key + keylen; -- append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, -- LDST_CLASS_IND_CCB | -- LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); -- append_move(desc, MOVE_WAITCOMP | -- MOVE_SRC_OUTFIFO | -- MOVE_DEST_CLASS1CTX | -- (16 << MOVE_OFFSET_SHIFT) | -- (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); -- } -- set_jump_tgt_here(desc, key_jump_cmd); -- -- /* Generate IV */ -- geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | -- NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | -- NFIFOENTRY_PTYPE_RND | (crt->ivsize << NFIFOENTRY_DLEN_SHIFT); -- append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | -- LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); -- append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); -- append_move(desc, MOVE_WAITCOMP | -- MOVE_SRC_INFIFO | -- MOVE_DEST_CLASS1CTX | -- (crt->ivsize << MOVE_LEN_SHIFT) | -- (ctx1_iv_off << MOVE_OFFSET_SHIFT)); -- append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); -- -- /* Copy generated IV to memory */ -- append_seq_store(desc, crt->ivsize, -- LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | -- (ctx1_iv_off << LDST_OFFSET_SHIFT)); -- -- /* Load Counter into CONTEXT1 reg */ -- if (is_rfc3686) -- append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | -- LDST_SRCDST_BYTE_CONTEXT | -- ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << -- LDST_OFFSET_SHIFT)); -- -- if (ctx1_iv_off) -- append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP | -- (1 << JUMP_OFFSET_SHIFT)); -- -- /* Load operation */ -- append_operation(desc, ctx->class1_alg_type | -- OP_ALG_AS_INITFINAL | OP_ALG_ENCRYPT); -- -- /* Perform operation */ -- ablkcipher_append_src_dst(desc); -- -- ctx->sh_desc_givenc_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_givenc_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, -- "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -- -- return ret; -+ return 0; - } - - static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, -@@ -1760,8 +718,7 @@ static int xts_ablkcipher_setkey(struct - { - struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); - struct device *jrdev = ctx->jrdev; -- u32 *key_jump_cmd, *desc; -- __be64 sector_size = cpu_to_be64(512); -+ u32 *desc; - - if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { - crypto_ablkcipher_set_flags(ablkcipher, -@@ -1771,126 +728,38 @@ static int xts_ablkcipher_setkey(struct - } - - memcpy(ctx->key, key, keylen); -- ctx->key_dma = dma_map_single(jrdev, ctx->key, keylen, DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->key_dma)) { -- dev_err(jrdev, "unable to map key i/o memory\n"); -- return -ENOMEM; -- } -- ctx->enckeylen = keylen; -+ dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); -+ ctx->cdata.keylen = keylen; -+ ctx->cdata.key_virt = ctx->key; -+ ctx->cdata.key_inline = true; - - /* xts_ablkcipher_encrypt shared descriptor */ - desc = ctx->sh_desc_enc; -- init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); -- /* Skip if already shared */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -- JUMP_COND_SHRD); -- -- /* Load class1 keys only */ -- append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, -- ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); -- -- /* Load sector size with index 40 bytes (0x28) */ -- append_cmd(desc, CMD_LOAD | IMMEDIATE | LDST_SRCDST_BYTE_CONTEXT | -- LDST_CLASS_1_CCB | (0x28 << LDST_OFFSET_SHIFT) | 8); -- append_data(desc, (void *)§or_size, 8); -- -- set_jump_tgt_here(desc, key_jump_cmd); -- -- /* -- * create sequence for loading the sector index -- * Upper 8B of IV - will be used as sector index -- * Lower 8B of IV - will be discarded -- */ -- append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | -- LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT) | 8); -- append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); -- -- /* Load operation */ -- append_operation(desc, ctx->class1_alg_type | OP_ALG_AS_INITFINAL | -- OP_ALG_ENCRYPT); -- -- /* Perform operation */ -- ablkcipher_append_src_dst(desc); -- -- ctx->sh_desc_enc_dma = dma_map_single(jrdev, desc, desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_enc_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, -- "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); --#endif -+ cnstr_shdsc_xts_ablkcipher_encap(desc, &ctx->cdata); -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_enc_dma, -+ desc_bytes(desc), ctx->dir); - - /* xts_ablkcipher_decrypt shared descriptor */ - desc = ctx->sh_desc_dec; -- -- init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); -- /* Skip if already shared */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -- JUMP_COND_SHRD); -- -- /* Load class1 key only */ -- append_key_as_imm(desc, (void *)ctx->key, ctx->enckeylen, -- ctx->enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); -- -- /* Load sector size with index 40 bytes (0x28) */ -- append_cmd(desc, CMD_LOAD | IMMEDIATE | LDST_SRCDST_BYTE_CONTEXT | -- LDST_CLASS_1_CCB | (0x28 << LDST_OFFSET_SHIFT) | 8); -- append_data(desc, (void *)§or_size, 8); -- -- set_jump_tgt_here(desc, key_jump_cmd); -- -- /* -- * create sequence for loading the sector index -- * Upper 8B of IV - will be used as sector index -- * Lower 8B of IV - will be discarded -- */ -- append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | -- LDST_CLASS_1_CCB | (0x20 << LDST_OFFSET_SHIFT) | 8); -- append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); -- -- /* Load operation */ -- append_dec_op1(desc, ctx->class1_alg_type); -- -- /* Perform operation */ -- ablkcipher_append_src_dst(desc); -- -- ctx->sh_desc_dec_dma = dma_map_single(jrdev, desc, desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_dec_dma)) { -- dma_unmap_single(jrdev, ctx->sh_desc_enc_dma, -- desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE); -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, -- "xts ablkcipher dec shdesc@" __stringify(__LINE__) ": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); --#endif -+ cnstr_shdsc_xts_ablkcipher_decap(desc, &ctx->cdata); -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_dec_dma, -+ desc_bytes(desc), ctx->dir); - - return 0; - } - - /* - * aead_edesc - s/w-extended aead descriptor -- * @assoc_nents: number of segments in associated data (SPI+Seq) scatterlist -- * @src_nents: number of segments in input scatterlist -- * @dst_nents: number of segments in output scatterlist -- * @iv_dma: dma address of iv for checking continuity and link table -- * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) -+ * @src_nents: number of segments in input s/w scatterlist -+ * @dst_nents: number of segments in output s/w scatterlist - * @sec4_sg_bytes: length of dma mapped sec4_sg space - * @sec4_sg_dma: bus physical mapped address of h/w link table -+ * @sec4_sg: pointer to h/w link table - * @hw_desc: the h/w job descriptor followed by any referenced link tables - */ - struct aead_edesc { -- int assoc_nents; - int src_nents; - int dst_nents; -- dma_addr_t iv_dma; - int sec4_sg_bytes; - dma_addr_t sec4_sg_dma; - struct sec4_sg_entry *sec4_sg; -@@ -1899,12 +768,12 @@ struct aead_edesc { - - /* - * ablkcipher_edesc - s/w-extended ablkcipher descriptor -- * @src_nents: number of segments in input scatterlist -- * @dst_nents: number of segments in output scatterlist -+ * @src_nents: number of segments in input s/w scatterlist -+ * @dst_nents: number of segments in output s/w scatterlist - * @iv_dma: dma address of iv for checking continuity and link table -- * @desc: h/w descriptor (variable length; must not exceed MAX_CAAM_DESCSIZE) - * @sec4_sg_bytes: length of dma mapped sec4_sg space - * @sec4_sg_dma: bus physical mapped address of h/w link table -+ * @sec4_sg: pointer to h/w link table - * @hw_desc: the h/w job descriptor followed by any referenced link tables - */ - struct ablkcipher_edesc { -@@ -1924,10 +793,11 @@ static void caam_unmap(struct device *de - int sec4_sg_bytes) - { - if (dst != src) { -- dma_unmap_sg(dev, src, src_nents ? : 1, DMA_TO_DEVICE); -- dma_unmap_sg(dev, dst, dst_nents ? : 1, DMA_FROM_DEVICE); -+ if (src_nents) -+ dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); -+ dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); - } else { -- dma_unmap_sg(dev, src, src_nents ? : 1, DMA_BIDIRECTIONAL); -+ dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); - } - - if (iv_dma) -@@ -2021,8 +891,7 @@ static void ablkcipher_encrypt_done(stru - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); - #endif - -- edesc = (struct ablkcipher_edesc *)((char *)desc - -- offsetof(struct ablkcipher_edesc, hw_desc)); -+ edesc = container_of(desc, struct ablkcipher_edesc, hw_desc[0]); - - if (err) - caam_jr_strstatus(jrdev, err); -@@ -2031,10 +900,10 @@ static void ablkcipher_encrypt_done(stru - print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->info, - edesc->src_nents > 1 ? 100 : ivsize, 1); -- dbg_dump_sg(KERN_ERR, "dst @"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, req->dst, -- edesc->dst_nents > 1 ? 100 : req->nbytes, 1, true); - #endif -+ caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, req->dst, -+ edesc->dst_nents > 1 ? 100 : req->nbytes, 1); - - ablkcipher_unmap(jrdev, edesc, req); - -@@ -2062,8 +931,7 @@ static void ablkcipher_decrypt_done(stru - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); - #endif - -- edesc = (struct ablkcipher_edesc *)((char *)desc - -- offsetof(struct ablkcipher_edesc, hw_desc)); -+ edesc = container_of(desc, struct ablkcipher_edesc, hw_desc[0]); - if (err) - caam_jr_strstatus(jrdev, err); - -@@ -2071,10 +939,10 @@ static void ablkcipher_decrypt_done(stru - print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->info, - ivsize, 1); -- dbg_dump_sg(KERN_ERR, "dst @"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, req->dst, -- edesc->dst_nents > 1 ? 100 : req->nbytes, 1, true); - #endif -+ caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, req->dst, -+ edesc->dst_nents > 1 ? 100 : req->nbytes, 1); - - ablkcipher_unmap(jrdev, edesc, req); - -@@ -2114,7 +982,7 @@ static void init_aead_job(struct aead_re - init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); - - if (all_contig) { -- src_dma = sg_dma_address(req->src); -+ src_dma = edesc->src_nents ? sg_dma_address(req->src) : 0; - in_options = 0; - } else { - src_dma = edesc->sec4_sg_dma; -@@ -2129,7 +997,7 @@ static void init_aead_job(struct aead_re - out_options = in_options; - - if (unlikely(req->src != req->dst)) { -- if (!edesc->dst_nents) { -+ if (edesc->dst_nents == 1) { - dst_dma = sg_dma_address(req->dst); - } else { - dst_dma = edesc->sec4_sg_dma + -@@ -2147,9 +1015,6 @@ static void init_aead_job(struct aead_re - append_seq_out_ptr(desc, dst_dma, - req->assoclen + req->cryptlen - authsize, - out_options); -- -- /* REG3 = assoclen */ -- append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); - } - - static void init_gcm_job(struct aead_request *req, -@@ -2164,6 +1029,7 @@ static void init_gcm_job(struct aead_req - unsigned int last; - - init_aead_job(req, edesc, all_contig, encrypt); -+ append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); - - /* BUG This should not be specific to generic GCM. */ - last = 0; -@@ -2175,7 +1041,7 @@ static void init_gcm_job(struct aead_req - FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | 12 | last); - /* Append Salt */ - if (!generic_gcm) -- append_data(desc, ctx->key + ctx->enckeylen, 4); -+ append_data(desc, ctx->key + ctx->cdata.keylen, 4); - /* Append IV */ - append_data(desc, req->iv, ivsize); - /* End of blank commands */ -@@ -2190,7 +1056,8 @@ static void init_authenc_job(struct aead - struct caam_aead_alg, aead); - unsigned int ivsize = crypto_aead_ivsize(aead); - struct caam_ctx *ctx = crypto_aead_ctx(aead); -- const bool ctr_mode = ((ctx->class1_alg_type & OP_ALG_AAI_MASK) == -+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); -+ const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == - OP_ALG_AAI_CTR_MOD128); - const bool is_rfc3686 = alg->caam.rfc3686; - u32 *desc = edesc->hw_desc; -@@ -2213,6 +1080,15 @@ static void init_authenc_job(struct aead - - init_aead_job(req, edesc, all_contig, encrypt); - -+ /* -+ * {REG3, DPOVRD} = assoclen, depending on whether MATH command supports -+ * having DPOVRD as destination. -+ */ -+ if (ctrlpriv->era < 3) -+ append_math_add_imm_u32(desc, REG3, ZERO, IMM, req->assoclen); -+ else -+ append_math_add_imm_u32(desc, DPOVRD, ZERO, IMM, req->assoclen); -+ - if (ivsize && ((is_rfc3686 && encrypt) || !alg->caam.geniv)) - append_load_as_imm(desc, req->iv, ivsize, - LDST_CLASS_1_CCB | -@@ -2236,16 +1112,15 @@ static void init_ablkcipher_job(u32 *sh_ - int len, sec4_sg_index = 0; - - #ifdef DEBUG -- bool may_sleep = ((req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | -- CRYPTO_TFM_REQ_MAY_SLEEP)) != 0); - print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->info, - ivsize, 1); -- printk(KERN_ERR "asked=%d, nbytes%d\n", (int)edesc->src_nents ? 100 : req->nbytes, req->nbytes); -- dbg_dump_sg(KERN_ERR, "src @"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, req->src, -- edesc->src_nents ? 100 : req->nbytes, 1, may_sleep); -+ pr_err("asked=%d, nbytes%d\n", -+ (int)edesc->src_nents > 1 ? 100 : req->nbytes, req->nbytes); - #endif -+ caam_dump_sg(KERN_ERR, "src @" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, req->src, -+ edesc->src_nents > 1 ? 100 : req->nbytes, 1); - - len = desc_len(sh_desc); - init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); -@@ -2261,7 +1136,7 @@ static void init_ablkcipher_job(u32 *sh_ - append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options); - - if (likely(req->src == req->dst)) { -- if (!edesc->src_nents && iv_contig) { -+ if (edesc->src_nents == 1 && iv_contig) { - dst_dma = sg_dma_address(req->src); - } else { - dst_dma = edesc->sec4_sg_dma + -@@ -2269,7 +1144,7 @@ static void init_ablkcipher_job(u32 *sh_ - out_options = LDST_SGF; - } - } else { -- if (!edesc->dst_nents) { -+ if (edesc->dst_nents == 1) { - dst_dma = sg_dma_address(req->dst); - } else { - dst_dma = edesc->sec4_sg_dma + -@@ -2296,20 +1171,18 @@ static void init_ablkcipher_giv_job(u32 - int len, sec4_sg_index = 0; - - #ifdef DEBUG -- bool may_sleep = ((req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | -- CRYPTO_TFM_REQ_MAY_SLEEP)) != 0); - print_hex_dump(KERN_ERR, "presciv@" __stringify(__LINE__) ": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->info, - ivsize, 1); -- dbg_dump_sg(KERN_ERR, "src @" __stringify(__LINE__) ": ", -- DUMP_PREFIX_ADDRESS, 16, 4, req->src, -- edesc->src_nents ? 100 : req->nbytes, 1, may_sleep); - #endif -+ caam_dump_sg(KERN_ERR, "src @" __stringify(__LINE__) ": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, req->src, -+ edesc->src_nents > 1 ? 100 : req->nbytes, 1); - - len = desc_len(sh_desc); - init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); - -- if (!edesc->src_nents) { -+ if (edesc->src_nents == 1) { - src_dma = sg_dma_address(req->src); - in_options = 0; - } else { -@@ -2340,87 +1213,100 @@ static struct aead_edesc *aead_edesc_all - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct caam_ctx *ctx = crypto_aead_ctx(aead); - struct device *jrdev = ctx->jrdev; -- gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | -- CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; -- int src_nents, dst_nents = 0; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; -+ int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; - struct aead_edesc *edesc; -- int sgc; -- bool all_contig = true; -- int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; -+ int sec4_sg_index, sec4_sg_len, sec4_sg_bytes; - unsigned int authsize = ctx->authsize; - - if (unlikely(req->dst != req->src)) { -- src_nents = sg_count(req->src, req->assoclen + req->cryptlen); -- dst_nents = sg_count(req->dst, -- req->assoclen + req->cryptlen + -- (encrypt ? authsize : (-authsize))); -- } else { -- src_nents = sg_count(req->src, -- req->assoclen + req->cryptlen + -- (encrypt ? authsize : 0)); -- } -- -- /* Check if data are contiguous. */ -- all_contig = !src_nents; -- if (!all_contig) { -- src_nents = src_nents ? : 1; -- sec4_sg_len = src_nents; -- } -- -- sec4_sg_len += dst_nents; -- -- sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); -+ src_nents = sg_nents_for_len(req->src, req->assoclen + -+ req->cryptlen); -+ if (unlikely(src_nents < 0)) { -+ dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", -+ req->assoclen + req->cryptlen); -+ return ERR_PTR(src_nents); -+ } - -- /* allocate space for base edesc and hw desc commands, link tables */ -- edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, -- GFP_DMA | flags); -- if (!edesc) { -- dev_err(jrdev, "could not allocate extended descriptor\n"); -- return ERR_PTR(-ENOMEM); -+ dst_nents = sg_nents_for_len(req->dst, req->assoclen + -+ req->cryptlen + -+ (encrypt ? authsize : -+ (-authsize))); -+ if (unlikely(dst_nents < 0)) { -+ dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", -+ req->assoclen + req->cryptlen + -+ (encrypt ? authsize : (-authsize))); -+ return ERR_PTR(dst_nents); -+ } -+ } else { -+ src_nents = sg_nents_for_len(req->src, req->assoclen + -+ req->cryptlen + -+ (encrypt ? authsize : 0)); -+ if (unlikely(src_nents < 0)) { -+ dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", -+ req->assoclen + req->cryptlen + -+ (encrypt ? authsize : 0)); -+ return ERR_PTR(src_nents); -+ } - } - - if (likely(req->src == req->dst)) { -- sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, -- DMA_BIDIRECTIONAL); -- if (unlikely(!sgc)) { -+ mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, -+ DMA_BIDIRECTIONAL); -+ if (unlikely(!mapped_src_nents)) { - dev_err(jrdev, "unable to map source\n"); -- kfree(edesc); - return ERR_PTR(-ENOMEM); - } - } else { -- sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, -- DMA_TO_DEVICE); -- if (unlikely(!sgc)) { -- dev_err(jrdev, "unable to map source\n"); -- kfree(edesc); -- return ERR_PTR(-ENOMEM); -+ /* Cover also the case of null (zero length) input data */ -+ if (src_nents) { -+ mapped_src_nents = dma_map_sg(jrdev, req->src, -+ src_nents, DMA_TO_DEVICE); -+ if (unlikely(!mapped_src_nents)) { -+ dev_err(jrdev, "unable to map source\n"); -+ return ERR_PTR(-ENOMEM); -+ } -+ } else { -+ mapped_src_nents = 0; - } - -- sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, -- DMA_FROM_DEVICE); -- if (unlikely(!sgc)) { -+ mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents, -+ DMA_FROM_DEVICE); -+ if (unlikely(!mapped_dst_nents)) { - dev_err(jrdev, "unable to map destination\n"); -- dma_unmap_sg(jrdev, req->src, src_nents ? : 1, -- DMA_TO_DEVICE); -- kfree(edesc); -+ dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); - return ERR_PTR(-ENOMEM); - } - } - -+ sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0; -+ sec4_sg_len += mapped_dst_nents > 1 ? mapped_dst_nents : 0; -+ sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); -+ -+ /* allocate space for base edesc and hw desc commands, link tables */ -+ edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, -+ GFP_DMA | flags); -+ if (!edesc) { -+ caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0, -+ 0, 0, 0); -+ return ERR_PTR(-ENOMEM); -+ } -+ - edesc->src_nents = src_nents; - edesc->dst_nents = dst_nents; - edesc->sec4_sg = (void *)edesc + sizeof(struct aead_edesc) + - desc_bytes; -- *all_contig_ptr = all_contig; -+ *all_contig_ptr = !(mapped_src_nents > 1); - - sec4_sg_index = 0; -- if (!all_contig) { -- sg_to_sec4_sg_last(req->src, src_nents, -- edesc->sec4_sg + sec4_sg_index, 0); -- sec4_sg_index += src_nents; -+ if (mapped_src_nents > 1) { -+ sg_to_sec4_sg_last(req->src, mapped_src_nents, -+ edesc->sec4_sg + sec4_sg_index, 0); -+ sec4_sg_index += mapped_src_nents; - } -- if (dst_nents) { -- sg_to_sec4_sg_last(req->dst, dst_nents, -+ if (mapped_dst_nents > 1) { -+ sg_to_sec4_sg_last(req->dst, mapped_dst_nents, - edesc->sec4_sg + sec4_sg_index, 0); - } - -@@ -2573,13 +1459,9 @@ static int aead_decrypt(struct aead_requ - u32 *desc; - int ret = 0; - --#ifdef DEBUG -- bool may_sleep = ((req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | -- CRYPTO_TFM_REQ_MAY_SLEEP)) != 0); -- dbg_dump_sg(KERN_ERR, "dec src@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, req->src, -- req->assoclen + req->cryptlen, 1, may_sleep); --#endif -+ caam_dump_sg(KERN_ERR, "dec src@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, req->src, -+ req->assoclen + req->cryptlen, 1); - - /* allocate extended descriptor */ - edesc = aead_edesc_alloc(req, AUTHENC_DESC_JOB_IO_LEN, -@@ -2619,51 +1501,80 @@ static struct ablkcipher_edesc *ablkciph - struct device *jrdev = ctx->jrdev; - gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? - GFP_KERNEL : GFP_ATOMIC; -- int src_nents, dst_nents = 0, sec4_sg_bytes; -+ int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; - struct ablkcipher_edesc *edesc; - dma_addr_t iv_dma = 0; -- bool iv_contig = false; -- int sgc; -+ bool in_contig; - int ivsize = crypto_ablkcipher_ivsize(ablkcipher); -- int sec4_sg_index; -+ int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes; - -- src_nents = sg_count(req->src, req->nbytes); -+ src_nents = sg_nents_for_len(req->src, req->nbytes); -+ if (unlikely(src_nents < 0)) { -+ dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", -+ req->nbytes); -+ return ERR_PTR(src_nents); -+ } - -- if (req->dst != req->src) -- dst_nents = sg_count(req->dst, req->nbytes); -+ if (req->dst != req->src) { -+ dst_nents = sg_nents_for_len(req->dst, req->nbytes); -+ if (unlikely(dst_nents < 0)) { -+ dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", -+ req->nbytes); -+ return ERR_PTR(dst_nents); -+ } -+ } - - if (likely(req->src == req->dst)) { -- sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, -- DMA_BIDIRECTIONAL); -+ mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, -+ DMA_BIDIRECTIONAL); -+ if (unlikely(!mapped_src_nents)) { -+ dev_err(jrdev, "unable to map source\n"); -+ return ERR_PTR(-ENOMEM); -+ } - } else { -- sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, -- DMA_TO_DEVICE); -- sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, -- DMA_FROM_DEVICE); -+ mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, -+ DMA_TO_DEVICE); -+ if (unlikely(!mapped_src_nents)) { -+ dev_err(jrdev, "unable to map source\n"); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents, -+ DMA_FROM_DEVICE); -+ if (unlikely(!mapped_dst_nents)) { -+ dev_err(jrdev, "unable to map destination\n"); -+ dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); -+ return ERR_PTR(-ENOMEM); -+ } - } - - iv_dma = dma_map_single(jrdev, req->info, ivsize, DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, iv_dma)) { - dev_err(jrdev, "unable to map IV\n"); -+ caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0, -+ 0, 0, 0); - return ERR_PTR(-ENOMEM); - } - -- /* -- * Check if iv can be contiguous with source and destination. -- * If so, include it. If not, create scatterlist. -- */ -- if (!src_nents && iv_dma + ivsize == sg_dma_address(req->src)) -- iv_contig = true; -- else -- src_nents = src_nents ? : 1; -- sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) * -- sizeof(struct sec4_sg_entry); -+ if (mapped_src_nents == 1 && -+ iv_dma + ivsize == sg_dma_address(req->src)) { -+ in_contig = true; -+ sec4_sg_ents = 0; -+ } else { -+ in_contig = false; -+ sec4_sg_ents = 1 + mapped_src_nents; -+ } -+ dst_sg_idx = sec4_sg_ents; -+ sec4_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; -+ sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry); - - /* allocate space for base edesc and hw desc commands, link tables */ - edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, - GFP_DMA | flags); - if (!edesc) { - dev_err(jrdev, "could not allocate extended descriptor\n"); -+ caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, -+ iv_dma, ivsize, 0, 0); - return ERR_PTR(-ENOMEM); - } - -@@ -2673,23 +1584,24 @@ static struct ablkcipher_edesc *ablkciph - edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + - desc_bytes; - -- sec4_sg_index = 0; -- if (!iv_contig) { -+ if (!in_contig) { - dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0); -- sg_to_sec4_sg_last(req->src, src_nents, -+ sg_to_sec4_sg_last(req->src, mapped_src_nents, - edesc->sec4_sg + 1, 0); -- sec4_sg_index += 1 + src_nents; - } - -- if (dst_nents) { -- sg_to_sec4_sg_last(req->dst, dst_nents, -- edesc->sec4_sg + sec4_sg_index, 0); -+ if (mapped_dst_nents > 1) { -+ sg_to_sec4_sg_last(req->dst, mapped_dst_nents, -+ edesc->sec4_sg + dst_sg_idx, 0); - } - - edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, - sec4_sg_bytes, DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { - dev_err(jrdev, "unable to map S/G table\n"); -+ caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, -+ iv_dma, ivsize, 0, 0); -+ kfree(edesc); - return ERR_PTR(-ENOMEM); - } - -@@ -2701,7 +1613,7 @@ static struct ablkcipher_edesc *ablkciph - sec4_sg_bytes, 1); - #endif - -- *iv_contig_out = iv_contig; -+ *iv_contig_out = in_contig; - return edesc; - } - -@@ -2792,30 +1704,54 @@ static struct ablkcipher_edesc *ablkciph - struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); - struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); - struct device *jrdev = ctx->jrdev; -- gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | -- CRYPTO_TFM_REQ_MAY_SLEEP)) ? -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? - GFP_KERNEL : GFP_ATOMIC; -- int src_nents, dst_nents = 0, sec4_sg_bytes; -+ int src_nents, mapped_src_nents, dst_nents, mapped_dst_nents; - struct ablkcipher_edesc *edesc; - dma_addr_t iv_dma = 0; -- bool iv_contig = false; -- int sgc; -+ bool out_contig; - int ivsize = crypto_ablkcipher_ivsize(ablkcipher); -- int sec4_sg_index; -+ int dst_sg_idx, sec4_sg_ents, sec4_sg_bytes; - -- src_nents = sg_count(req->src, req->nbytes); -- -- if (unlikely(req->dst != req->src)) -- dst_nents = sg_count(req->dst, req->nbytes); -+ src_nents = sg_nents_for_len(req->src, req->nbytes); -+ if (unlikely(src_nents < 0)) { -+ dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", -+ req->nbytes); -+ return ERR_PTR(src_nents); -+ } - - if (likely(req->src == req->dst)) { -- sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, -- DMA_BIDIRECTIONAL); -+ mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, -+ DMA_BIDIRECTIONAL); -+ if (unlikely(!mapped_src_nents)) { -+ dev_err(jrdev, "unable to map source\n"); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ dst_nents = src_nents; -+ mapped_dst_nents = src_nents; - } else { -- sgc = dma_map_sg(jrdev, req->src, src_nents ? : 1, -- DMA_TO_DEVICE); -- sgc = dma_map_sg(jrdev, req->dst, dst_nents ? : 1, -- DMA_FROM_DEVICE); -+ mapped_src_nents = dma_map_sg(jrdev, req->src, src_nents, -+ DMA_TO_DEVICE); -+ if (unlikely(!mapped_src_nents)) { -+ dev_err(jrdev, "unable to map source\n"); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ dst_nents = sg_nents_for_len(req->dst, req->nbytes); -+ if (unlikely(dst_nents < 0)) { -+ dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", -+ req->nbytes); -+ return ERR_PTR(dst_nents); -+ } -+ -+ mapped_dst_nents = dma_map_sg(jrdev, req->dst, dst_nents, -+ DMA_FROM_DEVICE); -+ if (unlikely(!mapped_dst_nents)) { -+ dev_err(jrdev, "unable to map destination\n"); -+ dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); -+ return ERR_PTR(-ENOMEM); -+ } - } - - /* -@@ -2825,21 +1761,29 @@ static struct ablkcipher_edesc *ablkciph - iv_dma = dma_map_single(jrdev, greq->giv, ivsize, DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, iv_dma)) { - dev_err(jrdev, "unable to map IV\n"); -+ caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, 0, -+ 0, 0, 0); - return ERR_PTR(-ENOMEM); - } - -- if (!dst_nents && iv_dma + ivsize == sg_dma_address(req->dst)) -- iv_contig = true; -- else -- dst_nents = dst_nents ? : 1; -- sec4_sg_bytes = ((iv_contig ? 0 : 1) + src_nents + dst_nents) * -- sizeof(struct sec4_sg_entry); -+ sec4_sg_ents = mapped_src_nents > 1 ? mapped_src_nents : 0; -+ dst_sg_idx = sec4_sg_ents; -+ if (mapped_dst_nents == 1 && -+ iv_dma + ivsize == sg_dma_address(req->dst)) { -+ out_contig = true; -+ } else { -+ out_contig = false; -+ sec4_sg_ents += 1 + mapped_dst_nents; -+ } - - /* allocate space for base edesc and hw desc commands, link tables */ -+ sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry); - edesc = kzalloc(sizeof(*edesc) + desc_bytes + sec4_sg_bytes, - GFP_DMA | flags); - if (!edesc) { - dev_err(jrdev, "could not allocate extended descriptor\n"); -+ caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, -+ iv_dma, ivsize, 0, 0); - return ERR_PTR(-ENOMEM); - } - -@@ -2849,24 +1793,24 @@ static struct ablkcipher_edesc *ablkciph - edesc->sec4_sg = (void *)edesc + sizeof(struct ablkcipher_edesc) + - desc_bytes; - -- sec4_sg_index = 0; -- if (src_nents) { -- sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0); -- sec4_sg_index += src_nents; -- } -+ if (mapped_src_nents > 1) -+ sg_to_sec4_sg_last(req->src, mapped_src_nents, edesc->sec4_sg, -+ 0); - -- if (!iv_contig) { -- dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, -+ if (!out_contig) { -+ dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx, - iv_dma, ivsize, 0); -- sec4_sg_index += 1; -- sg_to_sec4_sg_last(req->dst, dst_nents, -- edesc->sec4_sg + sec4_sg_index, 0); -+ sg_to_sec4_sg_last(req->dst, mapped_dst_nents, -+ edesc->sec4_sg + dst_sg_idx + 1, 0); - } - - edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, - sec4_sg_bytes, DMA_TO_DEVICE); - if (dma_mapping_error(jrdev, edesc->sec4_sg_dma)) { - dev_err(jrdev, "unable to map S/G table\n"); -+ caam_unmap(jrdev, req->src, req->dst, src_nents, dst_nents, -+ iv_dma, ivsize, 0, 0); -+ kfree(edesc); - return ERR_PTR(-ENOMEM); - } - edesc->iv_dma = iv_dma; -@@ -2878,7 +1822,7 @@ static struct ablkcipher_edesc *ablkciph - sec4_sg_bytes, 1); - #endif - -- *iv_contig_out = iv_contig; -+ *iv_contig_out = out_contig; - return edesc; - } - -@@ -2889,7 +1833,7 @@ static int ablkcipher_givencrypt(struct - struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); - struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); - struct device *jrdev = ctx->jrdev; -- bool iv_contig; -+ bool iv_contig = false; - u32 *desc; - int ret = 0; - -@@ -2933,7 +1877,6 @@ struct caam_alg_template { - } template_u; - u32 class1_alg_type; - u32 class2_alg_type; -- u32 alg_op; - }; - - static struct caam_alg_template driver_algs[] = { -@@ -3118,7 +2061,6 @@ static struct caam_aead_alg driver_aeads - .caam = { - .class2_alg_type = OP_ALG_ALGSEL_MD5 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -3140,7 +2082,6 @@ static struct caam_aead_alg driver_aeads - .caam = { - .class2_alg_type = OP_ALG_ALGSEL_SHA1 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -3162,7 +2103,6 @@ static struct caam_aead_alg driver_aeads - .caam = { - .class2_alg_type = OP_ALG_ALGSEL_SHA224 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -3184,7 +2124,6 @@ static struct caam_aead_alg driver_aeads - .caam = { - .class2_alg_type = OP_ALG_ALGSEL_SHA256 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -3206,7 +2145,6 @@ static struct caam_aead_alg driver_aeads - .caam = { - .class2_alg_type = OP_ALG_ALGSEL_SHA384 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -3228,7 +2166,6 @@ static struct caam_aead_alg driver_aeads - .caam = { - .class2_alg_type = OP_ALG_ALGSEL_SHA512 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -3250,7 +2187,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_MD5 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -3273,7 +2209,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_MD5 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, - .geniv = true, - }, - }, -@@ -3296,7 +2231,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA1 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -3319,7 +2253,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA1 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, - .geniv = true, - }, - }, -@@ -3342,7 +2275,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA224 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -3365,7 +2297,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA224 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, - .geniv = true, - }, - }, -@@ -3388,7 +2319,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA256 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -3411,7 +2341,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA256 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, - .geniv = true, - }, - }, -@@ -3434,7 +2363,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA384 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -3457,7 +2385,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA384 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, - .geniv = true, - }, - }, -@@ -3480,7 +2407,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA512 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -3503,7 +2429,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA512 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, - .geniv = true, - }, - }, -@@ -3526,7 +2451,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_MD5 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, - } - }, - { -@@ -3549,7 +2473,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_MD5 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, - .geniv = true, - } - }, -@@ -3573,7 +2496,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA1 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -3597,7 +2519,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA1 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, - .geniv = true, - }, - }, -@@ -3621,7 +2542,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA224 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -3645,7 +2565,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA224 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, - .geniv = true, - }, - }, -@@ -3669,7 +2588,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA256 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -3693,7 +2611,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA256 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, - .geniv = true, - }, - }, -@@ -3717,7 +2634,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA384 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -3741,7 +2657,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA384 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, - .geniv = true, - }, - }, -@@ -3765,7 +2680,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA512 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -3789,7 +2703,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA512 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, - .geniv = true, - }, - }, -@@ -3812,7 +2725,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_MD5 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -3835,7 +2747,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_MD5 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, - .geniv = true, - }, - }, -@@ -3858,7 +2769,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA1 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -3881,7 +2791,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA1 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, - .geniv = true, - }, - }, -@@ -3904,7 +2813,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA224 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -3927,7 +2835,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA224 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, - .geniv = true, - }, - }, -@@ -3950,7 +2857,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA256 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -3973,7 +2879,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA256 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, - .geniv = true, - }, - }, -@@ -3996,7 +2901,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA384 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -4019,7 +2923,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA384 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, - .geniv = true, - }, - }, -@@ -4042,7 +2945,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA512 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, - }, - }, - { -@@ -4065,7 +2967,6 @@ static struct caam_aead_alg driver_aeads - .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, - .class2_alg_type = OP_ALG_ALGSEL_SHA512 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, - .geniv = true, - }, - }, -@@ -4090,7 +2991,6 @@ static struct caam_aead_alg driver_aeads - OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_MD5 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, - .rfc3686 = true, - }, - }, -@@ -4115,7 +3015,6 @@ static struct caam_aead_alg driver_aeads - OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_MD5 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, - .rfc3686 = true, - .geniv = true, - }, -@@ -4141,7 +3040,6 @@ static struct caam_aead_alg driver_aeads - OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_SHA1 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, - .rfc3686 = true, - }, - }, -@@ -4166,7 +3064,6 @@ static struct caam_aead_alg driver_aeads - OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_SHA1 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, - .rfc3686 = true, - .geniv = true, - }, -@@ -4192,7 +3089,6 @@ static struct caam_aead_alg driver_aeads - OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_SHA224 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, - .rfc3686 = true, - }, - }, -@@ -4217,7 +3113,6 @@ static struct caam_aead_alg driver_aeads - OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_SHA224 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, - .rfc3686 = true, - .geniv = true, - }, -@@ -4243,7 +3138,6 @@ static struct caam_aead_alg driver_aeads - OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_SHA256 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, - .rfc3686 = true, - }, - }, -@@ -4268,7 +3162,6 @@ static struct caam_aead_alg driver_aeads - OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_SHA256 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, - .rfc3686 = true, - .geniv = true, - }, -@@ -4294,7 +3187,6 @@ static struct caam_aead_alg driver_aeads - OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_SHA384 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, - .rfc3686 = true, - }, - }, -@@ -4319,7 +3211,6 @@ static struct caam_aead_alg driver_aeads - OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_SHA384 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, - .rfc3686 = true, - .geniv = true, - }, -@@ -4345,7 +3236,6 @@ static struct caam_aead_alg driver_aeads - OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_SHA512 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, - .rfc3686 = true, - }, - }, -@@ -4370,7 +3260,6 @@ static struct caam_aead_alg driver_aeads - OP_ALG_AAI_CTR_MOD128, - .class2_alg_type = OP_ALG_ALGSEL_SHA512 | - OP_ALG_AAI_HMAC_PRECOMP, -- .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, - .rfc3686 = true, - .geniv = true, - }, -@@ -4383,18 +3272,44 @@ struct caam_crypto_alg { - struct caam_alg_entry caam; - }; - --static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam) -+static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, -+ bool uses_dkp) - { -+ dma_addr_t dma_addr; -+ struct caam_drv_private *priv; -+ - ctx->jrdev = caam_jr_alloc(); - if (IS_ERR(ctx->jrdev)) { - pr_err("Job Ring Device allocation for transform failed\n"); - return PTR_ERR(ctx->jrdev); - } - -+ priv = dev_get_drvdata(ctx->jrdev->parent); -+ if (priv->era >= 6 && uses_dkp) -+ ctx->dir = DMA_BIDIRECTIONAL; -+ else -+ ctx->dir = DMA_TO_DEVICE; -+ -+ dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_enc, -+ offsetof(struct caam_ctx, -+ sh_desc_enc_dma), -+ ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); -+ if (dma_mapping_error(ctx->jrdev, dma_addr)) { -+ dev_err(ctx->jrdev, "unable to map key, shared descriptors\n"); -+ caam_jr_free(ctx->jrdev); -+ return -ENOMEM; -+ } -+ -+ ctx->sh_desc_enc_dma = dma_addr; -+ ctx->sh_desc_dec_dma = dma_addr + offsetof(struct caam_ctx, -+ sh_desc_dec); -+ ctx->sh_desc_givenc_dma = dma_addr + offsetof(struct caam_ctx, -+ sh_desc_givenc); -+ ctx->key_dma = dma_addr + offsetof(struct caam_ctx, key); -+ - /* copy descriptor header template value */ -- ctx->class1_alg_type = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; -- ctx->class2_alg_type = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; -- ctx->alg_op = OP_TYPE_CLASS2_ALG | caam->alg_op; -+ ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; -+ ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; - - return 0; - } -@@ -4406,7 +3321,7 @@ static int caam_cra_init(struct crypto_t - container_of(alg, struct caam_crypto_alg, crypto_alg); - struct caam_ctx *ctx = crypto_tfm_ctx(tfm); - -- return caam_init_common(ctx, &caam_alg->caam); -+ return caam_init_common(ctx, &caam_alg->caam, false); - } - - static int caam_aead_init(struct crypto_aead *tfm) -@@ -4416,30 +3331,15 @@ static int caam_aead_init(struct crypto_ - container_of(alg, struct caam_aead_alg, aead); - struct caam_ctx *ctx = crypto_aead_ctx(tfm); - -- return caam_init_common(ctx, &caam_alg->caam); -+ return caam_init_common(ctx, &caam_alg->caam, -+ alg->setkey == aead_setkey); - } - - static void caam_exit_common(struct caam_ctx *ctx) - { -- if (ctx->sh_desc_enc_dma && -- !dma_mapping_error(ctx->jrdev, ctx->sh_desc_enc_dma)) -- dma_unmap_single(ctx->jrdev, ctx->sh_desc_enc_dma, -- desc_bytes(ctx->sh_desc_enc), DMA_TO_DEVICE); -- if (ctx->sh_desc_dec_dma && -- !dma_mapping_error(ctx->jrdev, ctx->sh_desc_dec_dma)) -- dma_unmap_single(ctx->jrdev, ctx->sh_desc_dec_dma, -- desc_bytes(ctx->sh_desc_dec), DMA_TO_DEVICE); -- if (ctx->sh_desc_givenc_dma && -- !dma_mapping_error(ctx->jrdev, ctx->sh_desc_givenc_dma)) -- dma_unmap_single(ctx->jrdev, ctx->sh_desc_givenc_dma, -- desc_bytes(ctx->sh_desc_givenc), -- DMA_TO_DEVICE); -- if (ctx->key_dma && -- !dma_mapping_error(ctx->jrdev, ctx->key_dma)) -- dma_unmap_single(ctx->jrdev, ctx->key_dma, -- ctx->enckeylen + ctx->split_key_pad_len, -- DMA_TO_DEVICE); -- -+ dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_enc_dma, -+ offsetof(struct caam_ctx, sh_desc_enc_dma), -+ ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); - caam_jr_free(ctx->jrdev); - } - -@@ -4515,7 +3415,6 @@ static struct caam_crypto_alg *caam_alg_ - - t_alg->caam.class1_alg_type = template->class1_alg_type; - t_alg->caam.class2_alg_type = template->class2_alg_type; -- t_alg->caam.alg_op = template->alg_op; - - return t_alg; - } ---- /dev/null -+++ b/drivers/crypto/caam/caamalg_desc.c -@@ -0,0 +1,1961 @@ -+/* -+ * Shared descriptors for aead, ablkcipher algorithms -+ * -+ * Copyright 2016 NXP -+ */ -+ -+#include "compat.h" -+#include "desc_constr.h" -+#include "caamalg_desc.h" -+ -+/* -+ * For aead functions, read payload and write payload, -+ * both of which are specified in req->src and req->dst -+ */ -+static inline void aead_append_src_dst(u32 *desc, u32 msg_type) -+{ -+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | -+ KEY_VLF | msg_type | FIFOLD_TYPE_LASTBOTH); -+} -+ -+/* Set DK bit in class 1 operation if shared */ -+static inline void append_dec_op1(u32 *desc, u32 type) -+{ -+ u32 *jump_cmd, *uncond_jump_cmd; -+ -+ /* DK bit is valid only for AES */ -+ if ((type & OP_ALG_ALGSEL_MASK) != OP_ALG_ALGSEL_AES) { -+ append_operation(desc, type | OP_ALG_AS_INITFINAL | -+ OP_ALG_DECRYPT); -+ return; -+ } -+ -+ jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); -+ append_operation(desc, type | OP_ALG_AS_INITFINAL | -+ OP_ALG_DECRYPT); -+ uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL); -+ set_jump_tgt_here(desc, jump_cmd); -+ append_operation(desc, type | OP_ALG_AS_INITFINAL | -+ OP_ALG_DECRYPT | OP_ALG_AAI_DK); -+ set_jump_tgt_here(desc, uncond_jump_cmd); -+} -+ -+/** -+ * cnstr_shdsc_aead_null_encap - IPSec ESP encapsulation shared descriptor -+ * (non-protocol) with no (null) encryption. -+ * @desc: pointer to buffer used for descriptor construction -+ * @adata: pointer to authentication transform definitions. -+ * A split key is required for SEC Era < 6; the size of the split key -+ * is specified in this case. Valid algorithm values - one of -+ * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed -+ * with OP_ALG_AAI_HMAC_PRECOMP. -+ * @icvsize: integrity check value (ICV) size (truncated or full) -+ * @era: SEC Era -+ */ -+void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, -+ unsigned int icvsize, int era) -+{ -+ u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; -+ -+ init_sh_desc(desc, HDR_SHARE_SERIAL); -+ -+ /* Skip if already shared */ -+ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_SHRD); -+ if (era < 6) { -+ if (adata->key_inline) -+ append_key_as_imm(desc, adata->key_virt, -+ adata->keylen_pad, adata->keylen, -+ CLASS_2 | KEY_DEST_MDHA_SPLIT | -+ KEY_ENC); -+ else -+ append_key(desc, adata->key_dma, adata->keylen, -+ CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); -+ } else { -+ append_proto_dkp(desc, adata); -+ } -+ set_jump_tgt_here(desc, key_jump_cmd); -+ -+ /* assoclen + cryptlen = seqinlen */ -+ append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); -+ -+ /* Prepare to read and write cryptlen + assoclen bytes */ -+ append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); -+ append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); -+ -+ /* -+ * MOVE_LEN opcode is not available in all SEC HW revisions, -+ * thus need to do some magic, i.e. self-patch the descriptor -+ * buffer. -+ */ -+ read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | -+ MOVE_DEST_MATH3 | -+ (0x6 << MOVE_LEN_SHIFT)); -+ write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | -+ MOVE_DEST_DESCBUF | -+ MOVE_WAITCOMP | -+ (0x8 << MOVE_LEN_SHIFT)); -+ -+ /* Class 2 operation */ -+ append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | -+ OP_ALG_ENCRYPT); -+ -+ /* Read and write cryptlen bytes */ -+ aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); -+ -+ set_move_tgt_here(desc, read_move_cmd); -+ set_move_tgt_here(desc, write_move_cmd); -+ append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); -+ append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | -+ MOVE_AUX_LS); -+ -+ /* Write ICV */ -+ append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | -+ LDST_SRCDST_BYTE_CONTEXT); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, -+ "aead null enc shdesc@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+} -+EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap); -+ -+/** -+ * cnstr_shdsc_aead_null_decap - IPSec ESP decapsulation shared descriptor -+ * (non-protocol) with no (null) decryption. -+ * @desc: pointer to buffer used for descriptor construction -+ * @adata: pointer to authentication transform definitions. -+ * A split key is required for SEC Era < 6; the size of the split key -+ * is specified in this case. Valid algorithm values - one of -+ * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed -+ * with OP_ALG_AAI_HMAC_PRECOMP. -+ * @icvsize: integrity check value (ICV) size (truncated or full) -+ * @era: SEC Era -+ */ -+void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata, -+ unsigned int icvsize, int era) -+{ -+ u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd, *jump_cmd; -+ -+ init_sh_desc(desc, HDR_SHARE_SERIAL); -+ -+ /* Skip if already shared */ -+ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_SHRD); -+ if (era < 6) { -+ if (adata->key_inline) -+ append_key_as_imm(desc, adata->key_virt, -+ adata->keylen_pad, adata->keylen, -+ CLASS_2 | KEY_DEST_MDHA_SPLIT | -+ KEY_ENC); -+ else -+ append_key(desc, adata->key_dma, adata->keylen, -+ CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); -+ } else { -+ append_proto_dkp(desc, adata); -+ } -+ set_jump_tgt_here(desc, key_jump_cmd); -+ -+ /* Class 2 operation */ -+ append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | -+ OP_ALG_DECRYPT | OP_ALG_ICV_ON); -+ -+ /* assoclen + cryptlen = seqoutlen */ -+ append_math_sub(desc, REG2, SEQOUTLEN, REG0, CAAM_CMD_SZ); -+ -+ /* Prepare to read and write cryptlen + assoclen bytes */ -+ append_math_add(desc, VARSEQINLEN, ZERO, REG2, CAAM_CMD_SZ); -+ append_math_add(desc, VARSEQOUTLEN, ZERO, REG2, CAAM_CMD_SZ); -+ -+ /* -+ * MOVE_LEN opcode is not available in all SEC HW revisions, -+ * thus need to do some magic, i.e. self-patch the descriptor -+ * buffer. -+ */ -+ read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | -+ MOVE_DEST_MATH2 | -+ (0x6 << MOVE_LEN_SHIFT)); -+ write_move_cmd = append_move(desc, MOVE_SRC_MATH2 | -+ MOVE_DEST_DESCBUF | -+ MOVE_WAITCOMP | -+ (0x8 << MOVE_LEN_SHIFT)); -+ -+ /* Read and write cryptlen bytes */ -+ aead_append_src_dst(desc, FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); -+ -+ /* -+ * Insert a NOP here, since we need at least 4 instructions between -+ * code patching the descriptor buffer and the location being patched. -+ */ -+ jump_cmd = append_jump(desc, JUMP_TEST_ALL); -+ set_jump_tgt_here(desc, jump_cmd); -+ -+ set_move_tgt_here(desc, read_move_cmd); -+ set_move_tgt_here(desc, write_move_cmd); -+ append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); -+ append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO | -+ MOVE_AUX_LS); -+ append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); -+ -+ /* Load ICV */ -+ append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | -+ FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, -+ "aead null dec shdesc@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+} -+EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap); -+ -+static void init_sh_desc_key_aead(u32 * const desc, -+ struct alginfo * const cdata, -+ struct alginfo * const adata, -+ const bool is_rfc3686, u32 *nonce, int era) -+{ -+ u32 *key_jump_cmd; -+ unsigned int enckeylen = cdata->keylen; -+ -+ /* Note: Context registers are saved. */ -+ init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); -+ -+ /* Skip if already shared */ -+ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_SHRD); -+ -+ /* -+ * RFC3686 specific: -+ * | key = {AUTH_KEY, ENC_KEY, NONCE} -+ * | enckeylen = encryption key size + nonce size -+ */ -+ if (is_rfc3686) -+ enckeylen -= CTR_RFC3686_NONCE_SIZE; -+ -+ if (era < 6) { -+ if (adata->key_inline) -+ append_key_as_imm(desc, adata->key_virt, -+ adata->keylen_pad, adata->keylen, -+ CLASS_2 | KEY_DEST_MDHA_SPLIT | -+ KEY_ENC); -+ else -+ append_key(desc, adata->key_dma, adata->keylen, -+ CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); -+ } else { -+ append_proto_dkp(desc, adata); -+ } -+ -+ if (cdata->key_inline) -+ append_key_as_imm(desc, cdata->key_virt, enckeylen, -+ enckeylen, CLASS_1 | KEY_DEST_CLASS_REG); -+ else -+ append_key(desc, cdata->key_dma, enckeylen, CLASS_1 | -+ KEY_DEST_CLASS_REG); -+ -+ /* Load Counter into CONTEXT1 reg */ -+ if (is_rfc3686) { -+ append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, -+ LDST_CLASS_IND_CCB | -+ LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); -+ append_move(desc, -+ MOVE_SRC_OUTFIFO | -+ MOVE_DEST_CLASS1CTX | -+ (16 << MOVE_OFFSET_SHIFT) | -+ (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); -+ } -+ -+ set_jump_tgt_here(desc, key_jump_cmd); -+} -+ -+/** -+ * cnstr_shdsc_aead_encap - IPSec ESP encapsulation shared descriptor -+ * (non-protocol). -+ * @desc: pointer to buffer used for descriptor construction -+ * @cdata: pointer to block cipher transform definitions -+ * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed -+ * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. -+ * @adata: pointer to authentication transform definitions. -+ * A split key is required for SEC Era < 6; the size of the split key -+ * is specified in this case. Valid algorithm values - one of -+ * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed -+ * with OP_ALG_AAI_HMAC_PRECOMP. -+ * @ivsize: initialization vector size -+ * @icvsize: integrity check value (ICV) size (truncated or full) -+ * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template -+ * @nonce: pointer to rfc3686 nonce -+ * @ctx1_iv_off: IV offset in CONTEXT1 register -+ * @is_qi: true when called from caam/qi -+ * @era: SEC Era -+ */ -+void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata, -+ struct alginfo *adata, unsigned int ivsize, -+ unsigned int icvsize, const bool is_rfc3686, -+ u32 *nonce, const u32 ctx1_iv_off, const bool is_qi, -+ int era) -+{ -+ /* Note: Context registers are saved. */ -+ init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era); -+ -+ /* Class 2 operation */ -+ append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | -+ OP_ALG_ENCRYPT); -+ -+ if (is_qi) { -+ u32 *wait_load_cmd; -+ -+ /* REG3 = assoclen */ -+ append_seq_load(desc, 4, LDST_CLASS_DECO | -+ LDST_SRCDST_WORD_DECO_MATH3 | -+ (4 << LDST_OFFSET_SHIFT)); -+ -+ wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_CALM | JUMP_COND_NCP | -+ JUMP_COND_NOP | JUMP_COND_NIP | -+ JUMP_COND_NIFP); -+ set_jump_tgt_here(desc, wait_load_cmd); -+ -+ append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | -+ LDST_SRCDST_BYTE_CONTEXT | -+ (ctx1_iv_off << LDST_OFFSET_SHIFT)); -+ } -+ -+ /* Read and write assoclen bytes */ -+ if (is_qi || era < 3) { -+ append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); -+ append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); -+ } else { -+ append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ); -+ append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ); -+ } -+ -+ /* Skip assoc data */ -+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); -+ -+ /* read assoc before reading payload */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | -+ FIFOLDST_VLF); -+ -+ /* Load Counter into CONTEXT1 reg */ -+ if (is_rfc3686) -+ append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | -+ LDST_SRCDST_BYTE_CONTEXT | -+ ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << -+ LDST_OFFSET_SHIFT)); -+ -+ /* Class 1 operation */ -+ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | -+ OP_ALG_ENCRYPT); -+ -+ /* Read and write cryptlen bytes */ -+ append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -+ append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -+ aead_append_src_dst(desc, FIFOLD_TYPE_MSG1OUT2); -+ -+ /* Write ICV */ -+ append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | -+ LDST_SRCDST_BYTE_CONTEXT); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "aead enc shdesc@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+} -+EXPORT_SYMBOL(cnstr_shdsc_aead_encap); -+ -+/** -+ * cnstr_shdsc_aead_decap - IPSec ESP decapsulation shared descriptor -+ * (non-protocol). -+ * @desc: pointer to buffer used for descriptor construction -+ * @cdata: pointer to block cipher transform definitions -+ * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed -+ * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. -+ * @adata: pointer to authentication transform definitions. -+ * A split key is required for SEC Era < 6; the size of the split key -+ * is specified in this case. Valid algorithm values - one of -+ * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed -+ * with OP_ALG_AAI_HMAC_PRECOMP. -+ * @ivsize: initialization vector size -+ * @icvsize: integrity check value (ICV) size (truncated or full) -+ * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template -+ * @nonce: pointer to rfc3686 nonce -+ * @ctx1_iv_off: IV offset in CONTEXT1 register -+ * @is_qi: true when called from caam/qi -+ * @era: SEC Era -+ */ -+void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata, -+ struct alginfo *adata, unsigned int ivsize, -+ unsigned int icvsize, const bool geniv, -+ const bool is_rfc3686, u32 *nonce, -+ const u32 ctx1_iv_off, const bool is_qi, int era) -+{ -+ /* Note: Context registers are saved. */ -+ init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era); -+ -+ /* Class 2 operation */ -+ append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | -+ OP_ALG_DECRYPT | OP_ALG_ICV_ON); -+ -+ if (is_qi) { -+ u32 *wait_load_cmd; -+ -+ /* REG3 = assoclen */ -+ append_seq_load(desc, 4, LDST_CLASS_DECO | -+ LDST_SRCDST_WORD_DECO_MATH3 | -+ (4 << LDST_OFFSET_SHIFT)); -+ -+ wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_CALM | JUMP_COND_NCP | -+ JUMP_COND_NOP | JUMP_COND_NIP | -+ JUMP_COND_NIFP); -+ set_jump_tgt_here(desc, wait_load_cmd); -+ -+ if (!geniv) -+ append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | -+ LDST_SRCDST_BYTE_CONTEXT | -+ (ctx1_iv_off << LDST_OFFSET_SHIFT)); -+ } -+ -+ /* Read and write assoclen bytes */ -+ if (is_qi || era < 3) { -+ append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); -+ if (geniv) -+ append_math_add_imm_u32(desc, VARSEQOUTLEN, REG3, IMM, -+ ivsize); -+ else -+ append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, -+ CAAM_CMD_SZ); -+ } else { -+ append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ); -+ if (geniv) -+ append_math_add_imm_u32(desc, VARSEQOUTLEN, DPOVRD, IMM, -+ ivsize); -+ else -+ append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, -+ CAAM_CMD_SZ); -+ } -+ -+ /* Skip assoc data */ -+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); -+ -+ /* read assoc before reading payload */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | -+ KEY_VLF); -+ -+ if (geniv) { -+ append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | -+ LDST_SRCDST_BYTE_CONTEXT | -+ (ctx1_iv_off << LDST_OFFSET_SHIFT)); -+ append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_CLASS2INFIFO | -+ (ctx1_iv_off << MOVE_OFFSET_SHIFT) | ivsize); -+ } -+ -+ /* Load Counter into CONTEXT1 reg */ -+ if (is_rfc3686) -+ append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | -+ LDST_SRCDST_BYTE_CONTEXT | -+ ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << -+ LDST_OFFSET_SHIFT)); -+ -+ /* Choose operation */ -+ if (ctx1_iv_off) -+ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | -+ OP_ALG_DECRYPT); -+ else -+ append_dec_op1(desc, cdata->algtype); -+ -+ /* Read and write cryptlen bytes */ -+ append_math_add(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); -+ append_math_add(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); -+ aead_append_src_dst(desc, FIFOLD_TYPE_MSG); -+ -+ /* Load ICV */ -+ append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | -+ FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "aead dec shdesc@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+} -+EXPORT_SYMBOL(cnstr_shdsc_aead_decap); -+ -+/** -+ * cnstr_shdsc_aead_givencap - IPSec ESP encapsulation shared descriptor -+ * (non-protocol) with HW-generated initialization -+ * vector. -+ * @desc: pointer to buffer used for descriptor construction -+ * @cdata: pointer to block cipher transform definitions -+ * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed -+ * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. -+ * @adata: pointer to authentication transform definitions. -+ * A split key is required for SEC Era < 6; the size of the split key -+ * is specified in this case. Valid algorithm values - one of -+ * OP_ALG_ALGSEL_{MD5, SHA1, SHA224, SHA256, SHA384, SHA512} ANDed -+ * with OP_ALG_AAI_HMAC_PRECOMP. -+ * @ivsize: initialization vector size -+ * @icvsize: integrity check value (ICV) size (truncated or full) -+ * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template -+ * @nonce: pointer to rfc3686 nonce -+ * @ctx1_iv_off: IV offset in CONTEXT1 register -+ * @is_qi: true when called from caam/qi -+ * @era: SEC Era -+ */ -+void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata, -+ struct alginfo *adata, unsigned int ivsize, -+ unsigned int icvsize, const bool is_rfc3686, -+ u32 *nonce, const u32 ctx1_iv_off, -+ const bool is_qi, int era) -+{ -+ u32 geniv, moveiv; -+ -+ /* Note: Context registers are saved. */ -+ init_sh_desc_key_aead(desc, cdata, adata, is_rfc3686, nonce, era); -+ -+ if (is_qi) { -+ u32 *wait_load_cmd; -+ -+ /* REG3 = assoclen */ -+ append_seq_load(desc, 4, LDST_CLASS_DECO | -+ LDST_SRCDST_WORD_DECO_MATH3 | -+ (4 << LDST_OFFSET_SHIFT)); -+ -+ wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_CALM | JUMP_COND_NCP | -+ JUMP_COND_NOP | JUMP_COND_NIP | -+ JUMP_COND_NIFP); -+ set_jump_tgt_here(desc, wait_load_cmd); -+ } -+ -+ if (is_rfc3686) { -+ if (is_qi) -+ append_seq_load(desc, ivsize, LDST_CLASS_1_CCB | -+ LDST_SRCDST_BYTE_CONTEXT | -+ (ctx1_iv_off << LDST_OFFSET_SHIFT)); -+ -+ goto copy_iv; -+ } -+ -+ /* Generate IV */ -+ geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | -+ NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | -+ NFIFOENTRY_PTYPE_RND | (ivsize << NFIFOENTRY_DLEN_SHIFT); -+ append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | -+ LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); -+ append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); -+ append_move(desc, MOVE_WAITCOMP | -+ MOVE_SRC_INFIFO | MOVE_DEST_CLASS1CTX | -+ (ctx1_iv_off << MOVE_OFFSET_SHIFT) | -+ (ivsize << MOVE_LEN_SHIFT)); -+ append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); -+ -+copy_iv: -+ /* Copy IV to class 1 context */ -+ append_move(desc, MOVE_SRC_CLASS1CTX | MOVE_DEST_OUTFIFO | -+ (ctx1_iv_off << MOVE_OFFSET_SHIFT) | -+ (ivsize << MOVE_LEN_SHIFT)); -+ -+ /* Return to encryption */ -+ append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | -+ OP_ALG_ENCRYPT); -+ -+ /* Read and write assoclen bytes */ -+ if (is_qi || era < 3) { -+ append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); -+ append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); -+ } else { -+ append_math_add(desc, VARSEQINLEN, ZERO, DPOVRD, CAAM_CMD_SZ); -+ append_math_add(desc, VARSEQOUTLEN, ZERO, DPOVRD, CAAM_CMD_SZ); -+ } -+ -+ /* Skip assoc data */ -+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); -+ -+ /* read assoc before reading payload */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG | -+ KEY_VLF); -+ -+ /* Copy iv from outfifo to class 2 fifo */ -+ moveiv = NFIFOENTRY_STYPE_OFIFO | NFIFOENTRY_DEST_CLASS2 | -+ NFIFOENTRY_DTYPE_MSG | (ivsize << NFIFOENTRY_DLEN_SHIFT); -+ append_load_imm_u32(desc, moveiv, LDST_CLASS_IND_CCB | -+ LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); -+ append_load_imm_u32(desc, ivsize, LDST_CLASS_2_CCB | -+ LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); -+ -+ /* Load Counter into CONTEXT1 reg */ -+ if (is_rfc3686) -+ append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | -+ LDST_SRCDST_BYTE_CONTEXT | -+ ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << -+ LDST_OFFSET_SHIFT)); -+ -+ /* Class 1 operation */ -+ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | -+ OP_ALG_ENCRYPT); -+ -+ /* Will write ivsize + cryptlen */ -+ append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -+ -+ /* Not need to reload iv */ -+ append_seq_fifo_load(desc, ivsize, -+ FIFOLD_CLASS_SKIP); -+ -+ /* Will read cryptlen */ -+ append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | KEY_VLF | -+ FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LASTBOTH); -+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); -+ -+ /* Write ICV */ -+ append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | -+ LDST_SRCDST_BYTE_CONTEXT); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, -+ "aead givenc shdesc@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+} -+EXPORT_SYMBOL(cnstr_shdsc_aead_givencap); -+ -+/** -+ * cnstr_shdsc_tls_encap - tls encapsulation shared descriptor -+ * @desc: pointer to buffer used for descriptor construction -+ * @cdata: pointer to block cipher transform definitions -+ * Valid algorithm values - one of OP_ALG_ALGSEL_AES ANDed -+ * with OP_ALG_AAI_CBC -+ * @adata: pointer to authentication transform definitions. -+ * A split key is required for SEC Era < 6; the size of the split key -+ * is specified in this case. Valid algorithm values OP_ALG_ALGSEL_SHA1 -+ * ANDed with OP_ALG_AAI_HMAC_PRECOMP. -+ * @assoclen: associated data length -+ * @ivsize: initialization vector size -+ * @authsize: authentication data size -+ * @blocksize: block cipher size -+ * @era: SEC Era -+ */ -+void cnstr_shdsc_tls_encap(u32 * const desc, struct alginfo *cdata, -+ struct alginfo *adata, unsigned int assoclen, -+ unsigned int ivsize, unsigned int authsize, -+ unsigned int blocksize, int era) -+{ -+ u32 *key_jump_cmd, *zero_payload_jump_cmd; -+ u32 genpad, idx_ld_datasz, idx_ld_pad, stidx; -+ -+ /* -+ * Compute the index (in bytes) for the LOAD with destination of -+ * Class 1 Data Size Register and for the LOAD that generates padding -+ */ -+ if (adata->key_inline) { -+ idx_ld_datasz = DESC_TLS10_ENC_LEN + adata->keylen_pad + -+ cdata->keylen - 4 * CAAM_CMD_SZ; -+ idx_ld_pad = DESC_TLS10_ENC_LEN + adata->keylen_pad + -+ cdata->keylen - 2 * CAAM_CMD_SZ; -+ } else { -+ idx_ld_datasz = DESC_TLS10_ENC_LEN + 2 * CAAM_PTR_SZ - -+ 4 * CAAM_CMD_SZ; -+ idx_ld_pad = DESC_TLS10_ENC_LEN + 2 * CAAM_PTR_SZ - -+ 2 * CAAM_CMD_SZ; -+ } -+ -+ stidx = 1 << HDR_START_IDX_SHIFT; -+ init_sh_desc(desc, HDR_SHARE_SERIAL | stidx); -+ -+ /* skip key loading if they are loaded due to sharing */ -+ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_SHRD); -+ -+ if (era < 6) { -+ if (adata->key_inline) -+ append_key_as_imm(desc, adata->key_virt, -+ adata->keylen_pad, adata->keylen, -+ CLASS_2 | KEY_DEST_MDHA_SPLIT | -+ KEY_ENC); -+ else -+ append_key(desc, adata->key_dma, adata->keylen, -+ CLASS_2 | KEY_DEST_MDHA_SPLIT | KEY_ENC); -+ } else { -+ append_proto_dkp(desc, adata); -+ } -+ -+ if (cdata->key_inline) -+ append_key_as_imm(desc, cdata->key_virt, cdata->keylen, -+ cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); -+ else -+ append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | -+ KEY_DEST_CLASS_REG); -+ -+ set_jump_tgt_here(desc, key_jump_cmd); -+ -+ /* class 2 operation */ -+ append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | -+ OP_ALG_ENCRYPT); -+ /* class 1 operation */ -+ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | -+ OP_ALG_ENCRYPT); -+ -+ /* payloadlen = input data length - (assoclen + ivlen) */ -+ append_math_sub_imm_u32(desc, REG0, SEQINLEN, IMM, assoclen + ivsize); -+ -+ /* math1 = payloadlen + icvlen */ -+ append_math_add_imm_u32(desc, REG1, REG0, IMM, authsize); -+ -+ /* padlen = block_size - math1 % block_size */ -+ append_math_and_imm_u32(desc, REG3, REG1, IMM, blocksize - 1); -+ append_math_sub_imm_u32(desc, REG2, IMM, REG3, blocksize); -+ -+ /* cryptlen = payloadlen + icvlen + padlen */ -+ append_math_add(desc, VARSEQOUTLEN, REG1, REG2, 4); -+ -+ /* -+ * update immediate data with the padding length value -+ * for the LOAD in the class 1 data size register. -+ */ -+ append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH2 | -+ (idx_ld_datasz << MOVE_OFFSET_SHIFT) | 7); -+ append_move(desc, MOVE_WAITCOMP | MOVE_SRC_MATH2 | MOVE_DEST_DESCBUF | -+ (idx_ld_datasz << MOVE_OFFSET_SHIFT) | 8); -+ -+ /* overwrite PL field for the padding iNFO FIFO entry */ -+ append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH2 | -+ (idx_ld_pad << MOVE_OFFSET_SHIFT) | 7); -+ append_move(desc, MOVE_WAITCOMP | MOVE_SRC_MATH2 | MOVE_DEST_DESCBUF | -+ (idx_ld_pad << MOVE_OFFSET_SHIFT) | 8); -+ -+ /* store encrypted payload, icv and padding */ -+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | LDST_VLF); -+ -+ /* if payload length is zero, jump to zero-payload commands */ -+ append_math_add(desc, VARSEQINLEN, ZERO, REG0, 4); -+ zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | -+ JUMP_COND_MATH_Z); -+ -+ /* load iv in context1 */ -+ append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_WORD_CLASS_CTX | -+ LDST_CLASS_1_CCB | ivsize); -+ -+ /* read assoc for authentication */ -+ append_seq_fifo_load(desc, assoclen, FIFOLD_CLASS_CLASS2 | -+ FIFOLD_TYPE_MSG); -+ /* insnoop payload */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLD_TYPE_MSG | -+ FIFOLD_TYPE_LAST2 | FIFOLDST_VLF); -+ -+ /* jump the zero-payload commands */ -+ append_jump(desc, JUMP_TEST_ALL | 3); -+ -+ /* zero-payload commands */ -+ set_jump_tgt_here(desc, zero_payload_jump_cmd); -+ -+ /* load iv in context1 */ -+ append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_WORD_CLASS_CTX | -+ LDST_CLASS_1_CCB | ivsize); -+ -+ /* assoc data is the only data for authentication */ -+ append_seq_fifo_load(desc, assoclen, FIFOLD_CLASS_CLASS2 | -+ FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST2); -+ -+ /* send icv to encryption */ -+ append_move(desc, MOVE_SRC_CLASS2CTX | MOVE_DEST_CLASS1INFIFO | -+ authsize); -+ -+ /* update class 1 data size register with padding length */ -+ append_load_imm_u32(desc, 0, LDST_CLASS_1_CCB | -+ LDST_SRCDST_WORD_DATASZ_REG | LDST_IMM); -+ -+ /* generate padding and send it to encryption */ -+ genpad = NFIFOENTRY_DEST_CLASS1 | NFIFOENTRY_LC1 | NFIFOENTRY_FC1 | -+ NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_PTYPE_N; -+ append_load_imm_u32(desc, genpad, LDST_CLASS_IND_CCB | -+ LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "tls enc shdesc@" __stringify(__LINE__) ": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, -+ desc_bytes(desc), 1); -+#endif -+} -+EXPORT_SYMBOL(cnstr_shdsc_tls_encap); -+ -+/** -+ * cnstr_shdsc_tls_decap - tls decapsulation shared descriptor -+ * @desc: pointer to buffer used for descriptor construction -+ * @cdata: pointer to block cipher transform definitions -+ * Valid algorithm values - one of OP_ALG_ALGSEL_AES ANDed -+ * with OP_ALG_AAI_CBC -+ * @adata: pointer to authentication transform definitions. -+ * A split key is required for SEC Era < 6; the size of the split key -+ * is specified in this case. Valid algorithm values OP_ALG_ALGSEL_SHA1 -+ * ANDed with OP_ALG_AAI_HMAC_PRECOMP. -+ * @assoclen: associated data length -+ * @ivsize: initialization vector size -+ * @authsize: authentication data size -+ * @blocksize: block cipher size -+ * @era: SEC Era -+ */ -+void cnstr_shdsc_tls_decap(u32 * const desc, struct alginfo *cdata, -+ struct alginfo *adata, unsigned int assoclen, -+ unsigned int ivsize, unsigned int authsize, -+ unsigned int blocksize, int era) -+{ -+ u32 stidx, jumpback; -+ u32 *key_jump_cmd, *zero_payload_jump_cmd, *skip_zero_jump_cmd; -+ /* -+ * Pointer Size bool determines the size of address pointers. -+ * false - Pointers fit in one 32-bit word. -+ * true - Pointers fit in two 32-bit words. -+ */ -+ static const bool ps = (CAAM_PTR_SZ != CAAM_CMD_SZ); -+ -+ stidx = 1 << HDR_START_IDX_SHIFT; -+ init_sh_desc(desc, HDR_SHARE_SERIAL | stidx); -+ -+ /* skip key loading if they are loaded due to sharing */ -+ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_SHRD); -+ -+ if (era < 6) -+ append_key(desc, adata->key_dma, adata->keylen, CLASS_2 | -+ KEY_DEST_MDHA_SPLIT | KEY_ENC); -+ else -+ append_proto_dkp(desc, adata); -+ -+ append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | -+ KEY_DEST_CLASS_REG); -+ -+ set_jump_tgt_here(desc, key_jump_cmd); -+ -+ /* class 2 operation */ -+ append_operation(desc, adata->algtype | OP_ALG_AS_INITFINAL | -+ OP_ALG_DECRYPT | OP_ALG_ICV_ON); -+ /* class 1 operation */ -+ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | -+ OP_ALG_DECRYPT); -+ -+ /* VSIL = input data length - 2 * block_size */ -+ append_math_sub_imm_u32(desc, VARSEQINLEN, SEQINLEN, IMM, 2 * -+ blocksize); -+ -+ /* -+ * payloadlen + icvlen + padlen = input data length - (assoclen + -+ * ivsize) -+ */ -+ append_math_sub_imm_u32(desc, REG3, SEQINLEN, IMM, assoclen + ivsize); -+ -+ /* skip data to the last but one cipher block */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_SKIP | LDST_VLF); -+ -+ /* load iv for the last cipher block */ -+ append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_WORD_CLASS_CTX | -+ LDST_CLASS_1_CCB | ivsize); -+ -+ /* read last cipher block */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG | -+ FIFOLD_TYPE_LAST1 | blocksize); -+ -+ /* move decrypted block into math0 and math1 */ -+ append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | MOVE_DEST_MATH0 | -+ blocksize); -+ -+ /* reset AES CHA */ -+ append_load_imm_u32(desc, CCTRL_RESET_CHA_AESA, LDST_CLASS_IND_CCB | -+ LDST_SRCDST_WORD_CHACTRL | LDST_IMM); -+ -+ /* rewind input sequence */ -+ append_seq_in_ptr_intlen(desc, 0, 65535, SQIN_RTO); -+ -+ /* key1 is in decryption form */ -+ append_operation(desc, cdata->algtype | OP_ALG_AAI_DK | -+ OP_ALG_AS_INITFINAL | OP_ALG_DECRYPT); -+ -+ /* load iv in context1 */ -+ append_cmd(desc, CMD_SEQ_LOAD | LDST_CLASS_1_CCB | -+ LDST_SRCDST_WORD_CLASS_CTX | ivsize); -+ -+ /* read sequence number */ -+ append_seq_fifo_load(desc, 8, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_MSG); -+ /* load Type, Version and Len fields in math0 */ -+ append_cmd(desc, CMD_SEQ_LOAD | LDST_CLASS_DECO | -+ LDST_SRCDST_WORD_DECO_MATH0 | (3 << LDST_OFFSET_SHIFT) | 5); -+ -+ /* compute (padlen - 1) */ -+ append_math_and_imm_u64(desc, REG1, REG1, IMM, 255); -+ -+ /* math2 = icvlen + (padlen - 1) + 1 */ -+ append_math_add_imm_u32(desc, REG2, REG1, IMM, authsize + 1); -+ -+ append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM | 1); -+ -+ /* VSOL = payloadlen + icvlen + padlen */ -+ append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, 4); -+ -+ if (caam_little_end) -+ append_moveb(desc, MOVE_WAITCOMP | -+ MOVE_SRC_MATH0 | MOVE_DEST_MATH0 | 8); -+ -+ /* update Len field */ -+ append_math_sub(desc, REG0, REG0, REG2, 8); -+ -+ /* store decrypted payload, icv and padding */ -+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | LDST_VLF); -+ -+ /* VSIL = (payloadlen + icvlen + padlen) - (icvlen + padlen)*/ -+ append_math_sub(desc, VARSEQINLEN, REG3, REG2, 4); -+ -+ zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | -+ JUMP_COND_MATH_Z); -+ -+ /* send Type, Version and Len(pre ICV) fields to authentication */ -+ append_move(desc, MOVE_WAITCOMP | -+ MOVE_SRC_MATH0 | MOVE_DEST_CLASS2INFIFO | -+ (3 << MOVE_OFFSET_SHIFT) | 5); -+ -+ /* outsnooping payload */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | -+ FIFOLD_TYPE_MSG1OUT2 | FIFOLD_TYPE_LAST2 | -+ FIFOLDST_VLF); -+ skip_zero_jump_cmd = append_jump(desc, JUMP_TEST_ALL | 2); -+ -+ set_jump_tgt_here(desc, zero_payload_jump_cmd); -+ /* send Type, Version and Len(pre ICV) fields to authentication */ -+ append_move(desc, MOVE_WAITCOMP | MOVE_AUX_LS | -+ MOVE_SRC_MATH0 | MOVE_DEST_CLASS2INFIFO | -+ (3 << MOVE_OFFSET_SHIFT) | 5); -+ -+ set_jump_tgt_here(desc, skip_zero_jump_cmd); -+ append_math_add(desc, VARSEQINLEN, ZERO, REG2, 4); -+ -+ /* load icvlen and padlen */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG | -+ FIFOLD_TYPE_LAST1 | FIFOLDST_VLF); -+ -+ /* VSIL = (payloadlen + icvlen + padlen) - icvlen + padlen */ -+ append_math_sub(desc, VARSEQINLEN, REG3, REG2, 4); -+ -+ /* -+ * Start a new input sequence using the SEQ OUT PTR command options, -+ * pointer and length used when the current output sequence was defined. -+ */ -+ if (ps) { -+ /* -+ * Move the lower 32 bits of Shared Descriptor address, the -+ * SEQ OUT PTR command, Output Pointer (2 words) and -+ * Output Length into math registers. -+ */ -+ if (caam_little_end) -+ append_move(desc, MOVE_WAITCOMP | MOVE_SRC_DESCBUF | -+ MOVE_DEST_MATH0 | -+ (55 * 4 << MOVE_OFFSET_SHIFT) | 20); -+ else -+ append_move(desc, MOVE_WAITCOMP | MOVE_SRC_DESCBUF | -+ MOVE_DEST_MATH0 | -+ (54 * 4 << MOVE_OFFSET_SHIFT) | 20); -+ -+ /* Transform SEQ OUT PTR command in SEQ IN PTR command */ -+ append_math_and_imm_u32(desc, REG0, REG0, IMM, -+ ~(CMD_SEQ_IN_PTR ^ CMD_SEQ_OUT_PTR)); -+ /* Append a JUMP command after the copied fields */ -+ jumpback = CMD_JUMP | (char)-9; -+ append_load_imm_u32(desc, jumpback, LDST_CLASS_DECO | LDST_IMM | -+ LDST_SRCDST_WORD_DECO_MATH2 | -+ (4 << LDST_OFFSET_SHIFT)); -+ append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM | 1); -+ /* Move the updated fields back to the Job Descriptor */ -+ if (caam_little_end) -+ append_move(desc, MOVE_WAITCOMP | MOVE_SRC_MATH0 | -+ MOVE_DEST_DESCBUF | -+ (55 * 4 << MOVE_OFFSET_SHIFT) | 24); -+ else -+ append_move(desc, MOVE_WAITCOMP | MOVE_SRC_MATH0 | -+ MOVE_DEST_DESCBUF | -+ (54 * 4 << MOVE_OFFSET_SHIFT) | 24); -+ -+ /* -+ * Read the new SEQ IN PTR command, Input Pointer, Input Length -+ * and then jump back to the next command from the -+ * Shared Descriptor. -+ */ -+ append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM | 6); -+ } else { -+ /* -+ * Move the SEQ OUT PTR command, Output Pointer (1 word) and -+ * Output Length into math registers. -+ */ -+ if (caam_little_end) -+ append_move(desc, MOVE_WAITCOMP | MOVE_SRC_DESCBUF | -+ MOVE_DEST_MATH0 | -+ (54 * 4 << MOVE_OFFSET_SHIFT) | 12); -+ else -+ append_move(desc, MOVE_WAITCOMP | MOVE_SRC_DESCBUF | -+ MOVE_DEST_MATH0 | -+ (53 * 4 << MOVE_OFFSET_SHIFT) | 12); -+ -+ /* Transform SEQ OUT PTR command in SEQ IN PTR command */ -+ append_math_and_imm_u64(desc, REG0, REG0, IMM, -+ ~(((u64)(CMD_SEQ_IN_PTR ^ -+ CMD_SEQ_OUT_PTR)) << 32)); -+ /* Append a JUMP command after the copied fields */ -+ jumpback = CMD_JUMP | (char)-7; -+ append_load_imm_u32(desc, jumpback, LDST_CLASS_DECO | LDST_IMM | -+ LDST_SRCDST_WORD_DECO_MATH1 | -+ (4 << LDST_OFFSET_SHIFT)); -+ append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM | 1); -+ /* Move the updated fields back to the Job Descriptor */ -+ if (caam_little_end) -+ append_move(desc, MOVE_WAITCOMP | MOVE_SRC_MATH0 | -+ MOVE_DEST_DESCBUF | -+ (54 * 4 << MOVE_OFFSET_SHIFT) | 16); -+ else -+ append_move(desc, MOVE_WAITCOMP | MOVE_SRC_MATH0 | -+ MOVE_DEST_DESCBUF | -+ (53 * 4 << MOVE_OFFSET_SHIFT) | 16); -+ -+ /* -+ * Read the new SEQ IN PTR command, Input Pointer, Input Length -+ * and then jump back to the next command from the -+ * Shared Descriptor. -+ */ -+ append_jump(desc, JUMP_TEST_ALL | JUMP_COND_CALM | 5); -+ } -+ -+ /* skip payload */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_SKIP | FIFOLDST_VLF); -+ /* check icv */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_ICV | -+ FIFOLD_TYPE_LAST2 | authsize); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "tls dec shdesc@" __stringify(__LINE__) ": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, -+ desc_bytes(desc), 1); -+#endif -+} -+EXPORT_SYMBOL(cnstr_shdsc_tls_decap); -+ -+/** -+ * cnstr_shdsc_gcm_encap - gcm encapsulation shared descriptor -+ * @desc: pointer to buffer used for descriptor construction -+ * @cdata: pointer to block cipher transform definitions -+ * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. -+ * @ivsize: initialization vector size -+ * @icvsize: integrity check value (ICV) size (truncated or full) -+ * @is_qi: true when called from caam/qi -+ */ -+void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata, -+ unsigned int ivsize, unsigned int icvsize, -+ const bool is_qi) -+{ -+ u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1, -+ *zero_assoc_jump_cmd2; -+ -+ init_sh_desc(desc, HDR_SHARE_SERIAL); -+ -+ /* skip key loading if they are loaded due to sharing */ -+ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_SHRD); -+ if (cdata->key_inline) -+ append_key_as_imm(desc, cdata->key_virt, cdata->keylen, -+ cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); -+ else -+ append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | -+ KEY_DEST_CLASS_REG); -+ set_jump_tgt_here(desc, key_jump_cmd); -+ -+ /* class 1 operation */ -+ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | -+ OP_ALG_ENCRYPT); -+ -+ if (is_qi) { -+ u32 *wait_load_cmd; -+ -+ /* REG3 = assoclen */ -+ append_seq_load(desc, 4, LDST_CLASS_DECO | -+ LDST_SRCDST_WORD_DECO_MATH3 | -+ (4 << LDST_OFFSET_SHIFT)); -+ -+ wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_CALM | JUMP_COND_NCP | -+ JUMP_COND_NOP | JUMP_COND_NIP | -+ JUMP_COND_NIFP); -+ set_jump_tgt_here(desc, wait_load_cmd); -+ -+ append_math_sub_imm_u32(desc, VARSEQOUTLEN, SEQINLEN, IMM, -+ ivsize); -+ } else { -+ append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, -+ CAAM_CMD_SZ); -+ } -+ -+ /* if assoclen + cryptlen is ZERO, skip to ICV write */ -+ zero_assoc_jump_cmd2 = append_jump(desc, JUMP_TEST_ALL | -+ JUMP_COND_MATH_Z); -+ -+ if (is_qi) -+ append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | -+ FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); -+ -+ /* if assoclen is ZERO, skip reading the assoc data */ -+ append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); -+ zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | -+ JUMP_COND_MATH_Z); -+ -+ append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); -+ -+ /* skip assoc data */ -+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); -+ -+ /* cryptlen = seqinlen - assoclen */ -+ append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG3, CAAM_CMD_SZ); -+ -+ /* if cryptlen is ZERO jump to zero-payload commands */ -+ zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | -+ JUMP_COND_MATH_Z); -+ -+ /* read assoc data */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -+ FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); -+ set_jump_tgt_here(desc, zero_assoc_jump_cmd1); -+ -+ append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -+ -+ /* write encrypted data */ -+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); -+ -+ /* read payload data */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -+ FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); -+ -+ /* jump to ICV writing */ -+ if (is_qi) -+ append_jump(desc, JUMP_TEST_ALL | 4); -+ else -+ append_jump(desc, JUMP_TEST_ALL | 2); -+ -+ /* zero-payload commands */ -+ set_jump_tgt_here(desc, zero_payload_jump_cmd); -+ -+ /* read assoc data */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -+ FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST1); -+ if (is_qi) -+ /* jump to ICV writing */ -+ append_jump(desc, JUMP_TEST_ALL | 2); -+ -+ /* There is no input data */ -+ set_jump_tgt_here(desc, zero_assoc_jump_cmd2); -+ -+ if (is_qi) -+ append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | -+ FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1 | -+ FIFOLD_TYPE_LAST1); -+ -+ /* write ICV */ -+ append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | -+ LDST_SRCDST_BYTE_CONTEXT); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "gcm enc shdesc@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+} -+EXPORT_SYMBOL(cnstr_shdsc_gcm_encap); -+ -+/** -+ * cnstr_shdsc_gcm_decap - gcm decapsulation shared descriptor -+ * @desc: pointer to buffer used for descriptor construction -+ * @cdata: pointer to block cipher transform definitions -+ * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. -+ * @ivsize: initialization vector size -+ * @icvsize: integrity check value (ICV) size (truncated or full) -+ * @is_qi: true when called from caam/qi -+ */ -+void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata, -+ unsigned int ivsize, unsigned int icvsize, -+ const bool is_qi) -+{ -+ u32 *key_jump_cmd, *zero_payload_jump_cmd, *zero_assoc_jump_cmd1; -+ -+ init_sh_desc(desc, HDR_SHARE_SERIAL); -+ -+ /* skip key loading if they are loaded due to sharing */ -+ key_jump_cmd = append_jump(desc, JUMP_JSL | -+ JUMP_TEST_ALL | JUMP_COND_SHRD); -+ if (cdata->key_inline) -+ append_key_as_imm(desc, cdata->key_virt, cdata->keylen, -+ cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); -+ else -+ append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | -+ KEY_DEST_CLASS_REG); -+ set_jump_tgt_here(desc, key_jump_cmd); -+ -+ /* class 1 operation */ -+ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | -+ OP_ALG_DECRYPT | OP_ALG_ICV_ON); -+ -+ if (is_qi) { -+ u32 *wait_load_cmd; -+ -+ /* REG3 = assoclen */ -+ append_seq_load(desc, 4, LDST_CLASS_DECO | -+ LDST_SRCDST_WORD_DECO_MATH3 | -+ (4 << LDST_OFFSET_SHIFT)); -+ -+ wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_CALM | JUMP_COND_NCP | -+ JUMP_COND_NOP | JUMP_COND_NIP | -+ JUMP_COND_NIFP); -+ set_jump_tgt_here(desc, wait_load_cmd); -+ -+ append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | -+ FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); -+ } -+ -+ /* if assoclen is ZERO, skip reading the assoc data */ -+ append_math_add(desc, VARSEQINLEN, ZERO, REG3, CAAM_CMD_SZ); -+ zero_assoc_jump_cmd1 = append_jump(desc, JUMP_TEST_ALL | -+ JUMP_COND_MATH_Z); -+ -+ append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); -+ -+ /* skip assoc data */ -+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); -+ -+ /* read assoc data */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -+ FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); -+ -+ set_jump_tgt_here(desc, zero_assoc_jump_cmd1); -+ -+ /* cryptlen = seqoutlen - assoclen */ -+ append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); -+ -+ /* jump to zero-payload command if cryptlen is zero */ -+ zero_payload_jump_cmd = append_jump(desc, JUMP_TEST_ALL | -+ JUMP_COND_MATH_Z); -+ -+ append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); -+ -+ /* store encrypted data */ -+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); -+ -+ /* read payload data */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -+ FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); -+ -+ /* zero-payload command */ -+ set_jump_tgt_here(desc, zero_payload_jump_cmd); -+ -+ /* read ICV */ -+ append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | -+ FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "gcm dec shdesc@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+} -+EXPORT_SYMBOL(cnstr_shdsc_gcm_decap); -+ -+/** -+ * cnstr_shdsc_rfc4106_encap - IPSec ESP gcm encapsulation shared descriptor -+ * (non-protocol). -+ * @desc: pointer to buffer used for descriptor construction -+ * @cdata: pointer to block cipher transform definitions -+ * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. -+ * @ivsize: initialization vector size -+ * @icvsize: integrity check value (ICV) size (truncated or full) -+ * @is_qi: true when called from caam/qi -+ */ -+void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata, -+ unsigned int ivsize, unsigned int icvsize, -+ const bool is_qi) -+{ -+ u32 *key_jump_cmd; -+ -+ init_sh_desc(desc, HDR_SHARE_SERIAL); -+ -+ /* Skip key loading if it is loaded due to sharing */ -+ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_SHRD); -+ if (cdata->key_inline) -+ append_key_as_imm(desc, cdata->key_virt, cdata->keylen, -+ cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); -+ else -+ append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | -+ KEY_DEST_CLASS_REG); -+ set_jump_tgt_here(desc, key_jump_cmd); -+ -+ /* Class 1 operation */ -+ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | -+ OP_ALG_ENCRYPT); -+ -+ if (is_qi) { -+ u32 *wait_load_cmd; -+ -+ /* REG3 = assoclen */ -+ append_seq_load(desc, 4, LDST_CLASS_DECO | -+ LDST_SRCDST_WORD_DECO_MATH3 | -+ (4 << LDST_OFFSET_SHIFT)); -+ -+ wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_CALM | JUMP_COND_NCP | -+ JUMP_COND_NOP | JUMP_COND_NIP | -+ JUMP_COND_NIFP); -+ set_jump_tgt_here(desc, wait_load_cmd); -+ -+ /* Read salt and IV */ -+ append_fifo_load_as_imm(desc, (void *)(cdata->key_virt + -+ cdata->keylen), 4, FIFOLD_CLASS_CLASS1 | -+ FIFOLD_TYPE_IV); -+ append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | -+ FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); -+ } -+ -+ append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize); -+ append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); -+ -+ /* Read assoc data */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -+ FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); -+ -+ /* Skip IV */ -+ append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP); -+ -+ /* Will read cryptlen bytes */ -+ append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -+ -+ /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); -+ -+ /* Skip assoc data */ -+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); -+ -+ /* cryptlen = seqoutlen - assoclen */ -+ append_math_sub(desc, VARSEQOUTLEN, VARSEQINLEN, REG0, CAAM_CMD_SZ); -+ -+ /* Write encrypted data */ -+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); -+ -+ /* Read payload data */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -+ FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); -+ -+ /* Write ICV */ -+ append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | -+ LDST_SRCDST_BYTE_CONTEXT); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, -+ "rfc4106 enc shdesc@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+} -+EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap); -+ -+/** -+ * cnstr_shdsc_rfc4106_decap - IPSec ESP gcm decapsulation shared descriptor -+ * (non-protocol). -+ * @desc: pointer to buffer used for descriptor construction -+ * @cdata: pointer to block cipher transform definitions -+ * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. -+ * @ivsize: initialization vector size -+ * @icvsize: integrity check value (ICV) size (truncated or full) -+ * @is_qi: true when called from caam/qi -+ */ -+void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata, -+ unsigned int ivsize, unsigned int icvsize, -+ const bool is_qi) -+{ -+ u32 *key_jump_cmd; -+ -+ init_sh_desc(desc, HDR_SHARE_SERIAL); -+ -+ /* Skip key loading if it is loaded due to sharing */ -+ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_SHRD); -+ if (cdata->key_inline) -+ append_key_as_imm(desc, cdata->key_virt, cdata->keylen, -+ cdata->keylen, CLASS_1 | -+ KEY_DEST_CLASS_REG); -+ else -+ append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | -+ KEY_DEST_CLASS_REG); -+ set_jump_tgt_here(desc, key_jump_cmd); -+ -+ /* Class 1 operation */ -+ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | -+ OP_ALG_DECRYPT | OP_ALG_ICV_ON); -+ -+ if (is_qi) { -+ u32 *wait_load_cmd; -+ -+ /* REG3 = assoclen */ -+ append_seq_load(desc, 4, LDST_CLASS_DECO | -+ LDST_SRCDST_WORD_DECO_MATH3 | -+ (4 << LDST_OFFSET_SHIFT)); -+ -+ wait_load_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_CALM | JUMP_COND_NCP | -+ JUMP_COND_NOP | JUMP_COND_NIP | -+ JUMP_COND_NIFP); -+ set_jump_tgt_here(desc, wait_load_cmd); -+ -+ /* Read salt and IV */ -+ append_fifo_load_as_imm(desc, (void *)(cdata->key_virt + -+ cdata->keylen), 4, FIFOLD_CLASS_CLASS1 | -+ FIFOLD_TYPE_IV); -+ append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | -+ FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); -+ } -+ -+ append_math_sub_imm_u32(desc, VARSEQINLEN, REG3, IMM, ivsize); -+ append_math_add(desc, VARSEQOUTLEN, ZERO, REG3, CAAM_CMD_SZ); -+ -+ /* Read assoc data */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -+ FIFOLD_TYPE_AAD | FIFOLD_TYPE_FLUSH1); -+ -+ /* Skip IV */ -+ append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_SKIP); -+ -+ /* Will read cryptlen bytes */ -+ append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG3, CAAM_CMD_SZ); -+ -+ /* Workaround for erratum A-005473 (simultaneous SEQ FIFO skips) */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_MSG); -+ -+ /* Skip assoc data */ -+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_SKIP | FIFOLDST_VLF); -+ -+ /* Will write cryptlen bytes */ -+ append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); -+ -+ /* Store payload data */ -+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); -+ -+ /* Read encrypted data */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | FIFOLDST_VLF | -+ FIFOLD_TYPE_MSG | FIFOLD_TYPE_FLUSH1); -+ -+ /* Read ICV */ -+ append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | -+ FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, -+ "rfc4106 dec shdesc@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+} -+EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap); -+ -+/** -+ * cnstr_shdsc_rfc4543_encap - IPSec ESP gmac encapsulation shared descriptor -+ * (non-protocol). -+ * @desc: pointer to buffer used for descriptor construction -+ * @cdata: pointer to block cipher transform definitions -+ * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. -+ * @ivsize: initialization vector size -+ * @icvsize: integrity check value (ICV) size (truncated or full) -+ * @is_qi: true when called from caam/qi -+ */ -+void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata, -+ unsigned int ivsize, unsigned int icvsize, -+ const bool is_qi) -+{ -+ u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; -+ -+ init_sh_desc(desc, HDR_SHARE_SERIAL); -+ -+ /* Skip key loading if it is loaded due to sharing */ -+ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_SHRD); -+ if (cdata->key_inline) -+ append_key_as_imm(desc, cdata->key_virt, cdata->keylen, -+ cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); -+ else -+ append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | -+ KEY_DEST_CLASS_REG); -+ set_jump_tgt_here(desc, key_jump_cmd); -+ -+ /* Class 1 operation */ -+ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | -+ OP_ALG_ENCRYPT); -+ -+ if (is_qi) { -+ /* assoclen is not needed, skip it */ -+ append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP); -+ -+ /* Read salt and IV */ -+ append_fifo_load_as_imm(desc, (void *)(cdata->key_virt + -+ cdata->keylen), 4, FIFOLD_CLASS_CLASS1 | -+ FIFOLD_TYPE_IV); -+ append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | -+ FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); -+ } -+ -+ /* assoclen + cryptlen = seqinlen */ -+ append_math_sub(desc, REG3, SEQINLEN, REG0, CAAM_CMD_SZ); -+ -+ /* -+ * MOVE_LEN opcode is not available in all SEC HW revisions, -+ * thus need to do some magic, i.e. self-patch the descriptor -+ * buffer. -+ */ -+ read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | -+ (0x6 << MOVE_LEN_SHIFT)); -+ write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | -+ (0x8 << MOVE_LEN_SHIFT)); -+ -+ /* Will read assoclen + cryptlen bytes */ -+ append_math_sub(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -+ -+ /* Will write assoclen + cryptlen bytes */ -+ append_math_sub(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -+ -+ /* Read and write assoclen + cryptlen bytes */ -+ aead_append_src_dst(desc, FIFOLD_TYPE_AAD); -+ -+ set_move_tgt_here(desc, read_move_cmd); -+ set_move_tgt_here(desc, write_move_cmd); -+ append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); -+ /* Move payload data to OFIFO */ -+ append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); -+ -+ /* Write ICV */ -+ append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | -+ LDST_SRCDST_BYTE_CONTEXT); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, -+ "rfc4543 enc shdesc@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+} -+EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap); -+ -+/** -+ * cnstr_shdsc_rfc4543_decap - IPSec ESP gmac decapsulation shared descriptor -+ * (non-protocol). -+ * @desc: pointer to buffer used for descriptor construction -+ * @cdata: pointer to block cipher transform definitions -+ * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_GCM. -+ * @ivsize: initialization vector size -+ * @icvsize: integrity check value (ICV) size (truncated or full) -+ * @is_qi: true when called from caam/qi -+ */ -+void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata, -+ unsigned int ivsize, unsigned int icvsize, -+ const bool is_qi) -+{ -+ u32 *key_jump_cmd, *read_move_cmd, *write_move_cmd; -+ -+ init_sh_desc(desc, HDR_SHARE_SERIAL); -+ -+ /* Skip key loading if it is loaded due to sharing */ -+ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_SHRD); -+ if (cdata->key_inline) -+ append_key_as_imm(desc, cdata->key_virt, cdata->keylen, -+ cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); -+ else -+ append_key(desc, cdata->key_dma, cdata->keylen, CLASS_1 | -+ KEY_DEST_CLASS_REG); -+ set_jump_tgt_here(desc, key_jump_cmd); -+ -+ /* Class 1 operation */ -+ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | -+ OP_ALG_DECRYPT | OP_ALG_ICV_ON); -+ -+ if (is_qi) { -+ /* assoclen is not needed, skip it */ -+ append_seq_fifo_load(desc, 4, FIFOLD_CLASS_SKIP); -+ -+ /* Read salt and IV */ -+ append_fifo_load_as_imm(desc, (void *)(cdata->key_virt + -+ cdata->keylen), 4, FIFOLD_CLASS_CLASS1 | -+ FIFOLD_TYPE_IV); -+ append_seq_fifo_load(desc, ivsize, FIFOLD_CLASS_CLASS1 | -+ FIFOLD_TYPE_IV | FIFOLD_TYPE_FLUSH1); -+ } -+ -+ /* assoclen + cryptlen = seqoutlen */ -+ append_math_sub(desc, REG3, SEQOUTLEN, REG0, CAAM_CMD_SZ); -+ -+ /* -+ * MOVE_LEN opcode is not available in all SEC HW revisions, -+ * thus need to do some magic, i.e. self-patch the descriptor -+ * buffer. -+ */ -+ read_move_cmd = append_move(desc, MOVE_SRC_DESCBUF | MOVE_DEST_MATH3 | -+ (0x6 << MOVE_LEN_SHIFT)); -+ write_move_cmd = append_move(desc, MOVE_SRC_MATH3 | MOVE_DEST_DESCBUF | -+ (0x8 << MOVE_LEN_SHIFT)); -+ -+ /* Will read assoclen + cryptlen bytes */ -+ append_math_sub(desc, VARSEQINLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); -+ -+ /* Will write assoclen + cryptlen bytes */ -+ append_math_sub(desc, VARSEQOUTLEN, SEQOUTLEN, REG0, CAAM_CMD_SZ); -+ -+ /* Store payload data */ -+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | FIFOLDST_VLF); -+ -+ /* In-snoop assoclen + cryptlen data */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_BOTH | FIFOLDST_VLF | -+ FIFOLD_TYPE_AAD | FIFOLD_TYPE_LAST2FLUSH1); -+ -+ set_move_tgt_here(desc, read_move_cmd); -+ set_move_tgt_here(desc, write_move_cmd); -+ append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); -+ /* Move payload data to OFIFO */ -+ append_move(desc, MOVE_SRC_INFIFO_CL | MOVE_DEST_OUTFIFO); -+ append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); -+ -+ /* Read ICV */ -+ append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | -+ FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, -+ "rfc4543 dec shdesc@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+} -+EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap); -+ -+/* -+ * For ablkcipher encrypt and decrypt, read from req->src and -+ * write to req->dst -+ */ -+static inline void ablkcipher_append_src_dst(u32 *desc) -+{ -+ append_math_add(desc, VARSEQOUTLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -+ append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS1 | -+ KEY_VLF | FIFOLD_TYPE_MSG | FIFOLD_TYPE_LAST1); -+ append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); -+} -+ -+/** -+ * cnstr_shdsc_ablkcipher_encap - ablkcipher encapsulation shared descriptor -+ * @desc: pointer to buffer used for descriptor construction -+ * @cdata: pointer to block cipher transform definitions -+ * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed -+ * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. -+ * @ivsize: initialization vector size -+ * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template -+ * @ctx1_iv_off: IV offset in CONTEXT1 register -+ */ -+void cnstr_shdsc_ablkcipher_encap(u32 * const desc, struct alginfo *cdata, -+ unsigned int ivsize, const bool is_rfc3686, -+ const u32 ctx1_iv_off) -+{ -+ u32 *key_jump_cmd; -+ -+ init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); -+ /* Skip if already shared */ -+ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_SHRD); -+ -+ /* Load class1 key only */ -+ append_key_as_imm(desc, cdata->key_virt, cdata->keylen, -+ cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); -+ -+ /* Load nonce into CONTEXT1 reg */ -+ if (is_rfc3686) { -+ const u8 *nonce = cdata->key_virt + cdata->keylen; -+ -+ append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, -+ LDST_CLASS_IND_CCB | -+ LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); -+ append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | -+ MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | -+ (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); -+ } -+ -+ set_jump_tgt_here(desc, key_jump_cmd); -+ -+ /* Load iv */ -+ append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | -+ LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); -+ -+ /* Load counter into CONTEXT1 reg */ -+ if (is_rfc3686) -+ append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | -+ LDST_SRCDST_BYTE_CONTEXT | -+ ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << -+ LDST_OFFSET_SHIFT)); -+ -+ /* Load operation */ -+ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | -+ OP_ALG_ENCRYPT); -+ -+ /* Perform operation */ -+ ablkcipher_append_src_dst(desc); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, -+ "ablkcipher enc shdesc@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+} -+EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_encap); -+ -+/** -+ * cnstr_shdsc_ablkcipher_decap - ablkcipher decapsulation shared descriptor -+ * @desc: pointer to buffer used for descriptor construction -+ * @cdata: pointer to block cipher transform definitions -+ * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed -+ * with OP_ALG_AAI_CBC or OP_ALG_AAI_CTR_MOD128. -+ * @ivsize: initialization vector size -+ * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template -+ * @ctx1_iv_off: IV offset in CONTEXT1 register -+ */ -+void cnstr_shdsc_ablkcipher_decap(u32 * const desc, struct alginfo *cdata, -+ unsigned int ivsize, const bool is_rfc3686, -+ const u32 ctx1_iv_off) -+{ -+ u32 *key_jump_cmd; -+ -+ init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); -+ /* Skip if already shared */ -+ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_SHRD); -+ -+ /* Load class1 key only */ -+ append_key_as_imm(desc, cdata->key_virt, cdata->keylen, -+ cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); -+ -+ /* Load nonce into CONTEXT1 reg */ -+ if (is_rfc3686) { -+ const u8 *nonce = cdata->key_virt + cdata->keylen; -+ -+ append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, -+ LDST_CLASS_IND_CCB | -+ LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); -+ append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | -+ MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | -+ (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); -+ } -+ -+ set_jump_tgt_here(desc, key_jump_cmd); -+ -+ /* load IV */ -+ append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | -+ LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); -+ -+ /* Load counter into CONTEXT1 reg */ -+ if (is_rfc3686) -+ append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | -+ LDST_SRCDST_BYTE_CONTEXT | -+ ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << -+ LDST_OFFSET_SHIFT)); -+ -+ /* Choose operation */ -+ if (ctx1_iv_off) -+ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | -+ OP_ALG_DECRYPT); -+ else -+ append_dec_op1(desc, cdata->algtype); -+ -+ /* Perform operation */ -+ ablkcipher_append_src_dst(desc); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, -+ "ablkcipher dec shdesc@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+} -+EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_decap); -+ -+/** -+ * cnstr_shdsc_ablkcipher_givencap - ablkcipher encapsulation shared descriptor -+ * with HW-generated initialization vector. -+ * @desc: pointer to buffer used for descriptor construction -+ * @cdata: pointer to block cipher transform definitions -+ * Valid algorithm values - one of OP_ALG_ALGSEL_{AES, DES, 3DES} ANDed -+ * with OP_ALG_AAI_CBC. -+ * @ivsize: initialization vector size -+ * @is_rfc3686: true when ctr(aes) is wrapped by rfc3686 template -+ * @ctx1_iv_off: IV offset in CONTEXT1 register -+ */ -+void cnstr_shdsc_ablkcipher_givencap(u32 * const desc, struct alginfo *cdata, -+ unsigned int ivsize, const bool is_rfc3686, -+ const u32 ctx1_iv_off) -+{ -+ u32 *key_jump_cmd, geniv; -+ -+ init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); -+ /* Skip if already shared */ -+ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_SHRD); -+ -+ /* Load class1 key only */ -+ append_key_as_imm(desc, cdata->key_virt, cdata->keylen, -+ cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); -+ -+ /* Load Nonce into CONTEXT1 reg */ -+ if (is_rfc3686) { -+ const u8 *nonce = cdata->key_virt + cdata->keylen; -+ -+ append_load_as_imm(desc, nonce, CTR_RFC3686_NONCE_SIZE, -+ LDST_CLASS_IND_CCB | -+ LDST_SRCDST_BYTE_OUTFIFO | LDST_IMM); -+ append_move(desc, MOVE_WAITCOMP | MOVE_SRC_OUTFIFO | -+ MOVE_DEST_CLASS1CTX | (16 << MOVE_OFFSET_SHIFT) | -+ (CTR_RFC3686_NONCE_SIZE << MOVE_LEN_SHIFT)); -+ } -+ set_jump_tgt_here(desc, key_jump_cmd); -+ -+ /* Generate IV */ -+ geniv = NFIFOENTRY_STYPE_PAD | NFIFOENTRY_DEST_DECO | -+ NFIFOENTRY_DTYPE_MSG | NFIFOENTRY_LC1 | NFIFOENTRY_PTYPE_RND | -+ (ivsize << NFIFOENTRY_DLEN_SHIFT); -+ append_load_imm_u32(desc, geniv, LDST_CLASS_IND_CCB | -+ LDST_SRCDST_WORD_INFO_FIFO | LDST_IMM); -+ append_cmd(desc, CMD_LOAD | DISABLE_AUTO_INFO_FIFO); -+ append_move(desc, MOVE_WAITCOMP | MOVE_SRC_INFIFO | -+ MOVE_DEST_CLASS1CTX | (ivsize << MOVE_LEN_SHIFT) | -+ (ctx1_iv_off << MOVE_OFFSET_SHIFT)); -+ append_cmd(desc, CMD_LOAD | ENABLE_AUTO_INFO_FIFO); -+ -+ /* Copy generated IV to memory */ -+ append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | -+ LDST_CLASS_1_CCB | (ctx1_iv_off << LDST_OFFSET_SHIFT)); -+ -+ /* Load Counter into CONTEXT1 reg */ -+ if (is_rfc3686) -+ append_load_imm_be32(desc, 1, LDST_IMM | LDST_CLASS_1_CCB | -+ LDST_SRCDST_BYTE_CONTEXT | -+ ((ctx1_iv_off + CTR_RFC3686_IV_SIZE) << -+ LDST_OFFSET_SHIFT)); -+ -+ if (ctx1_iv_off) -+ append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | JUMP_COND_NCP | -+ (1 << JUMP_OFFSET_SHIFT)); -+ -+ /* Load operation */ -+ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | -+ OP_ALG_ENCRYPT); -+ -+ /* Perform operation */ -+ ablkcipher_append_src_dst(desc); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, -+ "ablkcipher givenc shdesc@" __stringify(__LINE__) ": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+} -+EXPORT_SYMBOL(cnstr_shdsc_ablkcipher_givencap); -+ -+/** -+ * cnstr_shdsc_xts_ablkcipher_encap - xts ablkcipher encapsulation shared -+ * descriptor -+ * @desc: pointer to buffer used for descriptor construction -+ * @cdata: pointer to block cipher transform definitions -+ * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS. -+ */ -+void cnstr_shdsc_xts_ablkcipher_encap(u32 * const desc, struct alginfo *cdata) -+{ -+ __be64 sector_size = cpu_to_be64(512); -+ u32 *key_jump_cmd; -+ -+ init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); -+ /* Skip if already shared */ -+ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_SHRD); -+ -+ /* Load class1 keys only */ -+ append_key_as_imm(desc, cdata->key_virt, cdata->keylen, -+ cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); -+ -+ /* Load sector size with index 40 bytes (0x28) */ -+ append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB | -+ LDST_SRCDST_BYTE_CONTEXT | -+ (0x28 << LDST_OFFSET_SHIFT)); -+ -+ set_jump_tgt_here(desc, key_jump_cmd); -+ -+ /* -+ * create sequence for loading the sector index -+ * Upper 8B of IV - will be used as sector index -+ * Lower 8B of IV - will be discarded -+ */ -+ append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | -+ (0x20 << LDST_OFFSET_SHIFT)); -+ append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); -+ -+ /* Load operation */ -+ append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | -+ OP_ALG_ENCRYPT); -+ -+ /* Perform operation */ -+ ablkcipher_append_src_dst(desc); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, -+ "xts ablkcipher enc shdesc@" __stringify(__LINE__) ": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+} -+EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_encap); -+ -+/** -+ * cnstr_shdsc_xts_ablkcipher_decap - xts ablkcipher decapsulation shared -+ * descriptor -+ * @desc: pointer to buffer used for descriptor construction -+ * @cdata: pointer to block cipher transform definitions -+ * Valid algorithm values - OP_ALG_ALGSEL_AES ANDed with OP_ALG_AAI_XTS. -+ */ -+void cnstr_shdsc_xts_ablkcipher_decap(u32 * const desc, struct alginfo *cdata) -+{ -+ __be64 sector_size = cpu_to_be64(512); -+ u32 *key_jump_cmd; -+ -+ init_sh_desc(desc, HDR_SHARE_SERIAL | HDR_SAVECTX); -+ /* Skip if already shared */ -+ key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_SHRD); -+ -+ /* Load class1 key only */ -+ append_key_as_imm(desc, cdata->key_virt, cdata->keylen, -+ cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); -+ -+ /* Load sector size with index 40 bytes (0x28) */ -+ append_load_as_imm(desc, (void *)§or_size, 8, LDST_CLASS_1_CCB | -+ LDST_SRCDST_BYTE_CONTEXT | -+ (0x28 << LDST_OFFSET_SHIFT)); -+ -+ set_jump_tgt_here(desc, key_jump_cmd); -+ -+ /* -+ * create sequence for loading the sector index -+ * Upper 8B of IV - will be used as sector index -+ * Lower 8B of IV - will be discarded -+ */ -+ append_seq_load(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | -+ (0x20 << LDST_OFFSET_SHIFT)); -+ append_seq_fifo_load(desc, 8, FIFOLD_CLASS_SKIP); -+ -+ /* Load operation */ -+ append_dec_op1(desc, cdata->algtype); -+ -+ /* Perform operation */ -+ ablkcipher_append_src_dst(desc); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, -+ "xts ablkcipher dec shdesc@" __stringify(__LINE__) ": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+} -+EXPORT_SYMBOL(cnstr_shdsc_xts_ablkcipher_decap); -+ -+MODULE_LICENSE("GPL"); -+MODULE_DESCRIPTION("FSL CAAM descriptor support"); -+MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); ---- /dev/null -+++ b/drivers/crypto/caam/caamalg_desc.h -@@ -0,0 +1,127 @@ -+/* -+ * Shared descriptors for aead, ablkcipher algorithms -+ * -+ * Copyright 2016 NXP -+ */ -+ -+#ifndef _CAAMALG_DESC_H_ -+#define _CAAMALG_DESC_H_ -+ -+/* length of descriptors text */ -+#define DESC_AEAD_BASE (4 * CAAM_CMD_SZ) -+#define DESC_AEAD_ENC_LEN (DESC_AEAD_BASE + 11 * CAAM_CMD_SZ) -+#define DESC_AEAD_DEC_LEN (DESC_AEAD_BASE + 15 * CAAM_CMD_SZ) -+#define DESC_AEAD_GIVENC_LEN (DESC_AEAD_ENC_LEN + 7 * CAAM_CMD_SZ) -+#define DESC_QI_AEAD_ENC_LEN (DESC_AEAD_ENC_LEN + 3 * CAAM_CMD_SZ) -+#define DESC_QI_AEAD_DEC_LEN (DESC_AEAD_DEC_LEN + 3 * CAAM_CMD_SZ) -+#define DESC_QI_AEAD_GIVENC_LEN (DESC_AEAD_GIVENC_LEN + 3 * CAAM_CMD_SZ) -+ -+#define DESC_TLS_BASE (4 * CAAM_CMD_SZ) -+#define DESC_TLS10_ENC_LEN (DESC_TLS_BASE + 29 * CAAM_CMD_SZ) -+ -+/* Note: Nonce is counted in cdata.keylen */ -+#define DESC_AEAD_CTR_RFC3686_LEN (4 * CAAM_CMD_SZ) -+ -+#define DESC_AEAD_NULL_BASE (3 * CAAM_CMD_SZ) -+#define DESC_AEAD_NULL_ENC_LEN (DESC_AEAD_NULL_BASE + 11 * CAAM_CMD_SZ) -+#define DESC_AEAD_NULL_DEC_LEN (DESC_AEAD_NULL_BASE + 13 * CAAM_CMD_SZ) -+ -+#define DESC_GCM_BASE (3 * CAAM_CMD_SZ) -+#define DESC_GCM_ENC_LEN (DESC_GCM_BASE + 16 * CAAM_CMD_SZ) -+#define DESC_GCM_DEC_LEN (DESC_GCM_BASE + 12 * CAAM_CMD_SZ) -+#define DESC_QI_GCM_ENC_LEN (DESC_GCM_ENC_LEN + 6 * CAAM_CMD_SZ) -+#define DESC_QI_GCM_DEC_LEN (DESC_GCM_DEC_LEN + 3 * CAAM_CMD_SZ) -+ -+#define DESC_RFC4106_BASE (3 * CAAM_CMD_SZ) -+#define DESC_RFC4106_ENC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ) -+#define DESC_RFC4106_DEC_LEN (DESC_RFC4106_BASE + 13 * CAAM_CMD_SZ) -+#define DESC_QI_RFC4106_ENC_LEN (DESC_RFC4106_ENC_LEN + 5 * CAAM_CMD_SZ) -+#define DESC_QI_RFC4106_DEC_LEN (DESC_RFC4106_DEC_LEN + 5 * CAAM_CMD_SZ) -+ -+#define DESC_RFC4543_BASE (3 * CAAM_CMD_SZ) -+#define DESC_RFC4543_ENC_LEN (DESC_RFC4543_BASE + 11 * CAAM_CMD_SZ) -+#define DESC_RFC4543_DEC_LEN (DESC_RFC4543_BASE + 12 * CAAM_CMD_SZ) -+#define DESC_QI_RFC4543_ENC_LEN (DESC_RFC4543_ENC_LEN + 4 * CAAM_CMD_SZ) -+#define DESC_QI_RFC4543_DEC_LEN (DESC_RFC4543_DEC_LEN + 4 * CAAM_CMD_SZ) -+ -+#define DESC_ABLKCIPHER_BASE (3 * CAAM_CMD_SZ) -+#define DESC_ABLKCIPHER_ENC_LEN (DESC_ABLKCIPHER_BASE + \ -+ 20 * CAAM_CMD_SZ) -+#define DESC_ABLKCIPHER_DEC_LEN (DESC_ABLKCIPHER_BASE + \ -+ 15 * CAAM_CMD_SZ) -+ -+void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, -+ unsigned int icvsize, int era); -+ -+void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata, -+ unsigned int icvsize, int era); -+ -+void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata, -+ struct alginfo *adata, unsigned int ivsize, -+ unsigned int icvsize, const bool is_rfc3686, -+ u32 *nonce, const u32 ctx1_iv_off, -+ const bool is_qi, int era); -+ -+void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata, -+ struct alginfo *adata, unsigned int ivsize, -+ unsigned int icvsize, const bool geniv, -+ const bool is_rfc3686, u32 *nonce, -+ const u32 ctx1_iv_off, const bool is_qi, int era); -+ -+void cnstr_shdsc_aead_givencap(u32 * const desc, struct alginfo *cdata, -+ struct alginfo *adata, unsigned int ivsize, -+ unsigned int icvsize, const bool is_rfc3686, -+ u32 *nonce, const u32 ctx1_iv_off, -+ const bool is_qi, int era); -+ -+void cnstr_shdsc_tls_encap(u32 *const desc, struct alginfo *cdata, -+ struct alginfo *adata, unsigned int assoclen, -+ unsigned int ivsize, unsigned int authsize, -+ unsigned int blocksize, int era); -+ -+void cnstr_shdsc_tls_decap(u32 *const desc, struct alginfo *cdata, -+ struct alginfo *adata, unsigned int assoclen, -+ unsigned int ivsize, unsigned int authsize, -+ unsigned int blocksize, int era); -+ -+void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata, -+ unsigned int ivsize, unsigned int icvsize, -+ const bool is_qi); -+ -+void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata, -+ unsigned int ivsize, unsigned int icvsize, -+ const bool is_qi); -+ -+void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata, -+ unsigned int ivsize, unsigned int icvsize, -+ const bool is_qi); -+ -+void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata, -+ unsigned int ivsize, unsigned int icvsize, -+ const bool is_qi); -+ -+void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata, -+ unsigned int ivsize, unsigned int icvsize, -+ const bool is_qi); -+ -+void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata, -+ unsigned int ivsize, unsigned int icvsize, -+ const bool is_qi); -+ -+void cnstr_shdsc_ablkcipher_encap(u32 * const desc, struct alginfo *cdata, -+ unsigned int ivsize, const bool is_rfc3686, -+ const u32 ctx1_iv_off); -+ -+void cnstr_shdsc_ablkcipher_decap(u32 * const desc, struct alginfo *cdata, -+ unsigned int ivsize, const bool is_rfc3686, -+ const u32 ctx1_iv_off); -+ -+void cnstr_shdsc_ablkcipher_givencap(u32 * const desc, struct alginfo *cdata, -+ unsigned int ivsize, const bool is_rfc3686, -+ const u32 ctx1_iv_off); -+ -+void cnstr_shdsc_xts_ablkcipher_encap(u32 * const desc, struct alginfo *cdata); -+ -+void cnstr_shdsc_xts_ablkcipher_decap(u32 * const desc, struct alginfo *cdata); -+ -+#endif /* _CAAMALG_DESC_H_ */ ---- /dev/null -+++ b/drivers/crypto/caam/caamalg_qi.c -@@ -0,0 +1,3321 @@ -+/* -+ * Freescale FSL CAAM support for crypto API over QI backend. -+ * Based on caamalg.c -+ * -+ * Copyright 2013-2016 Freescale Semiconductor, Inc. -+ * Copyright 2016-2017 NXP -+ */ -+ -+#include "compat.h" -+#include "ctrl.h" -+#include "regs.h" -+#include "intern.h" -+#include "desc_constr.h" -+#include "error.h" -+#include "sg_sw_qm.h" -+#include "key_gen.h" -+#include "qi.h" -+#include "jr.h" -+#include "caamalg_desc.h" -+ -+/* -+ * crypto alg -+ */ -+#define CAAM_CRA_PRIORITY 2000 -+/* max key is sum of AES_MAX_KEY_SIZE, max split key size */ -+#define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + \ -+ SHA512_DIGEST_SIZE * 2) -+ -+#define DESC_MAX_USED_BYTES (DESC_QI_AEAD_GIVENC_LEN + \ -+ CAAM_MAX_KEY_SIZE) -+#define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) -+ -+struct caam_alg_entry { -+ int class1_alg_type; -+ int class2_alg_type; -+ bool rfc3686; -+ bool geniv; -+}; -+ -+struct caam_aead_alg { -+ struct aead_alg aead; -+ struct caam_alg_entry caam; -+ bool registered; -+}; -+ -+/* -+ * per-session context -+ */ -+struct caam_ctx { -+ struct device *jrdev; -+ u32 sh_desc_enc[DESC_MAX_USED_LEN]; -+ u32 sh_desc_dec[DESC_MAX_USED_LEN]; -+ u32 sh_desc_givenc[DESC_MAX_USED_LEN]; -+ u8 key[CAAM_MAX_KEY_SIZE]; -+ dma_addr_t key_dma; -+ enum dma_data_direction dir; -+ struct alginfo adata; -+ struct alginfo cdata; -+ unsigned int authsize; -+ struct device *qidev; -+ spinlock_t lock; /* Protects multiple init of driver context */ -+ struct caam_drv_ctx *drv_ctx[NUM_OP]; -+}; -+ -+static int aead_set_sh_desc(struct crypto_aead *aead) -+{ -+ struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), -+ typeof(*alg), aead); -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ unsigned int ivsize = crypto_aead_ivsize(aead); -+ u32 ctx1_iv_off = 0; -+ u32 *nonce = NULL; -+ unsigned int data_len[2]; -+ u32 inl_mask; -+ const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == -+ OP_ALG_AAI_CTR_MOD128); -+ const bool is_rfc3686 = alg->caam.rfc3686; -+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); -+ -+ if (!ctx->cdata.keylen || !ctx->authsize) -+ return 0; -+ -+ /* -+ * AES-CTR needs to load IV in CONTEXT1 reg -+ * at an offset of 128bits (16bytes) -+ * CONTEXT1[255:128] = IV -+ */ -+ if (ctr_mode) -+ ctx1_iv_off = 16; -+ -+ /* -+ * RFC3686 specific: -+ * CONTEXT1[255:128] = {NONCE, IV, COUNTER} -+ */ -+ if (is_rfc3686) { -+ ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; -+ nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + -+ ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); -+ } -+ -+ data_len[0] = ctx->adata.keylen_pad; -+ data_len[1] = ctx->cdata.keylen; -+ -+ if (alg->caam.geniv) -+ goto skip_enc; -+ -+ /* aead_encrypt shared descriptor */ -+ if (desc_inline_query(DESC_QI_AEAD_ENC_LEN + -+ (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), -+ DESC_JOB_IO_LEN, data_len, &inl_mask, -+ ARRAY_SIZE(data_len)) < 0) -+ return -EINVAL; -+ -+ if (inl_mask & 1) -+ ctx->adata.key_virt = ctx->key; -+ else -+ ctx->adata.key_dma = ctx->key_dma; -+ -+ if (inl_mask & 2) -+ ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; -+ else -+ ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; -+ -+ ctx->adata.key_inline = !!(inl_mask & 1); -+ ctx->cdata.key_inline = !!(inl_mask & 2); -+ -+ cnstr_shdsc_aead_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, -+ ivsize, ctx->authsize, is_rfc3686, nonce, -+ ctx1_iv_off, true, ctrlpriv->era); -+ -+skip_enc: -+ /* aead_decrypt shared descriptor */ -+ if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + -+ (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), -+ DESC_JOB_IO_LEN, data_len, &inl_mask, -+ ARRAY_SIZE(data_len)) < 0) -+ return -EINVAL; -+ -+ if (inl_mask & 1) -+ ctx->adata.key_virt = ctx->key; -+ else -+ ctx->adata.key_dma = ctx->key_dma; -+ -+ if (inl_mask & 2) -+ ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; -+ else -+ ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; -+ -+ ctx->adata.key_inline = !!(inl_mask & 1); -+ ctx->cdata.key_inline = !!(inl_mask & 2); -+ -+ cnstr_shdsc_aead_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata, -+ ivsize, ctx->authsize, alg->caam.geniv, -+ is_rfc3686, nonce, ctx1_iv_off, true, -+ ctrlpriv->era); -+ -+ if (!alg->caam.geniv) -+ goto skip_givenc; -+ -+ /* aead_givencrypt shared descriptor */ -+ if (desc_inline_query(DESC_QI_AEAD_GIVENC_LEN + -+ (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), -+ DESC_JOB_IO_LEN, data_len, &inl_mask, -+ ARRAY_SIZE(data_len)) < 0) -+ return -EINVAL; -+ -+ if (inl_mask & 1) -+ ctx->adata.key_virt = ctx->key; -+ else -+ ctx->adata.key_dma = ctx->key_dma; -+ -+ if (inl_mask & 2) -+ ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; -+ else -+ ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; -+ -+ ctx->adata.key_inline = !!(inl_mask & 1); -+ ctx->cdata.key_inline = !!(inl_mask & 2); -+ -+ cnstr_shdsc_aead_givencap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, -+ ivsize, ctx->authsize, is_rfc3686, nonce, -+ ctx1_iv_off, true, ctrlpriv->era); -+ -+skip_givenc: -+ return 0; -+} -+ -+static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(authenc); -+ -+ ctx->authsize = authsize; -+ aead_set_sh_desc(authenc); -+ -+ return 0; -+} -+ -+static int aead_setkey(struct crypto_aead *aead, const u8 *key, -+ unsigned int keylen) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ struct device *jrdev = ctx->jrdev; -+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); -+ struct crypto_authenc_keys keys; -+ int ret = 0; -+ -+ if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) -+ goto badkey; -+ -+#ifdef DEBUG -+ dev_err(jrdev, "keylen %d enckeylen %d authkeylen %d\n", -+ keys.authkeylen + keys.enckeylen, keys.enckeylen, -+ keys.authkeylen); -+ print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -+#endif -+ -+ /* -+ * If DKP is supported, use it in the shared descriptor to generate -+ * the split key. -+ */ -+ if (ctrlpriv->era >= 6) { -+ ctx->adata.keylen = keys.authkeylen; -+ ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & -+ OP_ALG_ALGSEL_MASK); -+ -+ if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) -+ goto badkey; -+ -+ memcpy(ctx->key, keys.authkey, keys.authkeylen); -+ memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, -+ keys.enckeylen); -+ dma_sync_single_for_device(jrdev, ctx->key_dma, -+ ctx->adata.keylen_pad + -+ keys.enckeylen, ctx->dir); -+ goto skip_split_key; -+ } -+ -+ ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey, -+ keys.authkeylen, CAAM_MAX_KEY_SIZE - -+ keys.enckeylen); -+ if (ret) -+ goto badkey; -+ -+ /* postpend encryption key to auth split key */ -+ memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); -+ dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + -+ keys.enckeylen, ctx->dir); -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, -+ ctx->adata.keylen_pad + keys.enckeylen, 1); -+#endif -+ -+skip_split_key: -+ ctx->cdata.keylen = keys.enckeylen; -+ -+ ret = aead_set_sh_desc(aead); -+ if (ret) -+ goto badkey; -+ -+ /* Now update the driver contexts with the new shared descriptor */ -+ if (ctx->drv_ctx[ENCRYPT]) { -+ ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], -+ ctx->sh_desc_enc); -+ if (ret) { -+ dev_err(jrdev, "driver enc context update failed\n"); -+ goto badkey; -+ } -+ } -+ -+ if (ctx->drv_ctx[DECRYPT]) { -+ ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], -+ ctx->sh_desc_dec); -+ if (ret) { -+ dev_err(jrdev, "driver dec context update failed\n"); -+ goto badkey; -+ } -+ } -+ -+ return ret; -+badkey: -+ crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); -+ return -EINVAL; -+} -+ -+static int tls_set_sh_desc(struct crypto_aead *tls) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(tls); -+ unsigned int ivsize = crypto_aead_ivsize(tls); -+ unsigned int blocksize = crypto_aead_blocksize(tls); -+ unsigned int assoclen = 13; /* always 13 bytes for TLS */ -+ unsigned int data_len[2]; -+ u32 inl_mask; -+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); -+ -+ if (!ctx->cdata.keylen || !ctx->authsize) -+ return 0; -+ -+ /* -+ * TLS 1.0 encrypt shared descriptor -+ * Job Descriptor and Shared Descriptor -+ * must fit into the 64-word Descriptor h/w Buffer -+ */ -+ data_len[0] = ctx->adata.keylen_pad; -+ data_len[1] = ctx->cdata.keylen; -+ -+ if (desc_inline_query(DESC_TLS10_ENC_LEN, DESC_JOB_IO_LEN, data_len, -+ &inl_mask, ARRAY_SIZE(data_len)) < 0) -+ return -EINVAL; -+ -+ if (inl_mask & 1) -+ ctx->adata.key_virt = ctx->key; -+ else -+ ctx->adata.key_dma = ctx->key_dma; -+ -+ if (inl_mask & 2) -+ ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; -+ else -+ ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; -+ -+ ctx->adata.key_inline = !!(inl_mask & 1); -+ ctx->cdata.key_inline = !!(inl_mask & 2); -+ -+ cnstr_shdsc_tls_encap(ctx->sh_desc_enc, &ctx->cdata, &ctx->adata, -+ assoclen, ivsize, ctx->authsize, blocksize, -+ ctrlpriv->era); -+ -+ /* -+ * TLS 1.0 decrypt shared descriptor -+ * Keys do not fit inline, regardless of algorithms used -+ */ -+ ctx->adata.key_inline = false; -+ ctx->adata.key_dma = ctx->key_dma; -+ ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; -+ -+ cnstr_shdsc_tls_decap(ctx->sh_desc_dec, &ctx->cdata, &ctx->adata, -+ assoclen, ivsize, ctx->authsize, blocksize, -+ ctrlpriv->era); -+ -+ return 0; -+} -+ -+static int tls_setauthsize(struct crypto_aead *tls, unsigned int authsize) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(tls); -+ -+ ctx->authsize = authsize; -+ tls_set_sh_desc(tls); -+ -+ return 0; -+} -+ -+static int tls_setkey(struct crypto_aead *tls, const u8 *key, -+ unsigned int keylen) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(tls); -+ struct device *jrdev = ctx->jrdev; -+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); -+ struct crypto_authenc_keys keys; -+ int ret = 0; -+ -+ if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) -+ goto badkey; -+ -+#ifdef DEBUG -+ dev_err(jrdev, "keylen %d enckeylen %d authkeylen %d\n", -+ keys.authkeylen + keys.enckeylen, keys.enckeylen, -+ keys.authkeylen); -+ print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -+#endif -+ -+ /* -+ * If DKP is supported, use it in the shared descriptor to generate -+ * the split key. -+ */ -+ if (ctrlpriv->era >= 6) { -+ ctx->adata.keylen = keys.authkeylen; -+ ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & -+ OP_ALG_ALGSEL_MASK); -+ -+ if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) -+ goto badkey; -+ -+ memcpy(ctx->key, keys.authkey, keys.authkeylen); -+ memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, -+ keys.enckeylen); -+ dma_sync_single_for_device(jrdev, ctx->key_dma, -+ ctx->adata.keylen_pad + -+ keys.enckeylen, ctx->dir); -+ goto skip_split_key; -+ } -+ -+ ret = gen_split_key(jrdev, ctx->key, &ctx->adata, keys.authkey, -+ keys.authkeylen, CAAM_MAX_KEY_SIZE - -+ keys.enckeylen); -+ if (ret) -+ goto badkey; -+ -+ /* postpend encryption key to auth split key */ -+ memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); -+ dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + -+ keys.enckeylen, ctx->dir); -+ -+#ifdef DEBUG -+ dev_err(jrdev, "split keylen %d split keylen padded %d\n", -+ ctx->adata.keylen, ctx->adata.keylen_pad); -+ print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, -+ ctx->adata.keylen_pad + keys.enckeylen, 1); -+#endif -+ -+skip_split_key: -+ ctx->cdata.keylen = keys.enckeylen; -+ -+ ret = tls_set_sh_desc(tls); -+ if (ret) -+ goto badkey; -+ -+ /* Now update the driver contexts with the new shared descriptor */ -+ if (ctx->drv_ctx[ENCRYPT]) { -+ ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], -+ ctx->sh_desc_enc); -+ if (ret) { -+ dev_err(jrdev, "driver enc context update failed\n"); -+ goto badkey; -+ } -+ } -+ -+ if (ctx->drv_ctx[DECRYPT]) { -+ ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], -+ ctx->sh_desc_dec); -+ if (ret) { -+ dev_err(jrdev, "driver dec context update failed\n"); -+ goto badkey; -+ } -+ } -+ -+ return ret; -+badkey: -+ crypto_aead_set_flags(tls, CRYPTO_TFM_RES_BAD_KEY_LEN); -+ return -EINVAL; -+} -+ -+static int gcm_set_sh_desc(struct crypto_aead *aead) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ unsigned int ivsize = crypto_aead_ivsize(aead); -+ int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - -+ ctx->cdata.keylen; -+ -+ if (!ctx->cdata.keylen || !ctx->authsize) -+ return 0; -+ -+ /* -+ * Job Descriptor and Shared Descriptor -+ * must fit into the 64-word Descriptor h/w Buffer -+ */ -+ if (rem_bytes >= DESC_QI_GCM_ENC_LEN) { -+ ctx->cdata.key_inline = true; -+ ctx->cdata.key_virt = ctx->key; -+ } else { -+ ctx->cdata.key_inline = false; -+ ctx->cdata.key_dma = ctx->key_dma; -+ } -+ -+ cnstr_shdsc_gcm_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, -+ ctx->authsize, true); -+ -+ /* -+ * Job Descriptor and Shared Descriptor -+ * must fit into the 64-word Descriptor h/w Buffer -+ */ -+ if (rem_bytes >= DESC_QI_GCM_DEC_LEN) { -+ ctx->cdata.key_inline = true; -+ ctx->cdata.key_virt = ctx->key; -+ } else { -+ ctx->cdata.key_inline = false; -+ ctx->cdata.key_dma = ctx->key_dma; -+ } -+ -+ cnstr_shdsc_gcm_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, -+ ctx->authsize, true); -+ -+ return 0; -+} -+ -+static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(authenc); -+ -+ ctx->authsize = authsize; -+ gcm_set_sh_desc(authenc); -+ -+ return 0; -+} -+ -+static int gcm_setkey(struct crypto_aead *aead, -+ const u8 *key, unsigned int keylen) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ struct device *jrdev = ctx->jrdev; -+ int ret; -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -+#endif -+ -+ memcpy(ctx->key, key, keylen); -+ dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir); -+ ctx->cdata.keylen = keylen; -+ -+ ret = gcm_set_sh_desc(aead); -+ if (ret) -+ return ret; -+ -+ /* Now update the driver contexts with the new shared descriptor */ -+ if (ctx->drv_ctx[ENCRYPT]) { -+ ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], -+ ctx->sh_desc_enc); -+ if (ret) { -+ dev_err(jrdev, "driver enc context update failed\n"); -+ return ret; -+ } -+ } -+ -+ if (ctx->drv_ctx[DECRYPT]) { -+ ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], -+ ctx->sh_desc_dec); -+ if (ret) { -+ dev_err(jrdev, "driver dec context update failed\n"); -+ return ret; -+ } -+ } -+ -+ return 0; -+} -+ -+static int rfc4106_set_sh_desc(struct crypto_aead *aead) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ unsigned int ivsize = crypto_aead_ivsize(aead); -+ int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - -+ ctx->cdata.keylen; -+ -+ if (!ctx->cdata.keylen || !ctx->authsize) -+ return 0; -+ -+ ctx->cdata.key_virt = ctx->key; -+ -+ /* -+ * Job Descriptor and Shared Descriptor -+ * must fit into the 64-word Descriptor h/w Buffer -+ */ -+ if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) { -+ ctx->cdata.key_inline = true; -+ } else { -+ ctx->cdata.key_inline = false; -+ ctx->cdata.key_dma = ctx->key_dma; -+ } -+ -+ cnstr_shdsc_rfc4106_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, -+ ctx->authsize, true); -+ -+ /* -+ * Job Descriptor and Shared Descriptor -+ * must fit into the 64-word Descriptor h/w Buffer -+ */ -+ if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) { -+ ctx->cdata.key_inline = true; -+ } else { -+ ctx->cdata.key_inline = false; -+ ctx->cdata.key_dma = ctx->key_dma; -+ } -+ -+ cnstr_shdsc_rfc4106_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, -+ ctx->authsize, true); -+ -+ return 0; -+} -+ -+static int rfc4106_setauthsize(struct crypto_aead *authenc, -+ unsigned int authsize) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(authenc); -+ -+ ctx->authsize = authsize; -+ rfc4106_set_sh_desc(authenc); -+ -+ return 0; -+} -+ -+static int rfc4106_setkey(struct crypto_aead *aead, -+ const u8 *key, unsigned int keylen) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ struct device *jrdev = ctx->jrdev; -+ int ret; -+ -+ if (keylen < 4) -+ return -EINVAL; -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -+#endif -+ -+ memcpy(ctx->key, key, keylen); -+ /* -+ * The last four bytes of the key material are used as the salt value -+ * in the nonce. Update the AES key length. -+ */ -+ ctx->cdata.keylen = keylen - 4; -+ dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, -+ ctx->dir); -+ -+ ret = rfc4106_set_sh_desc(aead); -+ if (ret) -+ return ret; -+ -+ /* Now update the driver contexts with the new shared descriptor */ -+ if (ctx->drv_ctx[ENCRYPT]) { -+ ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], -+ ctx->sh_desc_enc); -+ if (ret) { -+ dev_err(jrdev, "driver enc context update failed\n"); -+ return ret; -+ } -+ } -+ -+ if (ctx->drv_ctx[DECRYPT]) { -+ ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], -+ ctx->sh_desc_dec); -+ if (ret) { -+ dev_err(jrdev, "driver dec context update failed\n"); -+ return ret; -+ } -+ } -+ -+ return 0; -+} -+ -+static int rfc4543_set_sh_desc(struct crypto_aead *aead) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ unsigned int ivsize = crypto_aead_ivsize(aead); -+ int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - -+ ctx->cdata.keylen; -+ -+ if (!ctx->cdata.keylen || !ctx->authsize) -+ return 0; -+ -+ ctx->cdata.key_virt = ctx->key; -+ -+ /* -+ * Job Descriptor and Shared Descriptor -+ * must fit into the 64-word Descriptor h/w Buffer -+ */ -+ if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) { -+ ctx->cdata.key_inline = true; -+ } else { -+ ctx->cdata.key_inline = false; -+ ctx->cdata.key_dma = ctx->key_dma; -+ } -+ -+ cnstr_shdsc_rfc4543_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, -+ ctx->authsize, true); -+ -+ /* -+ * Job Descriptor and Shared Descriptor -+ * must fit into the 64-word Descriptor h/w Buffer -+ */ -+ if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) { -+ ctx->cdata.key_inline = true; -+ } else { -+ ctx->cdata.key_inline = false; -+ ctx->cdata.key_dma = ctx->key_dma; -+ } -+ -+ cnstr_shdsc_rfc4543_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, -+ ctx->authsize, true); -+ -+ return 0; -+} -+ -+static int rfc4543_setauthsize(struct crypto_aead *authenc, -+ unsigned int authsize) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(authenc); -+ -+ ctx->authsize = authsize; -+ rfc4543_set_sh_desc(authenc); -+ -+ return 0; -+} -+ -+static int rfc4543_setkey(struct crypto_aead *aead, -+ const u8 *key, unsigned int keylen) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ struct device *jrdev = ctx->jrdev; -+ int ret; -+ -+ if (keylen < 4) -+ return -EINVAL; -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -+#endif -+ -+ memcpy(ctx->key, key, keylen); -+ /* -+ * The last four bytes of the key material are used as the salt value -+ * in the nonce. Update the AES key length. -+ */ -+ ctx->cdata.keylen = keylen - 4; -+ dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, -+ ctx->dir); -+ -+ ret = rfc4543_set_sh_desc(aead); -+ if (ret) -+ return ret; -+ -+ /* Now update the driver contexts with the new shared descriptor */ -+ if (ctx->drv_ctx[ENCRYPT]) { -+ ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], -+ ctx->sh_desc_enc); -+ if (ret) { -+ dev_err(jrdev, "driver enc context update failed\n"); -+ return ret; -+ } -+ } -+ -+ if (ctx->drv_ctx[DECRYPT]) { -+ ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], -+ ctx->sh_desc_dec); -+ if (ret) { -+ dev_err(jrdev, "driver dec context update failed\n"); -+ return ret; -+ } -+ } -+ -+ return 0; -+} -+ -+static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, -+ const u8 *key, unsigned int keylen) -+{ -+ struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); -+ struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher); -+ const char *alg_name = crypto_tfm_alg_name(tfm); -+ struct device *jrdev = ctx->jrdev; -+ unsigned int ivsize = crypto_ablkcipher_ivsize(ablkcipher); -+ u32 ctx1_iv_off = 0; -+ const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == -+ OP_ALG_AAI_CTR_MOD128); -+ const bool is_rfc3686 = (ctr_mode && strstr(alg_name, "rfc3686")); -+ int ret = 0; -+ -+ memcpy(ctx->key, key, keylen); -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -+#endif -+ /* -+ * AES-CTR needs to load IV in CONTEXT1 reg -+ * at an offset of 128bits (16bytes) -+ * CONTEXT1[255:128] = IV -+ */ -+ if (ctr_mode) -+ ctx1_iv_off = 16; -+ -+ /* -+ * RFC3686 specific: -+ * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} -+ * | *key = {KEY, NONCE} -+ */ -+ if (is_rfc3686) { -+ ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; -+ keylen -= CTR_RFC3686_NONCE_SIZE; -+ } -+ -+ dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); -+ ctx->cdata.keylen = keylen; -+ ctx->cdata.key_virt = ctx->key; -+ ctx->cdata.key_inline = true; -+ -+ /* ablkcipher encrypt, decrypt, givencrypt shared descriptors */ -+ cnstr_shdsc_ablkcipher_encap(ctx->sh_desc_enc, &ctx->cdata, ivsize, -+ is_rfc3686, ctx1_iv_off); -+ cnstr_shdsc_ablkcipher_decap(ctx->sh_desc_dec, &ctx->cdata, ivsize, -+ is_rfc3686, ctx1_iv_off); -+ cnstr_shdsc_ablkcipher_givencap(ctx->sh_desc_givenc, &ctx->cdata, -+ ivsize, is_rfc3686, ctx1_iv_off); -+ -+ /* Now update the driver contexts with the new shared descriptor */ -+ if (ctx->drv_ctx[ENCRYPT]) { -+ ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], -+ ctx->sh_desc_enc); -+ if (ret) { -+ dev_err(jrdev, "driver enc context update failed\n"); -+ goto badkey; -+ } -+ } -+ -+ if (ctx->drv_ctx[DECRYPT]) { -+ ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], -+ ctx->sh_desc_dec); -+ if (ret) { -+ dev_err(jrdev, "driver dec context update failed\n"); -+ goto badkey; -+ } -+ } -+ -+ if (ctx->drv_ctx[GIVENCRYPT]) { -+ ret = caam_drv_ctx_update(ctx->drv_ctx[GIVENCRYPT], -+ ctx->sh_desc_givenc); -+ if (ret) { -+ dev_err(jrdev, "driver givenc context update failed\n"); -+ goto badkey; -+ } -+ } -+ -+ return ret; -+badkey: -+ crypto_ablkcipher_set_flags(ablkcipher, CRYPTO_TFM_RES_BAD_KEY_LEN); -+ return -EINVAL; -+} -+ -+static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, -+ const u8 *key, unsigned int keylen) -+{ -+ struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); -+ struct device *jrdev = ctx->jrdev; -+ int ret = 0; -+ -+ if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { -+ crypto_ablkcipher_set_flags(ablkcipher, -+ CRYPTO_TFM_RES_BAD_KEY_LEN); -+ dev_err(jrdev, "key size mismatch\n"); -+ return -EINVAL; -+ } -+ -+ memcpy(ctx->key, key, keylen); -+ dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); -+ ctx->cdata.keylen = keylen; -+ ctx->cdata.key_virt = ctx->key; -+ ctx->cdata.key_inline = true; -+ -+ /* xts ablkcipher encrypt, decrypt shared descriptors */ -+ cnstr_shdsc_xts_ablkcipher_encap(ctx->sh_desc_enc, &ctx->cdata); -+ cnstr_shdsc_xts_ablkcipher_decap(ctx->sh_desc_dec, &ctx->cdata); -+ -+ /* Now update the driver contexts with the new shared descriptor */ -+ if (ctx->drv_ctx[ENCRYPT]) { -+ ret = caam_drv_ctx_update(ctx->drv_ctx[ENCRYPT], -+ ctx->sh_desc_enc); -+ if (ret) { -+ dev_err(jrdev, "driver enc context update failed\n"); -+ goto badkey; -+ } -+ } -+ -+ if (ctx->drv_ctx[DECRYPT]) { -+ ret = caam_drv_ctx_update(ctx->drv_ctx[DECRYPT], -+ ctx->sh_desc_dec); -+ if (ret) { -+ dev_err(jrdev, "driver dec context update failed\n"); -+ goto badkey; -+ } -+ } -+ -+ return ret; -+badkey: -+ crypto_ablkcipher_set_flags(ablkcipher, CRYPTO_TFM_RES_BAD_KEY_LEN); -+ return 0; -+} -+ -+/* -+ * aead_edesc - s/w-extended aead descriptor -+ * @src_nents: number of segments in input scatterlist -+ * @dst_nents: number of segments in output scatterlist -+ * @iv_dma: dma address of iv for checking continuity and link table -+ * @qm_sg_bytes: length of dma mapped h/w link table -+ * @qm_sg_dma: bus physical mapped address of h/w link table -+ * @assoclen: associated data length, in CAAM endianness -+ * @assoclen_dma: bus physical mapped address of req->assoclen -+ * @drv_req: driver-specific request structure -+ * @sgt: the h/w link table -+ */ -+struct aead_edesc { -+ int src_nents; -+ int dst_nents; -+ dma_addr_t iv_dma; -+ int qm_sg_bytes; -+ dma_addr_t qm_sg_dma; -+ unsigned int assoclen; -+ dma_addr_t assoclen_dma; -+ struct caam_drv_req drv_req; -+#define CAAM_QI_MAX_AEAD_SG \ -+ ((CAAM_QI_MEMCACHE_SIZE - offsetof(struct aead_edesc, sgt)) / \ -+ sizeof(struct qm_sg_entry)) -+ struct qm_sg_entry sgt[0]; -+}; -+ -+/* -+ * tls_edesc - s/w-extended tls descriptor -+ * @src_nents: number of segments in input scatterlist -+ * @dst_nents: number of segments in output scatterlist -+ * @iv_dma: dma address of iv for checking continuity and link table -+ * @qm_sg_bytes: length of dma mapped h/w link table -+ * @tmp: array of scatterlists used by 'scatterwalk_ffwd' -+ * @qm_sg_dma: bus physical mapped address of h/w link table -+ * @drv_req: driver-specific request structure -+ * @sgt: the h/w link table -+ */ -+struct tls_edesc { -+ int src_nents; -+ int dst_nents; -+ dma_addr_t iv_dma; -+ int qm_sg_bytes; -+ dma_addr_t qm_sg_dma; -+ struct scatterlist tmp[2]; -+ struct scatterlist *dst; -+ struct caam_drv_req drv_req; -+ struct qm_sg_entry sgt[0]; -+}; -+ -+/* -+ * ablkcipher_edesc - s/w-extended ablkcipher descriptor -+ * @src_nents: number of segments in input scatterlist -+ * @dst_nents: number of segments in output scatterlist -+ * @iv_dma: dma address of iv for checking continuity and link table -+ * @qm_sg_bytes: length of dma mapped h/w link table -+ * @qm_sg_dma: bus physical mapped address of h/w link table -+ * @drv_req: driver-specific request structure -+ * @sgt: the h/w link table -+ */ -+struct ablkcipher_edesc { -+ int src_nents; -+ int dst_nents; -+ dma_addr_t iv_dma; -+ int qm_sg_bytes; -+ dma_addr_t qm_sg_dma; -+ struct caam_drv_req drv_req; -+#define CAAM_QI_MAX_ABLKCIPHER_SG \ -+ ((CAAM_QI_MEMCACHE_SIZE - offsetof(struct ablkcipher_edesc, sgt)) / \ -+ sizeof(struct qm_sg_entry)) -+ struct qm_sg_entry sgt[0]; -+}; -+ -+static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx, -+ enum optype type) -+{ -+ /* -+ * This function is called on the fast path with values of 'type' -+ * known at compile time. Invalid arguments are not expected and -+ * thus no checks are made. -+ */ -+ struct caam_drv_ctx *drv_ctx = ctx->drv_ctx[type]; -+ u32 *desc; -+ -+ if (unlikely(!drv_ctx)) { -+ spin_lock(&ctx->lock); -+ -+ /* Read again to check if some other core init drv_ctx */ -+ drv_ctx = ctx->drv_ctx[type]; -+ if (!drv_ctx) { -+ int cpu; -+ -+ if (type == ENCRYPT) -+ desc = ctx->sh_desc_enc; -+ else if (type == DECRYPT) -+ desc = ctx->sh_desc_dec; -+ else /* (type == GIVENCRYPT) */ -+ desc = ctx->sh_desc_givenc; -+ -+ cpu = smp_processor_id(); -+ drv_ctx = caam_drv_ctx_init(ctx->qidev, &cpu, desc); -+ if (likely(!IS_ERR_OR_NULL(drv_ctx))) -+ drv_ctx->op_type = type; -+ -+ ctx->drv_ctx[type] = drv_ctx; -+ } -+ -+ spin_unlock(&ctx->lock); -+ } -+ -+ return drv_ctx; -+} -+ -+static void caam_unmap(struct device *dev, struct scatterlist *src, -+ struct scatterlist *dst, int src_nents, -+ int dst_nents, dma_addr_t iv_dma, int ivsize, -+ enum optype op_type, dma_addr_t qm_sg_dma, -+ int qm_sg_bytes) -+{ -+ if (dst != src) { -+ if (src_nents) -+ dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); -+ dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); -+ } else { -+ dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); -+ } -+ -+ if (iv_dma) -+ dma_unmap_single(dev, iv_dma, ivsize, -+ op_type == GIVENCRYPT ? DMA_FROM_DEVICE : -+ DMA_TO_DEVICE); -+ if (qm_sg_bytes) -+ dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); -+} -+ -+static void aead_unmap(struct device *dev, -+ struct aead_edesc *edesc, -+ struct aead_request *req) -+{ -+ struct crypto_aead *aead = crypto_aead_reqtfm(req); -+ int ivsize = crypto_aead_ivsize(aead); -+ -+ caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, -+ edesc->iv_dma, ivsize, edesc->drv_req.drv_ctx->op_type, -+ edesc->qm_sg_dma, edesc->qm_sg_bytes); -+ dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); -+} -+ -+static void tls_unmap(struct device *dev, -+ struct tls_edesc *edesc, -+ struct aead_request *req) -+{ -+ struct crypto_aead *aead = crypto_aead_reqtfm(req); -+ int ivsize = crypto_aead_ivsize(aead); -+ -+ caam_unmap(dev, req->src, edesc->dst, edesc->src_nents, -+ edesc->dst_nents, edesc->iv_dma, ivsize, -+ edesc->drv_req.drv_ctx->op_type, edesc->qm_sg_dma, -+ edesc->qm_sg_bytes); -+} -+ -+static void ablkcipher_unmap(struct device *dev, -+ struct ablkcipher_edesc *edesc, -+ struct ablkcipher_request *req) -+{ -+ struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); -+ int ivsize = crypto_ablkcipher_ivsize(ablkcipher); -+ -+ caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, -+ edesc->iv_dma, ivsize, edesc->drv_req.drv_ctx->op_type, -+ edesc->qm_sg_dma, edesc->qm_sg_bytes); -+} -+ -+static void aead_done(struct caam_drv_req *drv_req, u32 status) -+{ -+ struct device *qidev; -+ struct aead_edesc *edesc; -+ struct aead_request *aead_req = drv_req->app_ctx; -+ struct crypto_aead *aead = crypto_aead_reqtfm(aead_req); -+ struct caam_ctx *caam_ctx = crypto_aead_ctx(aead); -+ int ecode = 0; -+ -+ qidev = caam_ctx->qidev; -+ -+ if (unlikely(status)) { -+ u32 ssrc = status & JRSTA_SSRC_MASK; -+ u8 err_id = status & JRSTA_CCBERR_ERRID_MASK; -+ -+ caam_jr_strstatus(qidev, status); -+ /* -+ * verify hw auth check passed else return -EBADMSG -+ */ -+ if (ssrc == JRSTA_SSRC_CCB_ERROR && -+ err_id == JRSTA_CCBERR_ERRID_ICVCHK) -+ ecode = -EBADMSG; -+ else -+ ecode = -EIO; -+ } -+ -+ edesc = container_of(drv_req, typeof(*edesc), drv_req); -+ aead_unmap(qidev, edesc, aead_req); -+ -+ aead_request_complete(aead_req, ecode); -+ qi_cache_free(edesc); -+} -+ -+/* -+ * allocate and map the aead extended descriptor -+ */ -+static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, -+ bool encrypt) -+{ -+ struct crypto_aead *aead = crypto_aead_reqtfm(req); -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), -+ typeof(*alg), aead); -+ struct device *qidev = ctx->qidev; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; -+ int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; -+ struct aead_edesc *edesc; -+ dma_addr_t qm_sg_dma, iv_dma = 0; -+ int ivsize = 0; -+ unsigned int authsize = ctx->authsize; -+ int qm_sg_index = 0, qm_sg_ents = 0, qm_sg_bytes; -+ int in_len, out_len; -+ struct qm_sg_entry *sg_table, *fd_sgt; -+ struct caam_drv_ctx *drv_ctx; -+ enum optype op_type = encrypt ? ENCRYPT : DECRYPT; -+ -+ drv_ctx = get_drv_ctx(ctx, op_type); -+ if (unlikely(IS_ERR_OR_NULL(drv_ctx))) -+ return (struct aead_edesc *)drv_ctx; -+ -+ /* allocate space for base edesc and hw desc commands, link tables */ -+ edesc = qi_cache_alloc(GFP_DMA | flags); -+ if (unlikely(!edesc)) { -+ dev_err(qidev, "could not allocate extended descriptor\n"); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ if (likely(req->src == req->dst)) { -+ src_nents = sg_nents_for_len(req->src, req->assoclen + -+ req->cryptlen + -+ (encrypt ? authsize : 0)); -+ if (unlikely(src_nents < 0)) { -+ dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", -+ req->assoclen + req->cryptlen + -+ (encrypt ? authsize : 0)); -+ qi_cache_free(edesc); -+ return ERR_PTR(src_nents); -+ } -+ -+ mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, -+ DMA_BIDIRECTIONAL); -+ if (unlikely(!mapped_src_nents)) { -+ dev_err(qidev, "unable to map source\n"); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ } else { -+ src_nents = sg_nents_for_len(req->src, req->assoclen + -+ req->cryptlen); -+ if (unlikely(src_nents < 0)) { -+ dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", -+ req->assoclen + req->cryptlen); -+ qi_cache_free(edesc); -+ return ERR_PTR(src_nents); -+ } -+ -+ dst_nents = sg_nents_for_len(req->dst, req->assoclen + -+ req->cryptlen + -+ (encrypt ? authsize : -+ (-authsize))); -+ if (unlikely(dst_nents < 0)) { -+ dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", -+ req->assoclen + req->cryptlen + -+ (encrypt ? authsize : (-authsize))); -+ qi_cache_free(edesc); -+ return ERR_PTR(dst_nents); -+ } -+ -+ if (src_nents) { -+ mapped_src_nents = dma_map_sg(qidev, req->src, -+ src_nents, DMA_TO_DEVICE); -+ if (unlikely(!mapped_src_nents)) { -+ dev_err(qidev, "unable to map source\n"); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ } else { -+ mapped_src_nents = 0; -+ } -+ -+ mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, -+ DMA_FROM_DEVICE); -+ if (unlikely(!mapped_dst_nents)) { -+ dev_err(qidev, "unable to map destination\n"); -+ dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ } -+ -+ if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) { -+ ivsize = crypto_aead_ivsize(aead); -+ iv_dma = dma_map_single(qidev, req->iv, ivsize, DMA_TO_DEVICE); -+ if (dma_mapping_error(qidev, iv_dma)) { -+ dev_err(qidev, "unable to map IV\n"); -+ caam_unmap(qidev, req->src, req->dst, src_nents, -+ dst_nents, 0, 0, op_type, 0, 0); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ } -+ -+ /* -+ * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. -+ * Input is not contiguous. -+ */ -+ qm_sg_ents = 1 + !!ivsize + mapped_src_nents + -+ (mapped_dst_nents > 1 ? mapped_dst_nents : 0); -+ if (unlikely(qm_sg_ents > CAAM_QI_MAX_AEAD_SG)) { -+ dev_err(qidev, "Insufficient S/G entries: %d > %lu\n", -+ qm_sg_ents, CAAM_QI_MAX_AEAD_SG); -+ caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, -+ iv_dma, ivsize, op_type, 0, 0); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ sg_table = &edesc->sgt[0]; -+ qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); -+ -+ edesc->src_nents = src_nents; -+ edesc->dst_nents = dst_nents; -+ edesc->iv_dma = iv_dma; -+ edesc->drv_req.app_ctx = req; -+ edesc->drv_req.cbk = aead_done; -+ edesc->drv_req.drv_ctx = drv_ctx; -+ -+ edesc->assoclen = cpu_to_caam32(req->assoclen); -+ edesc->assoclen_dma = dma_map_single(qidev, &edesc->assoclen, 4, -+ DMA_TO_DEVICE); -+ if (dma_mapping_error(qidev, edesc->assoclen_dma)) { -+ dev_err(qidev, "unable to map assoclen\n"); -+ caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, -+ iv_dma, ivsize, op_type, 0, 0); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); -+ qm_sg_index++; -+ if (ivsize) { -+ dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); -+ qm_sg_index++; -+ } -+ sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0); -+ qm_sg_index += mapped_src_nents; -+ -+ if (mapped_dst_nents > 1) -+ sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + -+ qm_sg_index, 0); -+ -+ qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); -+ if (dma_mapping_error(qidev, qm_sg_dma)) { -+ dev_err(qidev, "unable to map S/G table\n"); -+ dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); -+ caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, -+ iv_dma, ivsize, op_type, 0, 0); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ edesc->qm_sg_dma = qm_sg_dma; -+ edesc->qm_sg_bytes = qm_sg_bytes; -+ -+ out_len = req->assoclen + req->cryptlen + -+ (encrypt ? ctx->authsize : (-ctx->authsize)); -+ in_len = 4 + ivsize + req->assoclen + req->cryptlen; -+ -+ fd_sgt = &edesc->drv_req.fd_sgt[0]; -+ dma_to_qm_sg_one_last_ext(&fd_sgt[1], qm_sg_dma, in_len, 0); -+ -+ if (req->dst == req->src) { -+ if (mapped_src_nents == 1) -+ dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->src), -+ out_len, 0); -+ else -+ dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + -+ (1 + !!ivsize) * sizeof(*sg_table), -+ out_len, 0); -+ } else if (mapped_dst_nents == 1) { -+ dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len, -+ 0); -+ } else { -+ dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + sizeof(*sg_table) * -+ qm_sg_index, out_len, 0); -+ } -+ -+ return edesc; -+} -+ -+static inline int aead_crypt(struct aead_request *req, bool encrypt) -+{ -+ struct aead_edesc *edesc; -+ struct crypto_aead *aead = crypto_aead_reqtfm(req); -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ int ret; -+ -+ if (unlikely(caam_congested)) -+ return -EAGAIN; -+ -+ /* allocate extended descriptor */ -+ edesc = aead_edesc_alloc(req, encrypt); -+ if (IS_ERR_OR_NULL(edesc)) -+ return PTR_ERR(edesc); -+ -+ /* Create and submit job descriptor */ -+ ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); -+ if (!ret) { -+ ret = -EINPROGRESS; -+ } else { -+ aead_unmap(ctx->qidev, edesc, req); -+ qi_cache_free(edesc); -+ } -+ -+ return ret; -+} -+ -+static int aead_encrypt(struct aead_request *req) -+{ -+ return aead_crypt(req, true); -+} -+ -+static int aead_decrypt(struct aead_request *req) -+{ -+ return aead_crypt(req, false); -+} -+ -+static int ipsec_gcm_encrypt(struct aead_request *req) -+{ -+ if (req->assoclen < 8) -+ return -EINVAL; -+ -+ return aead_crypt(req, true); -+} -+ -+static int ipsec_gcm_decrypt(struct aead_request *req) -+{ -+ if (req->assoclen < 8) -+ return -EINVAL; -+ -+ return aead_crypt(req, false); -+} -+ -+static void tls_done(struct caam_drv_req *drv_req, u32 status) -+{ -+ struct device *qidev; -+ struct tls_edesc *edesc; -+ struct aead_request *aead_req = drv_req->app_ctx; -+ struct crypto_aead *aead = crypto_aead_reqtfm(aead_req); -+ struct caam_ctx *caam_ctx = crypto_aead_ctx(aead); -+ int ecode = 0; -+ -+ qidev = caam_ctx->qidev; -+ -+ if (unlikely(status)) { -+ caam_jr_strstatus(qidev, status); -+ ecode = -EIO; -+ } -+ -+ edesc = container_of(drv_req, typeof(*edesc), drv_req); -+ tls_unmap(qidev, edesc, aead_req); -+ -+ aead_request_complete(aead_req, ecode); -+ qi_cache_free(edesc); -+} -+ -+/* -+ * allocate and map the tls extended descriptor -+ */ -+static struct tls_edesc *tls_edesc_alloc(struct aead_request *req, bool encrypt) -+{ -+ struct crypto_aead *aead = crypto_aead_reqtfm(req); -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ unsigned int blocksize = crypto_aead_blocksize(aead); -+ unsigned int padsize, authsize; -+ struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), -+ typeof(*alg), aead); -+ struct device *qidev = ctx->qidev; -+ gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | -+ CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; -+ int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; -+ struct tls_edesc *edesc; -+ dma_addr_t qm_sg_dma, iv_dma = 0; -+ int ivsize = 0; -+ int qm_sg_index, qm_sg_ents = 0, qm_sg_bytes; -+ int in_len, out_len; -+ struct qm_sg_entry *sg_table, *fd_sgt; -+ struct caam_drv_ctx *drv_ctx; -+ enum optype op_type = encrypt ? ENCRYPT : DECRYPT; -+ struct scatterlist *dst; -+ -+ if (encrypt) { -+ padsize = blocksize - ((req->cryptlen + ctx->authsize) % -+ blocksize); -+ authsize = ctx->authsize + padsize; -+ } else { -+ authsize = ctx->authsize; -+ } -+ -+ drv_ctx = get_drv_ctx(ctx, op_type); -+ if (unlikely(IS_ERR_OR_NULL(drv_ctx))) -+ return (struct tls_edesc *)drv_ctx; -+ -+ /* allocate space for base edesc and hw desc commands, link tables */ -+ edesc = qi_cache_alloc(GFP_DMA | flags); -+ if (unlikely(!edesc)) { -+ dev_err(qidev, "could not allocate extended descriptor\n"); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ if (likely(req->src == req->dst)) { -+ src_nents = sg_nents_for_len(req->src, req->assoclen + -+ req->cryptlen + -+ (encrypt ? authsize : 0)); -+ if (unlikely(src_nents < 0)) { -+ dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", -+ req->assoclen + req->cryptlen + -+ (encrypt ? authsize : 0)); -+ qi_cache_free(edesc); -+ return ERR_PTR(src_nents); -+ } -+ -+ mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, -+ DMA_BIDIRECTIONAL); -+ if (unlikely(!mapped_src_nents)) { -+ dev_err(qidev, "unable to map source\n"); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ dst = req->dst; -+ } else { -+ src_nents = sg_nents_for_len(req->src, req->assoclen + -+ req->cryptlen); -+ if (unlikely(src_nents < 0)) { -+ dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", -+ req->assoclen + req->cryptlen); -+ qi_cache_free(edesc); -+ return ERR_PTR(src_nents); -+ } -+ -+ dst = scatterwalk_ffwd(edesc->tmp, req->dst, req->assoclen); -+ dst_nents = sg_nents_for_len(dst, req->cryptlen + -+ (encrypt ? authsize : 0)); -+ if (unlikely(dst_nents < 0)) { -+ dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", -+ req->cryptlen + -+ (encrypt ? authsize : 0)); -+ qi_cache_free(edesc); -+ return ERR_PTR(dst_nents); -+ } -+ -+ if (src_nents) { -+ mapped_src_nents = dma_map_sg(qidev, req->src, -+ src_nents, DMA_TO_DEVICE); -+ if (unlikely(!mapped_src_nents)) { -+ dev_err(qidev, "unable to map source\n"); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ } else { -+ mapped_src_nents = 0; -+ } -+ -+ mapped_dst_nents = dma_map_sg(qidev, dst, dst_nents, -+ DMA_FROM_DEVICE); -+ if (unlikely(!mapped_dst_nents)) { -+ dev_err(qidev, "unable to map destination\n"); -+ dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ } -+ -+ ivsize = crypto_aead_ivsize(aead); -+ iv_dma = dma_map_single(qidev, req->iv, ivsize, DMA_TO_DEVICE); -+ if (dma_mapping_error(qidev, iv_dma)) { -+ dev_err(qidev, "unable to map IV\n"); -+ caam_unmap(qidev, req->src, dst, src_nents, dst_nents, 0, 0, -+ op_type, 0, 0); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ /* -+ * Create S/G table: IV, src, dst. -+ * Input is not contiguous. -+ */ -+ qm_sg_ents = 1 + mapped_src_nents + -+ (mapped_dst_nents > 1 ? mapped_dst_nents : 0); -+ sg_table = &edesc->sgt[0]; -+ qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); -+ -+ edesc->src_nents = src_nents; -+ edesc->dst_nents = dst_nents; -+ edesc->dst = dst; -+ edesc->iv_dma = iv_dma; -+ edesc->drv_req.app_ctx = req; -+ edesc->drv_req.cbk = tls_done; -+ edesc->drv_req.drv_ctx = drv_ctx; -+ -+ dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); -+ qm_sg_index = 1; -+ -+ sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0); -+ qm_sg_index += mapped_src_nents; -+ -+ if (mapped_dst_nents > 1) -+ sg_to_qm_sg_last(dst, mapped_dst_nents, sg_table + -+ qm_sg_index, 0); -+ -+ qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); -+ if (dma_mapping_error(qidev, qm_sg_dma)) { -+ dev_err(qidev, "unable to map S/G table\n"); -+ caam_unmap(qidev, req->src, dst, src_nents, dst_nents, iv_dma, -+ ivsize, op_type, 0, 0); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ edesc->qm_sg_dma = qm_sg_dma; -+ edesc->qm_sg_bytes = qm_sg_bytes; -+ -+ out_len = req->cryptlen + (encrypt ? authsize : 0); -+ in_len = ivsize + req->assoclen + req->cryptlen; -+ -+ fd_sgt = &edesc->drv_req.fd_sgt[0]; -+ -+ dma_to_qm_sg_one_last_ext(&fd_sgt[1], qm_sg_dma, in_len, 0); -+ -+ if (req->dst == req->src) -+ dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + -+ (sg_nents_for_len(req->src, req->assoclen) + -+ 1) * sizeof(*sg_table), out_len, 0); -+ else if (mapped_dst_nents == 1) -+ dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(dst), out_len, 0); -+ else -+ dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + sizeof(*sg_table) * -+ qm_sg_index, out_len, 0); -+ -+ return edesc; -+} -+ -+static int tls_crypt(struct aead_request *req, bool encrypt) -+{ -+ struct tls_edesc *edesc; -+ struct crypto_aead *aead = crypto_aead_reqtfm(req); -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ int ret; -+ -+ if (unlikely(caam_congested)) -+ return -EAGAIN; -+ -+ edesc = tls_edesc_alloc(req, encrypt); -+ if (IS_ERR_OR_NULL(edesc)) -+ return PTR_ERR(edesc); -+ -+ ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); -+ if (!ret) { -+ ret = -EINPROGRESS; -+ } else { -+ tls_unmap(ctx->qidev, edesc, req); -+ qi_cache_free(edesc); -+ } -+ -+ return ret; -+} -+ -+static int tls_encrypt(struct aead_request *req) -+{ -+ return tls_crypt(req, true); -+} -+ -+static int tls_decrypt(struct aead_request *req) -+{ -+ return tls_crypt(req, false); -+} -+ -+static void ablkcipher_done(struct caam_drv_req *drv_req, u32 status) -+{ -+ struct ablkcipher_edesc *edesc; -+ struct ablkcipher_request *req = drv_req->app_ctx; -+ struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); -+ struct caam_ctx *caam_ctx = crypto_ablkcipher_ctx(ablkcipher); -+ struct device *qidev = caam_ctx->qidev; -+ int ivsize = crypto_ablkcipher_ivsize(ablkcipher); -+ -+#ifdef DEBUG -+ dev_err(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status); -+#endif -+ -+ edesc = container_of(drv_req, typeof(*edesc), drv_req); -+ -+ if (status) -+ caam_jr_strstatus(qidev, status); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "dstiv @" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, req->info, -+ edesc->src_nents > 1 ? 100 : ivsize, 1); -+ caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, req->dst, -+ edesc->dst_nents > 1 ? 100 : req->nbytes, 1); -+#endif -+ -+ ablkcipher_unmap(qidev, edesc, req); -+ qi_cache_free(edesc); -+ -+ /* -+ * The crypto API expects us to set the IV (req->info) to the last -+ * ciphertext block. This is used e.g. by the CTS mode. -+ */ -+ scatterwalk_map_and_copy(req->info, req->dst, req->nbytes - ivsize, -+ ivsize, 0); -+ -+ ablkcipher_request_complete(req, status); -+} -+ -+static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request -+ *req, bool encrypt) -+{ -+ struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); -+ struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); -+ struct device *qidev = ctx->qidev; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; -+ int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; -+ struct ablkcipher_edesc *edesc; -+ dma_addr_t iv_dma; -+ bool in_contig; -+ int ivsize = crypto_ablkcipher_ivsize(ablkcipher); -+ int dst_sg_idx, qm_sg_ents; -+ struct qm_sg_entry *sg_table, *fd_sgt; -+ struct caam_drv_ctx *drv_ctx; -+ enum optype op_type = encrypt ? ENCRYPT : DECRYPT; -+ -+ drv_ctx = get_drv_ctx(ctx, op_type); -+ if (unlikely(IS_ERR_OR_NULL(drv_ctx))) -+ return (struct ablkcipher_edesc *)drv_ctx; -+ -+ src_nents = sg_nents_for_len(req->src, req->nbytes); -+ if (unlikely(src_nents < 0)) { -+ dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", -+ req->nbytes); -+ return ERR_PTR(src_nents); -+ } -+ -+ if (unlikely(req->src != req->dst)) { -+ dst_nents = sg_nents_for_len(req->dst, req->nbytes); -+ if (unlikely(dst_nents < 0)) { -+ dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", -+ req->nbytes); -+ return ERR_PTR(dst_nents); -+ } -+ -+ mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, -+ DMA_TO_DEVICE); -+ if (unlikely(!mapped_src_nents)) { -+ dev_err(qidev, "unable to map source\n"); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, -+ DMA_FROM_DEVICE); -+ if (unlikely(!mapped_dst_nents)) { -+ dev_err(qidev, "unable to map destination\n"); -+ dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); -+ return ERR_PTR(-ENOMEM); -+ } -+ } else { -+ mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, -+ DMA_BIDIRECTIONAL); -+ if (unlikely(!mapped_src_nents)) { -+ dev_err(qidev, "unable to map source\n"); -+ return ERR_PTR(-ENOMEM); -+ } -+ } -+ -+ iv_dma = dma_map_single(qidev, req->info, ivsize, DMA_TO_DEVICE); -+ if (dma_mapping_error(qidev, iv_dma)) { -+ dev_err(qidev, "unable to map IV\n"); -+ caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, -+ 0, 0, 0, 0); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ if (mapped_src_nents == 1 && -+ iv_dma + ivsize == sg_dma_address(req->src)) { -+ in_contig = true; -+ qm_sg_ents = 0; -+ } else { -+ in_contig = false; -+ qm_sg_ents = 1 + mapped_src_nents; -+ } -+ dst_sg_idx = qm_sg_ents; -+ -+ qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; -+ if (unlikely(qm_sg_ents > CAAM_QI_MAX_ABLKCIPHER_SG)) { -+ dev_err(qidev, "Insufficient S/G entries: %d > %lu\n", -+ qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG); -+ caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, -+ iv_dma, ivsize, op_type, 0, 0); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ /* allocate space for base edesc and link tables */ -+ edesc = qi_cache_alloc(GFP_DMA | flags); -+ if (unlikely(!edesc)) { -+ dev_err(qidev, "could not allocate extended descriptor\n"); -+ caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, -+ iv_dma, ivsize, op_type, 0, 0); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ edesc->src_nents = src_nents; -+ edesc->dst_nents = dst_nents; -+ edesc->iv_dma = iv_dma; -+ sg_table = &edesc->sgt[0]; -+ edesc->qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); -+ edesc->drv_req.app_ctx = req; -+ edesc->drv_req.cbk = ablkcipher_done; -+ edesc->drv_req.drv_ctx = drv_ctx; -+ -+ if (!in_contig) { -+ dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); -+ sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0); -+ } -+ -+ if (mapped_dst_nents > 1) -+ sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + -+ dst_sg_idx, 0); -+ -+ edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes, -+ DMA_TO_DEVICE); -+ if (dma_mapping_error(qidev, edesc->qm_sg_dma)) { -+ dev_err(qidev, "unable to map S/G table\n"); -+ caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, -+ iv_dma, ivsize, op_type, 0, 0); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ fd_sgt = &edesc->drv_req.fd_sgt[0]; -+ -+ if (!in_contig) -+ dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma, -+ ivsize + req->nbytes, 0); -+ else -+ dma_to_qm_sg_one_last(&fd_sgt[1], iv_dma, ivsize + req->nbytes, -+ 0); -+ -+ if (req->src == req->dst) { -+ if (!in_contig) -+ dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + -+ sizeof(*sg_table), req->nbytes, 0); -+ else -+ dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->src), -+ req->nbytes, 0); -+ } else if (mapped_dst_nents > 1) { -+ dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx * -+ sizeof(*sg_table), req->nbytes, 0); -+ } else { -+ dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), -+ req->nbytes, 0); -+ } -+ -+ return edesc; -+} -+ -+static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( -+ struct skcipher_givcrypt_request *creq) -+{ -+ struct ablkcipher_request *req = &creq->creq; -+ struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); -+ struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); -+ struct device *qidev = ctx->qidev; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; -+ int src_nents, mapped_src_nents, dst_nents, mapped_dst_nents; -+ struct ablkcipher_edesc *edesc; -+ dma_addr_t iv_dma; -+ bool out_contig; -+ int ivsize = crypto_ablkcipher_ivsize(ablkcipher); -+ struct qm_sg_entry *sg_table, *fd_sgt; -+ int dst_sg_idx, qm_sg_ents; -+ struct caam_drv_ctx *drv_ctx; -+ -+ drv_ctx = get_drv_ctx(ctx, GIVENCRYPT); -+ if (unlikely(IS_ERR_OR_NULL(drv_ctx))) -+ return (struct ablkcipher_edesc *)drv_ctx; -+ -+ src_nents = sg_nents_for_len(req->src, req->nbytes); -+ if (unlikely(src_nents < 0)) { -+ dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", -+ req->nbytes); -+ return ERR_PTR(src_nents); -+ } -+ -+ if (unlikely(req->src != req->dst)) { -+ dst_nents = sg_nents_for_len(req->dst, req->nbytes); -+ if (unlikely(dst_nents < 0)) { -+ dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", -+ req->nbytes); -+ return ERR_PTR(dst_nents); -+ } -+ -+ mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, -+ DMA_TO_DEVICE); -+ if (unlikely(!mapped_src_nents)) { -+ dev_err(qidev, "unable to map source\n"); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ mapped_dst_nents = dma_map_sg(qidev, req->dst, dst_nents, -+ DMA_FROM_DEVICE); -+ if (unlikely(!mapped_dst_nents)) { -+ dev_err(qidev, "unable to map destination\n"); -+ dma_unmap_sg(qidev, req->src, src_nents, DMA_TO_DEVICE); -+ return ERR_PTR(-ENOMEM); -+ } -+ } else { -+ mapped_src_nents = dma_map_sg(qidev, req->src, src_nents, -+ DMA_BIDIRECTIONAL); -+ if (unlikely(!mapped_src_nents)) { -+ dev_err(qidev, "unable to map source\n"); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ dst_nents = src_nents; -+ mapped_dst_nents = src_nents; -+ } -+ -+ iv_dma = dma_map_single(qidev, creq->giv, ivsize, DMA_FROM_DEVICE); -+ if (dma_mapping_error(qidev, iv_dma)) { -+ dev_err(qidev, "unable to map IV\n"); -+ caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, -+ 0, 0, 0, 0); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ qm_sg_ents = mapped_src_nents > 1 ? mapped_src_nents : 0; -+ dst_sg_idx = qm_sg_ents; -+ if (mapped_dst_nents == 1 && -+ iv_dma + ivsize == sg_dma_address(req->dst)) { -+ out_contig = true; -+ } else { -+ out_contig = false; -+ qm_sg_ents += 1 + mapped_dst_nents; -+ } -+ -+ if (unlikely(qm_sg_ents > CAAM_QI_MAX_ABLKCIPHER_SG)) { -+ dev_err(qidev, "Insufficient S/G entries: %d > %lu\n", -+ qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG); -+ caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, -+ iv_dma, ivsize, GIVENCRYPT, 0, 0); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ /* allocate space for base edesc and link tables */ -+ edesc = qi_cache_alloc(GFP_DMA | flags); -+ if (!edesc) { -+ dev_err(qidev, "could not allocate extended descriptor\n"); -+ caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, -+ iv_dma, ivsize, GIVENCRYPT, 0, 0); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ edesc->src_nents = src_nents; -+ edesc->dst_nents = dst_nents; -+ edesc->iv_dma = iv_dma; -+ sg_table = &edesc->sgt[0]; -+ edesc->qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); -+ edesc->drv_req.app_ctx = req; -+ edesc->drv_req.cbk = ablkcipher_done; -+ edesc->drv_req.drv_ctx = drv_ctx; -+ -+ if (mapped_src_nents > 1) -+ sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table, 0); -+ -+ if (!out_contig) { -+ dma_to_qm_sg_one(sg_table + dst_sg_idx, iv_dma, ivsize, 0); -+ sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + -+ dst_sg_idx + 1, 0); -+ } -+ -+ edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes, -+ DMA_TO_DEVICE); -+ if (dma_mapping_error(qidev, edesc->qm_sg_dma)) { -+ dev_err(qidev, "unable to map S/G table\n"); -+ caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, -+ iv_dma, ivsize, GIVENCRYPT, 0, 0); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ fd_sgt = &edesc->drv_req.fd_sgt[0]; -+ -+ if (mapped_src_nents > 1) -+ dma_to_qm_sg_one_ext(&fd_sgt[1], edesc->qm_sg_dma, req->nbytes, -+ 0); -+ else -+ dma_to_qm_sg_one(&fd_sgt[1], sg_dma_address(req->src), -+ req->nbytes, 0); -+ -+ if (!out_contig) -+ dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx * -+ sizeof(*sg_table), ivsize + req->nbytes, -+ 0); -+ else -+ dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), -+ ivsize + req->nbytes, 0); -+ -+ return edesc; -+} -+ -+static inline int ablkcipher_crypt(struct ablkcipher_request *req, bool encrypt) -+{ -+ struct ablkcipher_edesc *edesc; -+ struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); -+ struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); -+ int ret; -+ -+ if (unlikely(caam_congested)) -+ return -EAGAIN; -+ -+ /* allocate extended descriptor */ -+ edesc = ablkcipher_edesc_alloc(req, encrypt); -+ if (IS_ERR(edesc)) -+ return PTR_ERR(edesc); -+ -+ ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); -+ if (!ret) { -+ ret = -EINPROGRESS; -+ } else { -+ ablkcipher_unmap(ctx->qidev, edesc, req); -+ qi_cache_free(edesc); -+ } -+ -+ return ret; -+} -+ -+static int ablkcipher_encrypt(struct ablkcipher_request *req) -+{ -+ return ablkcipher_crypt(req, true); -+} -+ -+static int ablkcipher_decrypt(struct ablkcipher_request *req) -+{ -+ return ablkcipher_crypt(req, false); -+} -+ -+static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *creq) -+{ -+ struct ablkcipher_request *req = &creq->creq; -+ struct ablkcipher_edesc *edesc; -+ struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); -+ struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); -+ int ret; -+ -+ if (unlikely(caam_congested)) -+ return -EAGAIN; -+ -+ /* allocate extended descriptor */ -+ edesc = ablkcipher_giv_edesc_alloc(creq); -+ if (IS_ERR(edesc)) -+ return PTR_ERR(edesc); -+ -+ ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); -+ if (!ret) { -+ ret = -EINPROGRESS; -+ } else { -+ ablkcipher_unmap(ctx->qidev, edesc, req); -+ qi_cache_free(edesc); -+ } -+ -+ return ret; -+} -+ -+#define template_ablkcipher template_u.ablkcipher -+struct caam_alg_template { -+ char name[CRYPTO_MAX_ALG_NAME]; -+ char driver_name[CRYPTO_MAX_ALG_NAME]; -+ unsigned int blocksize; -+ u32 type; -+ union { -+ struct ablkcipher_alg ablkcipher; -+ } template_u; -+ u32 class1_alg_type; -+ u32 class2_alg_type; -+}; -+ -+static struct caam_alg_template driver_algs[] = { -+ /* ablkcipher descriptor */ -+ { -+ .name = "cbc(aes)", -+ .driver_name = "cbc-aes-caam-qi", -+ .blocksize = AES_BLOCK_SIZE, -+ .type = CRYPTO_ALG_TYPE_GIVCIPHER, -+ .template_ablkcipher = { -+ .setkey = ablkcipher_setkey, -+ .encrypt = ablkcipher_encrypt, -+ .decrypt = ablkcipher_decrypt, -+ .givencrypt = ablkcipher_givencrypt, -+ .geniv = "<built-in>", -+ .min_keysize = AES_MIN_KEY_SIZE, -+ .max_keysize = AES_MAX_KEY_SIZE, -+ .ivsize = AES_BLOCK_SIZE, -+ }, -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ }, -+ { -+ .name = "cbc(des3_ede)", -+ .driver_name = "cbc-3des-caam-qi", -+ .blocksize = DES3_EDE_BLOCK_SIZE, -+ .type = CRYPTO_ALG_TYPE_GIVCIPHER, -+ .template_ablkcipher = { -+ .setkey = ablkcipher_setkey, -+ .encrypt = ablkcipher_encrypt, -+ .decrypt = ablkcipher_decrypt, -+ .givencrypt = ablkcipher_givencrypt, -+ .geniv = "<built-in>", -+ .min_keysize = DES3_EDE_KEY_SIZE, -+ .max_keysize = DES3_EDE_KEY_SIZE, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ }, -+ { -+ .name = "cbc(des)", -+ .driver_name = "cbc-des-caam-qi", -+ .blocksize = DES_BLOCK_SIZE, -+ .type = CRYPTO_ALG_TYPE_GIVCIPHER, -+ .template_ablkcipher = { -+ .setkey = ablkcipher_setkey, -+ .encrypt = ablkcipher_encrypt, -+ .decrypt = ablkcipher_decrypt, -+ .givencrypt = ablkcipher_givencrypt, -+ .geniv = "<built-in>", -+ .min_keysize = DES_KEY_SIZE, -+ .max_keysize = DES_KEY_SIZE, -+ .ivsize = DES_BLOCK_SIZE, -+ }, -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ }, -+ { -+ .name = "ctr(aes)", -+ .driver_name = "ctr-aes-caam-qi", -+ .blocksize = 1, -+ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, -+ .template_ablkcipher = { -+ .setkey = ablkcipher_setkey, -+ .encrypt = ablkcipher_encrypt, -+ .decrypt = ablkcipher_decrypt, -+ .geniv = "chainiv", -+ .min_keysize = AES_MIN_KEY_SIZE, -+ .max_keysize = AES_MAX_KEY_SIZE, -+ .ivsize = AES_BLOCK_SIZE, -+ }, -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, -+ }, -+ { -+ .name = "rfc3686(ctr(aes))", -+ .driver_name = "rfc3686-ctr-aes-caam-qi", -+ .blocksize = 1, -+ .type = CRYPTO_ALG_TYPE_GIVCIPHER, -+ .template_ablkcipher = { -+ .setkey = ablkcipher_setkey, -+ .encrypt = ablkcipher_encrypt, -+ .decrypt = ablkcipher_decrypt, -+ .givencrypt = ablkcipher_givencrypt, -+ .geniv = "<built-in>", -+ .min_keysize = AES_MIN_KEY_SIZE + -+ CTR_RFC3686_NONCE_SIZE, -+ .max_keysize = AES_MAX_KEY_SIZE + -+ CTR_RFC3686_NONCE_SIZE, -+ .ivsize = CTR_RFC3686_IV_SIZE, -+ }, -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, -+ }, -+ { -+ .name = "xts(aes)", -+ .driver_name = "xts-aes-caam-qi", -+ .blocksize = AES_BLOCK_SIZE, -+ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, -+ .template_ablkcipher = { -+ .setkey = xts_ablkcipher_setkey, -+ .encrypt = ablkcipher_encrypt, -+ .decrypt = ablkcipher_decrypt, -+ .geniv = "eseqiv", -+ .min_keysize = 2 * AES_MIN_KEY_SIZE, -+ .max_keysize = 2 * AES_MAX_KEY_SIZE, -+ .ivsize = AES_BLOCK_SIZE, -+ }, -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, -+ }, -+}; -+ -+static struct caam_aead_alg driver_aeads[] = { -+ { -+ .aead = { -+ .base = { -+ .cra_name = "rfc4106(gcm(aes))", -+ .cra_driver_name = "rfc4106-gcm-aes-caam-qi", -+ .cra_blocksize = 1, -+ }, -+ .setkey = rfc4106_setkey, -+ .setauthsize = rfc4106_setauthsize, -+ .encrypt = ipsec_gcm_encrypt, -+ .decrypt = ipsec_gcm_decrypt, -+ .ivsize = 8, -+ .maxauthsize = AES_BLOCK_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "rfc4543(gcm(aes))", -+ .cra_driver_name = "rfc4543-gcm-aes-caam-qi", -+ .cra_blocksize = 1, -+ }, -+ .setkey = rfc4543_setkey, -+ .setauthsize = rfc4543_setauthsize, -+ .encrypt = ipsec_gcm_encrypt, -+ .decrypt = ipsec_gcm_decrypt, -+ .ivsize = 8, -+ .maxauthsize = AES_BLOCK_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, -+ }, -+ }, -+ /* Galois Counter Mode */ -+ { -+ .aead = { -+ .base = { -+ .cra_name = "gcm(aes)", -+ .cra_driver_name = "gcm-aes-caam-qi", -+ .cra_blocksize = 1, -+ }, -+ .setkey = gcm_setkey, -+ .setauthsize = gcm_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = 12, -+ .maxauthsize = AES_BLOCK_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, -+ } -+ }, -+ /* single-pass ipsec_esp descriptor */ -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(md5),cbc(aes))", -+ .cra_driver_name = "authenc-hmac-md5-" -+ "cbc-aes-caam-qi", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = MD5_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_MD5 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(md5)," -+ "cbc(aes)))", -+ .cra_driver_name = "echainiv-authenc-hmac-md5-" -+ "cbc-aes-caam-qi", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = MD5_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_MD5 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha1),cbc(aes))", -+ .cra_driver_name = "authenc-hmac-sha1-" -+ "cbc-aes-caam-qi", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = SHA1_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA1 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha1)," -+ "cbc(aes)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha1-cbc-aes-caam-qi", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = SHA1_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA1 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha224),cbc(aes))", -+ .cra_driver_name = "authenc-hmac-sha224-" -+ "cbc-aes-caam-qi", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = SHA224_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA224 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha224)," -+ "cbc(aes)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha224-cbc-aes-caam-qi", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = SHA224_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA224 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha256),cbc(aes))", -+ .cra_driver_name = "authenc-hmac-sha256-" -+ "cbc-aes-caam-qi", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = SHA256_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA256 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha256)," -+ "cbc(aes)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha256-cbc-aes-" -+ "caam-qi", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = SHA256_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA256 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha384),cbc(aes))", -+ .cra_driver_name = "authenc-hmac-sha384-" -+ "cbc-aes-caam-qi", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = SHA384_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA384 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha384)," -+ "cbc(aes)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha384-cbc-aes-" -+ "caam-qi", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = SHA384_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA384 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha512),cbc(aes))", -+ .cra_driver_name = "authenc-hmac-sha512-" -+ "cbc-aes-caam-qi", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = SHA512_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA512 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha512)," -+ "cbc(aes)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha512-cbc-aes-" -+ "caam-qi", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = SHA512_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA512 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(md5),cbc(des3_ede))", -+ .cra_driver_name = "authenc-hmac-md5-" -+ "cbc-des3_ede-caam-qi", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = MD5_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_MD5 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(md5)," -+ "cbc(des3_ede)))", -+ .cra_driver_name = "echainiv-authenc-hmac-md5-" -+ "cbc-des3_ede-caam-qi", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = MD5_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_MD5 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha1)," -+ "cbc(des3_ede))", -+ .cra_driver_name = "authenc-hmac-sha1-" -+ "cbc-des3_ede-caam-qi", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = SHA1_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA1 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha1)," -+ "cbc(des3_ede)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha1-" -+ "cbc-des3_ede-caam-qi", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = SHA1_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA1 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha224)," -+ "cbc(des3_ede))", -+ .cra_driver_name = "authenc-hmac-sha224-" -+ "cbc-des3_ede-caam-qi", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = SHA224_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA224 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha224)," -+ "cbc(des3_ede)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha224-" -+ "cbc-des3_ede-caam-qi", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = SHA224_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA224 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha256)," -+ "cbc(des3_ede))", -+ .cra_driver_name = "authenc-hmac-sha256-" -+ "cbc-des3_ede-caam-qi", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = SHA256_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA256 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha256)," -+ "cbc(des3_ede)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha256-" -+ "cbc-des3_ede-caam-qi", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = SHA256_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA256 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha384)," -+ "cbc(des3_ede))", -+ .cra_driver_name = "authenc-hmac-sha384-" -+ "cbc-des3_ede-caam-qi", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = SHA384_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA384 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha384)," -+ "cbc(des3_ede)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha384-" -+ "cbc-des3_ede-caam-qi", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = SHA384_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA384 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha512)," -+ "cbc(des3_ede))", -+ .cra_driver_name = "authenc-hmac-sha512-" -+ "cbc-des3_ede-caam-qi", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = SHA512_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA512 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha512)," -+ "cbc(des3_ede)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha512-" -+ "cbc-des3_ede-caam-qi", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = SHA512_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA512 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(md5),cbc(des))", -+ .cra_driver_name = "authenc-hmac-md5-" -+ "cbc-des-caam-qi", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = MD5_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_MD5 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(md5)," -+ "cbc(des)))", -+ .cra_driver_name = "echainiv-authenc-hmac-md5-" -+ "cbc-des-caam-qi", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = MD5_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_MD5 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha1),cbc(des))", -+ .cra_driver_name = "authenc-hmac-sha1-" -+ "cbc-des-caam-qi", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = SHA1_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA1 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha1)," -+ "cbc(des)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha1-cbc-des-caam-qi", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = SHA1_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA1 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha224),cbc(des))", -+ .cra_driver_name = "authenc-hmac-sha224-" -+ "cbc-des-caam-qi", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = SHA224_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA224 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha224)," -+ "cbc(des)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha224-cbc-des-" -+ "caam-qi", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = SHA224_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA224 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha256),cbc(des))", -+ .cra_driver_name = "authenc-hmac-sha256-" -+ "cbc-des-caam-qi", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = SHA256_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA256 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha256)," -+ "cbc(des)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha256-cbc-des-" -+ "caam-qi", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = SHA256_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA256 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha384),cbc(des))", -+ .cra_driver_name = "authenc-hmac-sha384-" -+ "cbc-des-caam-qi", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = SHA384_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA384 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha384)," -+ "cbc(des)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha384-cbc-des-" -+ "caam-qi", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = SHA384_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA384 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha512),cbc(des))", -+ .cra_driver_name = "authenc-hmac-sha512-" -+ "cbc-des-caam-qi", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = SHA512_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA512 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha512)," -+ "cbc(des)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha512-cbc-des-" -+ "caam-qi", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = SHA512_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA512 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "tls10(hmac(sha1),cbc(aes))", -+ .cra_driver_name = "tls10-hmac-sha1-cbc-aes-caam-qi", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = tls_setkey, -+ .setauthsize = tls_setauthsize, -+ .encrypt = tls_encrypt, -+ .decrypt = tls_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = SHA1_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA1 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ } -+ } -+}; -+ -+struct caam_crypto_alg { -+ struct list_head entry; -+ struct crypto_alg crypto_alg; -+ struct caam_alg_entry caam; -+}; -+ -+static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, -+ bool uses_dkp) -+{ -+ struct caam_drv_private *priv; -+ /* Digest sizes for MD5, SHA1, SHA-224, SHA-256, SHA-384, SHA-512 */ -+ static const u8 digest_size[] = { -+ MD5_DIGEST_SIZE, -+ SHA1_DIGEST_SIZE, -+ SHA224_DIGEST_SIZE, -+ SHA256_DIGEST_SIZE, -+ SHA384_DIGEST_SIZE, -+ SHA512_DIGEST_SIZE -+ }; -+ u8 op_id; -+ -+ /* -+ * distribute tfms across job rings to ensure in-order -+ * crypto request processing per tfm -+ */ -+ ctx->jrdev = caam_jr_alloc(); -+ if (IS_ERR(ctx->jrdev)) { -+ pr_err("Job Ring Device allocation for transform failed\n"); -+ return PTR_ERR(ctx->jrdev); -+ } -+ -+ priv = dev_get_drvdata(ctx->jrdev->parent); -+ if (priv->era >= 6 && uses_dkp) -+ ctx->dir = DMA_BIDIRECTIONAL; -+ else -+ ctx->dir = DMA_TO_DEVICE; -+ -+ ctx->key_dma = dma_map_single(ctx->jrdev, ctx->key, sizeof(ctx->key), -+ ctx->dir); -+ if (dma_mapping_error(ctx->jrdev, ctx->key_dma)) { -+ dev_err(ctx->jrdev, "unable to map key\n"); -+ caam_jr_free(ctx->jrdev); -+ return -ENOMEM; -+ } -+ -+ /* copy descriptor header template value */ -+ ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; -+ ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; -+ -+ if (ctx->adata.algtype) { -+ op_id = (ctx->adata.algtype & OP_ALG_ALGSEL_SUBMASK) -+ >> OP_ALG_ALGSEL_SHIFT; -+ if (op_id < ARRAY_SIZE(digest_size)) { -+ ctx->authsize = digest_size[op_id]; -+ } else { -+ dev_err(ctx->jrdev, -+ "incorrect op_id %d; must be less than %zu\n", -+ op_id, ARRAY_SIZE(digest_size)); -+ caam_jr_free(ctx->jrdev); -+ return -EINVAL; -+ } -+ } else { -+ ctx->authsize = 0; -+ } -+ -+ ctx->qidev = priv->qidev; -+ -+ spin_lock_init(&ctx->lock); -+ ctx->drv_ctx[ENCRYPT] = NULL; -+ ctx->drv_ctx[DECRYPT] = NULL; -+ ctx->drv_ctx[GIVENCRYPT] = NULL; -+ -+ return 0; -+} -+ -+static int caam_cra_init(struct crypto_tfm *tfm) -+{ -+ struct crypto_alg *alg = tfm->__crt_alg; -+ struct caam_crypto_alg *caam_alg = container_of(alg, typeof(*caam_alg), -+ crypto_alg); -+ struct caam_ctx *ctx = crypto_tfm_ctx(tfm); -+ -+ return caam_init_common(ctx, &caam_alg->caam, false); -+} -+ -+static int caam_aead_init(struct crypto_aead *tfm) -+{ -+ struct aead_alg *alg = crypto_aead_alg(tfm); -+ struct caam_aead_alg *caam_alg = container_of(alg, typeof(*caam_alg), -+ aead); -+ struct caam_ctx *ctx = crypto_aead_ctx(tfm); -+ -+ return caam_init_common(ctx, &caam_alg->caam, -+ (alg->setkey == aead_setkey) || -+ (alg->setkey == tls_setkey)); -+} -+ -+static void caam_exit_common(struct caam_ctx *ctx) -+{ -+ caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]); -+ caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]); -+ caam_drv_ctx_rel(ctx->drv_ctx[GIVENCRYPT]); -+ -+ dma_unmap_single(ctx->jrdev, ctx->key_dma, sizeof(ctx->key), ctx->dir); -+ -+ caam_jr_free(ctx->jrdev); -+} -+ -+static void caam_cra_exit(struct crypto_tfm *tfm) -+{ -+ caam_exit_common(crypto_tfm_ctx(tfm)); -+} -+ -+static void caam_aead_exit(struct crypto_aead *tfm) -+{ -+ caam_exit_common(crypto_aead_ctx(tfm)); -+} -+ -+static struct list_head alg_list; -+static void __exit caam_qi_algapi_exit(void) -+{ -+ struct caam_crypto_alg *t_alg, *n; -+ int i; -+ -+ for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { -+ struct caam_aead_alg *t_alg = driver_aeads + i; -+ -+ if (t_alg->registered) -+ crypto_unregister_aead(&t_alg->aead); -+ } -+ -+ if (!alg_list.next) -+ return; -+ -+ list_for_each_entry_safe(t_alg, n, &alg_list, entry) { -+ crypto_unregister_alg(&t_alg->crypto_alg); -+ list_del(&t_alg->entry); -+ kfree(t_alg); -+ } -+} -+ -+static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template -+ *template) -+{ -+ struct caam_crypto_alg *t_alg; -+ struct crypto_alg *alg; -+ -+ t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); -+ if (!t_alg) -+ return ERR_PTR(-ENOMEM); -+ -+ alg = &t_alg->crypto_alg; -+ -+ snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name); -+ snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", -+ template->driver_name); -+ alg->cra_module = THIS_MODULE; -+ alg->cra_init = caam_cra_init; -+ alg->cra_exit = caam_cra_exit; -+ alg->cra_priority = CAAM_CRA_PRIORITY; -+ alg->cra_blocksize = template->blocksize; -+ alg->cra_alignmask = 0; -+ alg->cra_ctxsize = sizeof(struct caam_ctx); -+ alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | -+ template->type; -+ switch (template->type) { -+ case CRYPTO_ALG_TYPE_GIVCIPHER: -+ alg->cra_type = &crypto_givcipher_type; -+ alg->cra_ablkcipher = template->template_ablkcipher; -+ break; -+ case CRYPTO_ALG_TYPE_ABLKCIPHER: -+ alg->cra_type = &crypto_ablkcipher_type; -+ alg->cra_ablkcipher = template->template_ablkcipher; -+ break; -+ } -+ -+ t_alg->caam.class1_alg_type = template->class1_alg_type; -+ t_alg->caam.class2_alg_type = template->class2_alg_type; -+ -+ return t_alg; -+} -+ -+static void caam_aead_alg_init(struct caam_aead_alg *t_alg) -+{ -+ struct aead_alg *alg = &t_alg->aead; -+ -+ alg->base.cra_module = THIS_MODULE; -+ alg->base.cra_priority = CAAM_CRA_PRIORITY; -+ alg->base.cra_ctxsize = sizeof(struct caam_ctx); -+ alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; -+ -+ alg->init = caam_aead_init; -+ alg->exit = caam_aead_exit; -+} -+ -+static int __init caam_qi_algapi_init(void) -+{ -+ struct device_node *dev_node; -+ struct platform_device *pdev; -+ struct device *ctrldev; -+ struct caam_drv_private *priv; -+ int i = 0, err = 0; -+ u32 cha_vid, cha_inst, des_inst, aes_inst, md_inst; -+ unsigned int md_limit = SHA512_DIGEST_SIZE; -+ bool registered = false; -+ -+ dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); -+ if (!dev_node) { -+ dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); -+ if (!dev_node) -+ return -ENODEV; -+ } -+ -+ pdev = of_find_device_by_node(dev_node); -+ of_node_put(dev_node); -+ if (!pdev) -+ return -ENODEV; -+ -+ ctrldev = &pdev->dev; -+ priv = dev_get_drvdata(ctrldev); -+ -+ /* -+ * If priv is NULL, it's probably because the caam driver wasn't -+ * properly initialized (e.g. RNG4 init failed). Thus, bail out here. -+ */ -+ if (!priv || !priv->qi_present) -+ return -ENODEV; -+ -+ if (caam_dpaa2) { -+ dev_info(ctrldev, "caam/qi frontend driver not suitable for DPAA 2.x, aborting...\n"); -+ return -ENODEV; -+ } -+ -+ INIT_LIST_HEAD(&alg_list); -+ -+ /* -+ * Register crypto algorithms the device supports. -+ * First, detect presence and attributes of DES, AES, and MD blocks. -+ */ -+ cha_vid = rd_reg32(&priv->ctrl->perfmon.cha_id_ls); -+ cha_inst = rd_reg32(&priv->ctrl->perfmon.cha_num_ls); -+ des_inst = (cha_inst & CHA_ID_LS_DES_MASK) >> CHA_ID_LS_DES_SHIFT; -+ aes_inst = (cha_inst & CHA_ID_LS_AES_MASK) >> CHA_ID_LS_AES_SHIFT; -+ md_inst = (cha_inst & CHA_ID_LS_MD_MASK) >> CHA_ID_LS_MD_SHIFT; -+ -+ /* If MD is present, limit digest size based on LP256 */ -+ if (md_inst && ((cha_vid & CHA_ID_LS_MD_MASK) == CHA_ID_LS_MD_LP256)) -+ md_limit = SHA256_DIGEST_SIZE; -+ -+ for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { -+ struct caam_crypto_alg *t_alg; -+ struct caam_alg_template *alg = driver_algs + i; -+ u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK; -+ -+ /* Skip DES algorithms if not supported by device */ -+ if (!des_inst && -+ ((alg_sel == OP_ALG_ALGSEL_3DES) || -+ (alg_sel == OP_ALG_ALGSEL_DES))) -+ continue; -+ -+ /* Skip AES algorithms if not supported by device */ -+ if (!aes_inst && (alg_sel == OP_ALG_ALGSEL_AES)) -+ continue; -+ -+ t_alg = caam_alg_alloc(alg); -+ if (IS_ERR(t_alg)) { -+ err = PTR_ERR(t_alg); -+ dev_warn(priv->qidev, "%s alg allocation failed\n", -+ alg->driver_name); -+ continue; -+ } -+ -+ err = crypto_register_alg(&t_alg->crypto_alg); -+ if (err) { -+ dev_warn(priv->qidev, "%s alg registration failed\n", -+ t_alg->crypto_alg.cra_driver_name); -+ kfree(t_alg); -+ continue; -+ } -+ -+ list_add_tail(&t_alg->entry, &alg_list); -+ registered = true; -+ } -+ -+ for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { -+ struct caam_aead_alg *t_alg = driver_aeads + i; -+ u32 c1_alg_sel = t_alg->caam.class1_alg_type & -+ OP_ALG_ALGSEL_MASK; -+ u32 c2_alg_sel = t_alg->caam.class2_alg_type & -+ OP_ALG_ALGSEL_MASK; -+ u32 alg_aai = t_alg->caam.class1_alg_type & OP_ALG_AAI_MASK; -+ -+ /* Skip DES algorithms if not supported by device */ -+ if (!des_inst && -+ ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || -+ (c1_alg_sel == OP_ALG_ALGSEL_DES))) -+ continue; -+ -+ /* Skip AES algorithms if not supported by device */ -+ if (!aes_inst && (c1_alg_sel == OP_ALG_ALGSEL_AES)) -+ continue; -+ -+ /* -+ * Check support for AES algorithms not available -+ * on LP devices. -+ */ -+ if (((cha_vid & CHA_ID_LS_AES_MASK) == CHA_ID_LS_AES_LP) && -+ (alg_aai == OP_ALG_AAI_GCM)) -+ continue; -+ -+ /* -+ * Skip algorithms requiring message digests -+ * if MD or MD size is not supported by device. -+ */ -+ if (c2_alg_sel && -+ (!md_inst || (t_alg->aead.maxauthsize > md_limit))) -+ continue; -+ -+ caam_aead_alg_init(t_alg); -+ -+ err = crypto_register_aead(&t_alg->aead); -+ if (err) { -+ pr_warn("%s alg registration failed\n", -+ t_alg->aead.base.cra_driver_name); -+ continue; -+ } -+ -+ t_alg->registered = true; -+ registered = true; -+ } -+ -+ if (registered) -+ dev_info(priv->qidev, "algorithms registered in /proc/crypto\n"); -+ -+ return err; -+} -+ -+module_init(caam_qi_algapi_init); -+module_exit(caam_qi_algapi_exit); -+ -+MODULE_LICENSE("GPL"); -+MODULE_DESCRIPTION("Support for crypto API using CAAM-QI backend"); -+MODULE_AUTHOR("Freescale Semiconductor"); ---- /dev/null -+++ b/drivers/crypto/caam/caamalg_qi2.c -@@ -0,0 +1,5938 @@ -+/* -+ * Copyright 2015-2016 Freescale Semiconductor Inc. -+ * Copyright 2017 NXP -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the names of the above-listed copyright holders nor the -+ * names of any contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE -+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -+ * POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include <linux/fsl/mc.h> -+#include "compat.h" -+#include "regs.h" -+#include "caamalg_qi2.h" -+#include "dpseci_cmd.h" -+#include "desc_constr.h" -+#include "error.h" -+#include "sg_sw_sec4.h" -+#include "sg_sw_qm2.h" -+#include "key_gen.h" -+#include "caamalg_desc.h" -+#include "caamhash_desc.h" -+#include "../../../drivers/staging/fsl-mc/include/dpaa2-io.h" -+#include "../../../drivers/staging/fsl-mc/include/dpaa2-fd.h" -+ -+#define CAAM_CRA_PRIORITY 2000 -+ -+/* max key is sum of AES_MAX_KEY_SIZE, max split key size */ -+#define CAAM_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE + \ -+ SHA512_DIGEST_SIZE * 2) -+ -+#ifndef CONFIG_CRYPTO_DEV_FSL_CAAM -+bool caam_little_end; -+EXPORT_SYMBOL(caam_little_end); -+bool caam_imx; -+EXPORT_SYMBOL(caam_imx); -+#endif -+ -+/* -+ * This is a a cache of buffers, from which the users of CAAM QI driver -+ * can allocate short buffers. It's speedier than doing kmalloc on the hotpath. -+ * NOTE: A more elegant solution would be to have some headroom in the frames -+ * being processed. This can be added by the dpaa2-eth driver. This would -+ * pose a problem for userspace application processing which cannot -+ * know of this limitation. So for now, this will work. -+ * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here -+ */ -+static struct kmem_cache *qi_cache; -+ -+struct caam_alg_entry { -+ struct device *dev; -+ int class1_alg_type; -+ int class2_alg_type; -+ bool rfc3686; -+ bool geniv; -+}; -+ -+struct caam_aead_alg { -+ struct aead_alg aead; -+ struct caam_alg_entry caam; -+ bool registered; -+}; -+ -+/** -+ * caam_ctx - per-session context -+ * @flc: Flow Contexts array -+ * @key: virtual address of the key(s): [authentication key], encryption key -+ * @flc_dma: I/O virtual addresses of the Flow Contexts -+ * @key_dma: I/O virtual address of the key -+ * @dir: DMA direction for mapping key and Flow Contexts -+ * @dev: dpseci device -+ * @adata: authentication algorithm details -+ * @cdata: encryption algorithm details -+ * @authsize: authentication tag (a.k.a. ICV / MAC) size -+ */ -+struct caam_ctx { -+ struct caam_flc flc[NUM_OP]; -+ u8 key[CAAM_MAX_KEY_SIZE]; -+ dma_addr_t flc_dma[NUM_OP]; -+ dma_addr_t key_dma; -+ enum dma_data_direction dir; -+ struct device *dev; -+ struct alginfo adata; -+ struct alginfo cdata; -+ unsigned int authsize; -+}; -+ -+void *dpaa2_caam_iova_to_virt(struct dpaa2_caam_priv *priv, -+ dma_addr_t iova_addr) -+{ -+ phys_addr_t phys_addr; -+ -+ phys_addr = priv->domain ? iommu_iova_to_phys(priv->domain, iova_addr) : -+ iova_addr; -+ -+ return phys_to_virt(phys_addr); -+} -+ -+/* -+ * qi_cache_zalloc - Allocate buffers from CAAM-QI cache -+ * -+ * Allocate data on the hotpath. Instead of using kzalloc, one can use the -+ * services of the CAAM QI memory cache (backed by kmem_cache). The buffers -+ * will have a size of CAAM_QI_MEMCACHE_SIZE, which should be sufficient for -+ * hosting 16 SG entries. -+ * -+ * @flags - flags that would be used for the equivalent kmalloc(..) call -+ * -+ * Returns a pointer to a retrieved buffer on success or NULL on failure. -+ */ -+static inline void *qi_cache_zalloc(gfp_t flags) -+{ -+ return kmem_cache_zalloc(qi_cache, flags); -+} -+ -+/* -+ * qi_cache_free - Frees buffers allocated from CAAM-QI cache -+ * -+ * @obj - buffer previously allocated by qi_cache_zalloc -+ * -+ * No checking is being done, the call is a passthrough call to -+ * kmem_cache_free(...) -+ */ -+static inline void qi_cache_free(void *obj) -+{ -+ kmem_cache_free(qi_cache, obj); -+} -+ -+static struct caam_request *to_caam_req(struct crypto_async_request *areq) -+{ -+ switch (crypto_tfm_alg_type(areq->tfm)) { -+ case CRYPTO_ALG_TYPE_ABLKCIPHER: -+ case CRYPTO_ALG_TYPE_GIVCIPHER: -+ return ablkcipher_request_ctx(ablkcipher_request_cast(areq)); -+ case CRYPTO_ALG_TYPE_AEAD: -+ return aead_request_ctx(container_of(areq, struct aead_request, -+ base)); -+ case CRYPTO_ALG_TYPE_AHASH: -+ return ahash_request_ctx(ahash_request_cast(areq)); -+ default: -+ return ERR_PTR(-EINVAL); -+ } -+} -+ -+static void caam_unmap(struct device *dev, struct scatterlist *src, -+ struct scatterlist *dst, int src_nents, -+ int dst_nents, dma_addr_t iv_dma, int ivsize, -+ enum optype op_type, dma_addr_t qm_sg_dma, -+ int qm_sg_bytes) -+{ -+ if (dst != src) { -+ if (src_nents) -+ dma_unmap_sg(dev, src, src_nents, DMA_TO_DEVICE); -+ dma_unmap_sg(dev, dst, dst_nents, DMA_FROM_DEVICE); -+ } else { -+ dma_unmap_sg(dev, src, src_nents, DMA_BIDIRECTIONAL); -+ } -+ -+ if (iv_dma) -+ dma_unmap_single(dev, iv_dma, ivsize, -+ op_type == GIVENCRYPT ? DMA_FROM_DEVICE : -+ DMA_TO_DEVICE); -+ -+ if (qm_sg_bytes) -+ dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); -+} -+ -+static int aead_set_sh_desc(struct crypto_aead *aead) -+{ -+ struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), -+ typeof(*alg), aead); -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ unsigned int ivsize = crypto_aead_ivsize(aead); -+ struct device *dev = ctx->dev; -+ struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); -+ struct caam_flc *flc; -+ u32 *desc; -+ u32 ctx1_iv_off = 0; -+ u32 *nonce = NULL; -+ unsigned int data_len[2]; -+ u32 inl_mask; -+ const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == -+ OP_ALG_AAI_CTR_MOD128); -+ const bool is_rfc3686 = alg->caam.rfc3686; -+ -+ if (!ctx->cdata.keylen || !ctx->authsize) -+ return 0; -+ -+ /* -+ * AES-CTR needs to load IV in CONTEXT1 reg -+ * at an offset of 128bits (16bytes) -+ * CONTEXT1[255:128] = IV -+ */ -+ if (ctr_mode) -+ ctx1_iv_off = 16; -+ -+ /* -+ * RFC3686 specific: -+ * CONTEXT1[255:128] = {NONCE, IV, COUNTER} -+ */ -+ if (is_rfc3686) { -+ ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; -+ nonce = (u32 *)((void *)ctx->key + ctx->adata.keylen_pad + -+ ctx->cdata.keylen - CTR_RFC3686_NONCE_SIZE); -+ } -+ -+ data_len[0] = ctx->adata.keylen_pad; -+ data_len[1] = ctx->cdata.keylen; -+ -+ /* aead_encrypt shared descriptor */ -+ if (desc_inline_query((alg->caam.geniv ? DESC_QI_AEAD_GIVENC_LEN : -+ DESC_QI_AEAD_ENC_LEN) + -+ (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), -+ DESC_JOB_IO_LEN, data_len, &inl_mask, -+ ARRAY_SIZE(data_len)) < 0) -+ return -EINVAL; -+ -+ if (inl_mask & 1) -+ ctx->adata.key_virt = ctx->key; -+ else -+ ctx->adata.key_dma = ctx->key_dma; -+ -+ if (inl_mask & 2) -+ ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; -+ else -+ ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; -+ -+ ctx->adata.key_inline = !!(inl_mask & 1); -+ ctx->cdata.key_inline = !!(inl_mask & 2); -+ -+ flc = &ctx->flc[ENCRYPT]; -+ desc = flc->sh_desc; -+ -+ if (alg->caam.geniv) -+ cnstr_shdsc_aead_givencap(desc, &ctx->cdata, &ctx->adata, -+ ivsize, ctx->authsize, is_rfc3686, -+ nonce, ctx1_iv_off, true, -+ priv->sec_attr.era); -+ else -+ cnstr_shdsc_aead_encap(desc, &ctx->cdata, &ctx->adata, -+ ivsize, ctx->authsize, is_rfc3686, nonce, -+ ctx1_iv_off, true, priv->sec_attr.era); -+ -+ flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ -+ dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], -+ sizeof(flc->flc) + desc_bytes(desc), -+ ctx->dir); -+ -+ /* aead_decrypt shared descriptor */ -+ if (desc_inline_query(DESC_QI_AEAD_DEC_LEN + -+ (is_rfc3686 ? DESC_AEAD_CTR_RFC3686_LEN : 0), -+ DESC_JOB_IO_LEN, data_len, &inl_mask, -+ ARRAY_SIZE(data_len)) < 0) -+ return -EINVAL; -+ -+ if (inl_mask & 1) -+ ctx->adata.key_virt = ctx->key; -+ else -+ ctx->adata.key_dma = ctx->key_dma; -+ -+ if (inl_mask & 2) -+ ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; -+ else -+ ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; -+ -+ ctx->adata.key_inline = !!(inl_mask & 1); -+ ctx->cdata.key_inline = !!(inl_mask & 2); -+ -+ flc = &ctx->flc[DECRYPT]; -+ desc = flc->sh_desc; -+ cnstr_shdsc_aead_decap(desc, &ctx->cdata, &ctx->adata, -+ ivsize, ctx->authsize, alg->caam.geniv, -+ is_rfc3686, nonce, ctx1_iv_off, true, -+ priv->sec_attr.era); -+ flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ -+ dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], -+ sizeof(flc->flc) + desc_bytes(desc), -+ ctx->dir); -+ -+ return 0; -+} -+ -+static int aead_setauthsize(struct crypto_aead *authenc, unsigned int authsize) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(authenc); -+ -+ ctx->authsize = authsize; -+ aead_set_sh_desc(authenc); -+ -+ return 0; -+} -+ -+struct split_key_sh_result { -+ struct completion completion; -+ int err; -+ struct device *dev; -+}; -+ -+static void split_key_sh_done(void *cbk_ctx, u32 err) -+{ -+ struct split_key_sh_result *res = cbk_ctx; -+ -+#ifdef DEBUG -+ dev_err(res->dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -+#endif -+ -+ if (err) -+ caam_qi2_strstatus(res->dev, err); -+ -+ res->err = err; -+ complete(&res->completion); -+} -+ -+static int aead_setkey(struct crypto_aead *aead, const u8 *key, -+ unsigned int keylen) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ struct device *dev = ctx->dev; -+ struct crypto_authenc_keys keys; -+ -+ if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) -+ goto badkey; -+ -+#ifdef DEBUG -+ dev_err(dev, "keylen %d enckeylen %d authkeylen %d\n", -+ keys.authkeylen + keys.enckeylen, keys.enckeylen, -+ keys.authkeylen); -+ print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -+#endif -+ -+ ctx->adata.keylen = keys.authkeylen; -+ ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & -+ OP_ALG_ALGSEL_MASK); -+ -+ if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) -+ goto badkey; -+ -+ memcpy(ctx->key, keys.authkey, keys.authkeylen); -+ memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); -+ dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad + -+ keys.enckeylen, ctx->dir); -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, -+ ctx->adata.keylen_pad + keys.enckeylen, 1); -+#endif -+ -+ ctx->cdata.keylen = keys.enckeylen; -+ -+ return aead_set_sh_desc(aead); -+badkey: -+ crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); -+ return -EINVAL; -+} -+ -+static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, -+ bool encrypt) -+{ -+ struct crypto_aead *aead = crypto_aead_reqtfm(req); -+ struct caam_request *req_ctx = aead_request_ctx(req); -+ struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; -+ struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ struct caam_aead_alg *alg = container_of(crypto_aead_alg(aead), -+ typeof(*alg), aead); -+ struct device *dev = ctx->dev; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; -+ int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; -+ struct aead_edesc *edesc; -+ dma_addr_t qm_sg_dma, iv_dma = 0; -+ int ivsize = 0; -+ unsigned int authsize = ctx->authsize; -+ int qm_sg_index = 0, qm_sg_nents = 0, qm_sg_bytes; -+ int in_len, out_len; -+ struct dpaa2_sg_entry *sg_table; -+ enum optype op_type = encrypt ? ENCRYPT : DECRYPT; -+ -+ /* allocate space for base edesc and link tables */ -+ edesc = qi_cache_zalloc(GFP_DMA | flags); -+ if (unlikely(!edesc)) { -+ dev_err(dev, "could not allocate extended descriptor\n"); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ if (unlikely(req->dst != req->src)) { -+ src_nents = sg_nents_for_len(req->src, req->assoclen + -+ req->cryptlen); -+ if (unlikely(src_nents < 0)) { -+ dev_err(dev, "Insufficient bytes (%d) in src S/G\n", -+ req->assoclen + req->cryptlen); -+ qi_cache_free(edesc); -+ return ERR_PTR(src_nents); -+ } -+ -+ dst_nents = sg_nents_for_len(req->dst, req->assoclen + -+ req->cryptlen + -+ (encrypt ? authsize : -+ (-authsize))); -+ if (unlikely(dst_nents < 0)) { -+ dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", -+ req->assoclen + req->cryptlen + -+ (encrypt ? authsize : (-authsize))); -+ qi_cache_free(edesc); -+ return ERR_PTR(dst_nents); -+ } -+ -+ if (src_nents) { -+ mapped_src_nents = dma_map_sg(dev, req->src, src_nents, -+ DMA_TO_DEVICE); -+ if (unlikely(!mapped_src_nents)) { -+ dev_err(dev, "unable to map source\n"); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ } else { -+ mapped_src_nents = 0; -+ } -+ -+ mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, -+ DMA_FROM_DEVICE); -+ if (unlikely(!mapped_dst_nents)) { -+ dev_err(dev, "unable to map destination\n"); -+ dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ } else { -+ src_nents = sg_nents_for_len(req->src, req->assoclen + -+ req->cryptlen + -+ (encrypt ? authsize : 0)); -+ if (unlikely(src_nents < 0)) { -+ dev_err(dev, "Insufficient bytes (%d) in src S/G\n", -+ req->assoclen + req->cryptlen + -+ (encrypt ? authsize : 0)); -+ qi_cache_free(edesc); -+ return ERR_PTR(src_nents); -+ } -+ -+ mapped_src_nents = dma_map_sg(dev, req->src, src_nents, -+ DMA_BIDIRECTIONAL); -+ if (unlikely(!mapped_src_nents)) { -+ dev_err(dev, "unable to map source\n"); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ } -+ -+ if ((alg->caam.rfc3686 && encrypt) || !alg->caam.geniv) { -+ ivsize = crypto_aead_ivsize(aead); -+ iv_dma = dma_map_single(dev, req->iv, ivsize, DMA_TO_DEVICE); -+ if (dma_mapping_error(dev, iv_dma)) { -+ dev_err(dev, "unable to map IV\n"); -+ caam_unmap(dev, req->src, req->dst, src_nents, -+ dst_nents, 0, 0, op_type, 0, 0); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ } -+ -+ /* -+ * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. -+ * Input is not contiguous. -+ */ -+ qm_sg_nents = 1 + !!ivsize + mapped_src_nents + -+ (mapped_dst_nents > 1 ? mapped_dst_nents : 0); -+ if (unlikely(qm_sg_nents > CAAM_QI_MAX_AEAD_SG)) { -+ dev_err(dev, "Insufficient S/G entries: %d > %lu\n", -+ qm_sg_nents, CAAM_QI_MAX_AEAD_SG); -+ caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, -+ iv_dma, ivsize, op_type, 0, 0); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ sg_table = &edesc->sgt[0]; -+ qm_sg_bytes = qm_sg_nents * sizeof(*sg_table); -+ -+ edesc->src_nents = src_nents; -+ edesc->dst_nents = dst_nents; -+ edesc->iv_dma = iv_dma; -+ -+ edesc->assoclen = cpu_to_caam32(req->assoclen); -+ edesc->assoclen_dma = dma_map_single(dev, &edesc->assoclen, 4, -+ DMA_TO_DEVICE); -+ if (dma_mapping_error(dev, edesc->assoclen_dma)) { -+ dev_err(dev, "unable to map assoclen\n"); -+ caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, -+ iv_dma, ivsize, op_type, 0, 0); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ dma_to_qm_sg_one(sg_table, edesc->assoclen_dma, 4, 0); -+ qm_sg_index++; -+ if (ivsize) { -+ dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); -+ qm_sg_index++; -+ } -+ sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0); -+ qm_sg_index += mapped_src_nents; -+ -+ if (mapped_dst_nents > 1) -+ sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + -+ qm_sg_index, 0); -+ -+ qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); -+ if (dma_mapping_error(dev, qm_sg_dma)) { -+ dev_err(dev, "unable to map S/G table\n"); -+ dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); -+ caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, -+ iv_dma, ivsize, op_type, 0, 0); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ edesc->qm_sg_dma = qm_sg_dma; -+ edesc->qm_sg_bytes = qm_sg_bytes; -+ -+ out_len = req->assoclen + req->cryptlen + -+ (encrypt ? ctx->authsize : (-ctx->authsize)); -+ in_len = 4 + ivsize + req->assoclen + req->cryptlen; -+ -+ memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); -+ dpaa2_fl_set_final(in_fle, true); -+ dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); -+ dpaa2_fl_set_addr(in_fle, qm_sg_dma); -+ dpaa2_fl_set_len(in_fle, in_len); -+ -+ if (req->dst == req->src) { -+ if (mapped_src_nents == 1) { -+ dpaa2_fl_set_format(out_fle, dpaa2_fl_single); -+ dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src)); -+ } else { -+ dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); -+ dpaa2_fl_set_addr(out_fle, qm_sg_dma + -+ (1 + !!ivsize) * sizeof(*sg_table)); -+ } -+ } else if (mapped_dst_nents == 1) { -+ dpaa2_fl_set_format(out_fle, dpaa2_fl_single); -+ dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); -+ } else { -+ dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); -+ dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index * -+ sizeof(*sg_table)); -+ } -+ -+ dpaa2_fl_set_len(out_fle, out_len); -+ -+ return edesc; -+} -+ -+static struct tls_edesc *tls_edesc_alloc(struct aead_request *req, -+ bool encrypt) -+{ -+ struct crypto_aead *tls = crypto_aead_reqtfm(req); -+ unsigned int blocksize = crypto_aead_blocksize(tls); -+ unsigned int padsize, authsize; -+ struct caam_request *req_ctx = aead_request_ctx(req); -+ struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; -+ struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; -+ struct caam_ctx *ctx = crypto_aead_ctx(tls); -+ struct caam_aead_alg *alg = container_of(crypto_aead_alg(tls), -+ typeof(*alg), aead); -+ struct device *dev = ctx->dev; -+ gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | -+ CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; -+ int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; -+ struct tls_edesc *edesc; -+ dma_addr_t qm_sg_dma, iv_dma = 0; -+ int ivsize = 0; -+ int qm_sg_index, qm_sg_ents = 0, qm_sg_bytes; -+ int in_len, out_len; -+ struct dpaa2_sg_entry *sg_table; -+ enum optype op_type = encrypt ? ENCRYPT : DECRYPT; -+ struct scatterlist *dst; -+ -+ if (encrypt) { -+ padsize = blocksize - ((req->cryptlen + ctx->authsize) % -+ blocksize); -+ authsize = ctx->authsize + padsize; -+ } else { -+ authsize = ctx->authsize; -+ } -+ -+ /* allocate space for base edesc and link tables */ -+ edesc = qi_cache_zalloc(GFP_DMA | flags); -+ if (unlikely(!edesc)) { -+ dev_err(dev, "could not allocate extended descriptor\n"); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ if (likely(req->src == req->dst)) { -+ src_nents = sg_nents_for_len(req->src, req->assoclen + -+ req->cryptlen + -+ (encrypt ? authsize : 0)); -+ if (unlikely(src_nents < 0)) { -+ dev_err(dev, "Insufficient bytes (%d) in src S/G\n", -+ req->assoclen + req->cryptlen + -+ (encrypt ? authsize : 0)); -+ qi_cache_free(edesc); -+ return ERR_PTR(src_nents); -+ } -+ -+ mapped_src_nents = dma_map_sg(dev, req->src, src_nents, -+ DMA_BIDIRECTIONAL); -+ if (unlikely(!mapped_src_nents)) { -+ dev_err(dev, "unable to map source\n"); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ dst = req->dst; -+ } else { -+ src_nents = sg_nents_for_len(req->src, req->assoclen + -+ req->cryptlen); -+ if (unlikely(src_nents < 0)) { -+ dev_err(dev, "Insufficient bytes (%d) in src S/G\n", -+ req->assoclen + req->cryptlen); -+ qi_cache_free(edesc); -+ return ERR_PTR(src_nents); -+ } -+ -+ dst = scatterwalk_ffwd(edesc->tmp, req->dst, req->assoclen); -+ dst_nents = sg_nents_for_len(dst, req->cryptlen + -+ (encrypt ? authsize : 0)); -+ if (unlikely(dst_nents < 0)) { -+ dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", -+ req->cryptlen + -+ (encrypt ? authsize : 0)); -+ qi_cache_free(edesc); -+ return ERR_PTR(dst_nents); -+ } -+ -+ if (src_nents) { -+ mapped_src_nents = dma_map_sg(dev, req->src, -+ src_nents, DMA_TO_DEVICE); -+ if (unlikely(!mapped_src_nents)) { -+ dev_err(dev, "unable to map source\n"); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ } else { -+ mapped_src_nents = 0; -+ } -+ -+ mapped_dst_nents = dma_map_sg(dev, dst, dst_nents, -+ DMA_FROM_DEVICE); -+ if (unlikely(!mapped_dst_nents)) { -+ dev_err(dev, "unable to map destination\n"); -+ dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ } -+ -+ ivsize = crypto_aead_ivsize(tls); -+ iv_dma = dma_map_single(dev, req->iv, ivsize, DMA_TO_DEVICE); -+ if (dma_mapping_error(dev, iv_dma)) { -+ dev_err(dev, "unable to map IV\n"); -+ caam_unmap(dev, req->src, dst, src_nents, dst_nents, 0, 0, -+ op_type, 0, 0); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ /* -+ * Create S/G table: IV, src, dst. -+ * Input is not contiguous. -+ */ -+ qm_sg_ents = 1 + mapped_src_nents + -+ (mapped_dst_nents > 1 ? mapped_dst_nents : 0); -+ sg_table = &edesc->sgt[0]; -+ qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); -+ -+ edesc->src_nents = src_nents; -+ edesc->dst_nents = dst_nents; -+ edesc->dst = dst; -+ edesc->iv_dma = iv_dma; -+ -+ dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); -+ qm_sg_index = 1; -+ -+ sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0); -+ qm_sg_index += mapped_src_nents; -+ -+ if (mapped_dst_nents > 1) -+ sg_to_qm_sg_last(dst, mapped_dst_nents, sg_table + -+ qm_sg_index, 0); -+ -+ qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); -+ if (dma_mapping_error(dev, qm_sg_dma)) { -+ dev_err(dev, "unable to map S/G table\n"); -+ caam_unmap(dev, req->src, dst, src_nents, dst_nents, iv_dma, -+ ivsize, op_type, 0, 0); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ edesc->qm_sg_dma = qm_sg_dma; -+ edesc->qm_sg_bytes = qm_sg_bytes; -+ -+ out_len = req->cryptlen + (encrypt ? authsize : 0); -+ in_len = ivsize + req->assoclen + req->cryptlen; -+ -+ memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); -+ dpaa2_fl_set_final(in_fle, true); -+ dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); -+ dpaa2_fl_set_addr(in_fle, qm_sg_dma); -+ dpaa2_fl_set_len(in_fle, in_len); -+ -+ if (req->dst == req->src) { -+ dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); -+ dpaa2_fl_set_addr(out_fle, qm_sg_dma + -+ (sg_nents_for_len(req->src, req->assoclen) + -+ 1) * sizeof(*sg_table)); -+ } else if (mapped_dst_nents == 1) { -+ dpaa2_fl_set_format(out_fle, dpaa2_fl_single); -+ dpaa2_fl_set_addr(out_fle, sg_dma_address(dst)); -+ } else { -+ dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); -+ dpaa2_fl_set_addr(out_fle, qm_sg_dma + qm_sg_index * -+ sizeof(*sg_table)); -+ } -+ -+ dpaa2_fl_set_len(out_fle, out_len); -+ -+ return edesc; -+} -+ -+static int tls_set_sh_desc(struct crypto_aead *tls) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(tls); -+ unsigned int ivsize = crypto_aead_ivsize(tls); -+ unsigned int blocksize = crypto_aead_blocksize(tls); -+ struct device *dev = ctx->dev; -+ struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); -+ struct caam_flc *flc; -+ u32 *desc; -+ unsigned int assoclen = 13; /* always 13 bytes for TLS */ -+ unsigned int data_len[2]; -+ u32 inl_mask; -+ -+ if (!ctx->cdata.keylen || !ctx->authsize) -+ return 0; -+ -+ /* -+ * TLS 1.0 encrypt shared descriptor -+ * Job Descriptor and Shared Descriptor -+ * must fit into the 64-word Descriptor h/w Buffer -+ */ -+ data_len[0] = ctx->adata.keylen_pad; -+ data_len[1] = ctx->cdata.keylen; -+ -+ if (desc_inline_query(DESC_TLS10_ENC_LEN, DESC_JOB_IO_LEN, data_len, -+ &inl_mask, ARRAY_SIZE(data_len)) < 0) -+ return -EINVAL; -+ -+ if (inl_mask & 1) -+ ctx->adata.key_virt = ctx->key; -+ else -+ ctx->adata.key_dma = ctx->key_dma; -+ -+ if (inl_mask & 2) -+ ctx->cdata.key_virt = ctx->key + ctx->adata.keylen_pad; -+ else -+ ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; -+ -+ ctx->adata.key_inline = !!(inl_mask & 1); -+ ctx->cdata.key_inline = !!(inl_mask & 2); -+ -+ flc = &ctx->flc[ENCRYPT]; -+ desc = flc->sh_desc; -+ cnstr_shdsc_tls_encap(desc, &ctx->cdata, &ctx->adata, -+ assoclen, ivsize, ctx->authsize, blocksize, -+ priv->sec_attr.era); -+ flc->flc[1] = cpu_to_caam32(desc_len(desc)); -+ dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], -+ sizeof(flc->flc) + desc_bytes(desc), -+ ctx->dir); -+ -+ /* -+ * TLS 1.0 decrypt shared descriptor -+ * Keys do not fit inline, regardless of algorithms used -+ */ -+ ctx->adata.key_inline = false; -+ ctx->adata.key_dma = ctx->key_dma; -+ ctx->cdata.key_dma = ctx->key_dma + ctx->adata.keylen_pad; -+ -+ flc = &ctx->flc[DECRYPT]; -+ desc = flc->sh_desc; -+ cnstr_shdsc_tls_decap(desc, &ctx->cdata, &ctx->adata, assoclen, ivsize, -+ ctx->authsize, blocksize, priv->sec_attr.era); -+ flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ -+ dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], -+ sizeof(flc->flc) + desc_bytes(desc), -+ ctx->dir); -+ -+ return 0; -+} -+ -+static int tls_setkey(struct crypto_aead *tls, const u8 *key, -+ unsigned int keylen) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(tls); -+ struct device *dev = ctx->dev; -+ struct crypto_authenc_keys keys; -+ -+ if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) -+ goto badkey; -+ -+#ifdef DEBUG -+ dev_err(dev, "keylen %d enckeylen %d authkeylen %d\n", -+ keys.authkeylen + keys.enckeylen, keys.enckeylen, -+ keys.authkeylen); -+ print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -+#endif -+ -+ ctx->adata.keylen = keys.authkeylen; -+ ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & -+ OP_ALG_ALGSEL_MASK); -+ -+ if (ctx->adata.keylen_pad + keys.enckeylen > CAAM_MAX_KEY_SIZE) -+ goto badkey; -+ -+ memcpy(ctx->key, keys.authkey, keys.authkeylen); -+ memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); -+ dma_sync_single_for_device(dev, ctx->key_dma, ctx->adata.keylen_pad + -+ keys.enckeylen, ctx->dir); -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, -+ ctx->adata.keylen_pad + keys.enckeylen, 1); -+#endif -+ -+ ctx->cdata.keylen = keys.enckeylen; -+ -+ return tls_set_sh_desc(tls); -+badkey: -+ crypto_aead_set_flags(tls, CRYPTO_TFM_RES_BAD_KEY_LEN); -+ return -EINVAL; -+} -+ -+static int tls_setauthsize(struct crypto_aead *tls, unsigned int authsize) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(tls); -+ -+ ctx->authsize = authsize; -+ tls_set_sh_desc(tls); -+ -+ return 0; -+} -+ -+static int gcm_set_sh_desc(struct crypto_aead *aead) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ struct device *dev = ctx->dev; -+ unsigned int ivsize = crypto_aead_ivsize(aead); -+ struct caam_flc *flc; -+ u32 *desc; -+ int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - -+ ctx->cdata.keylen; -+ -+ if (!ctx->cdata.keylen || !ctx->authsize) -+ return 0; -+ -+ /* -+ * AES GCM encrypt shared descriptor -+ * Job Descriptor and Shared Descriptor -+ * must fit into the 64-word Descriptor h/w Buffer -+ */ -+ if (rem_bytes >= DESC_QI_GCM_ENC_LEN) { -+ ctx->cdata.key_inline = true; -+ ctx->cdata.key_virt = ctx->key; -+ } else { -+ ctx->cdata.key_inline = false; -+ ctx->cdata.key_dma = ctx->key_dma; -+ } -+ -+ flc = &ctx->flc[ENCRYPT]; -+ desc = flc->sh_desc; -+ cnstr_shdsc_gcm_encap(desc, &ctx->cdata, ivsize, ctx->authsize, true); -+ flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ -+ dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], -+ sizeof(flc->flc) + desc_bytes(desc), -+ ctx->dir); -+ -+ /* -+ * Job Descriptor and Shared Descriptors -+ * must all fit into the 64-word Descriptor h/w Buffer -+ */ -+ if (rem_bytes >= DESC_QI_GCM_DEC_LEN) { -+ ctx->cdata.key_inline = true; -+ ctx->cdata.key_virt = ctx->key; -+ } else { -+ ctx->cdata.key_inline = false; -+ ctx->cdata.key_dma = ctx->key_dma; -+ } -+ -+ flc = &ctx->flc[DECRYPT]; -+ desc = flc->sh_desc; -+ cnstr_shdsc_gcm_decap(desc, &ctx->cdata, ivsize, ctx->authsize, true); -+ flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ -+ dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], -+ sizeof(flc->flc) + desc_bytes(desc), -+ ctx->dir); -+ -+ return 0; -+} -+ -+static int gcm_setauthsize(struct crypto_aead *authenc, unsigned int authsize) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(authenc); -+ -+ ctx->authsize = authsize; -+ gcm_set_sh_desc(authenc); -+ -+ return 0; -+} -+ -+static int gcm_setkey(struct crypto_aead *aead, -+ const u8 *key, unsigned int keylen) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ struct device *dev = ctx->dev; -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -+#endif -+ -+ memcpy(ctx->key, key, keylen); -+ dma_sync_single_for_device(dev, ctx->key_dma, keylen, ctx->dir); -+ ctx->cdata.keylen = keylen; -+ -+ return gcm_set_sh_desc(aead); -+} -+ -+static int rfc4106_set_sh_desc(struct crypto_aead *aead) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ struct device *dev = ctx->dev; -+ unsigned int ivsize = crypto_aead_ivsize(aead); -+ struct caam_flc *flc; -+ u32 *desc; -+ int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - -+ ctx->cdata.keylen; -+ -+ if (!ctx->cdata.keylen || !ctx->authsize) -+ return 0; -+ -+ ctx->cdata.key_virt = ctx->key; -+ -+ /* -+ * RFC4106 encrypt shared descriptor -+ * Job Descriptor and Shared Descriptor -+ * must fit into the 64-word Descriptor h/w Buffer -+ */ -+ if (rem_bytes >= DESC_QI_RFC4106_ENC_LEN) { -+ ctx->cdata.key_inline = true; -+ } else { -+ ctx->cdata.key_inline = false; -+ ctx->cdata.key_dma = ctx->key_dma; -+ } -+ -+ flc = &ctx->flc[ENCRYPT]; -+ desc = flc->sh_desc; -+ cnstr_shdsc_rfc4106_encap(desc, &ctx->cdata, ivsize, ctx->authsize, -+ true); -+ flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ -+ dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], -+ sizeof(flc->flc) + desc_bytes(desc), -+ ctx->dir); -+ -+ /* -+ * Job Descriptor and Shared Descriptors -+ * must all fit into the 64-word Descriptor h/w Buffer -+ */ -+ if (rem_bytes >= DESC_QI_RFC4106_DEC_LEN) { -+ ctx->cdata.key_inline = true; -+ } else { -+ ctx->cdata.key_inline = false; -+ ctx->cdata.key_dma = ctx->key_dma; -+ } -+ -+ flc = &ctx->flc[DECRYPT]; -+ desc = flc->sh_desc; -+ cnstr_shdsc_rfc4106_decap(desc, &ctx->cdata, ivsize, ctx->authsize, -+ true); -+ flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ -+ dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], -+ sizeof(flc->flc) + desc_bytes(desc), -+ ctx->dir); -+ -+ return 0; -+} -+ -+static int rfc4106_setauthsize(struct crypto_aead *authenc, -+ unsigned int authsize) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(authenc); -+ -+ ctx->authsize = authsize; -+ rfc4106_set_sh_desc(authenc); -+ -+ return 0; -+} -+ -+static int rfc4106_setkey(struct crypto_aead *aead, -+ const u8 *key, unsigned int keylen) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ struct device *dev = ctx->dev; -+ -+ if (keylen < 4) -+ return -EINVAL; -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -+#endif -+ -+ memcpy(ctx->key, key, keylen); -+ /* -+ * The last four bytes of the key material are used as the salt value -+ * in the nonce. Update the AES key length. -+ */ -+ ctx->cdata.keylen = keylen - 4; -+ dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, -+ ctx->dir); -+ -+ return rfc4106_set_sh_desc(aead); -+} -+ -+static int rfc4543_set_sh_desc(struct crypto_aead *aead) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ struct device *dev = ctx->dev; -+ unsigned int ivsize = crypto_aead_ivsize(aead); -+ struct caam_flc *flc; -+ u32 *desc; -+ int rem_bytes = CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN - -+ ctx->cdata.keylen; -+ -+ if (!ctx->cdata.keylen || !ctx->authsize) -+ return 0; -+ -+ ctx->cdata.key_virt = ctx->key; -+ -+ /* -+ * RFC4543 encrypt shared descriptor -+ * Job Descriptor and Shared Descriptor -+ * must fit into the 64-word Descriptor h/w Buffer -+ */ -+ if (rem_bytes >= DESC_QI_RFC4543_ENC_LEN) { -+ ctx->cdata.key_inline = true; -+ } else { -+ ctx->cdata.key_inline = false; -+ ctx->cdata.key_dma = ctx->key_dma; -+ } -+ -+ flc = &ctx->flc[ENCRYPT]; -+ desc = flc->sh_desc; -+ cnstr_shdsc_rfc4543_encap(desc, &ctx->cdata, ivsize, ctx->authsize, -+ true); -+ flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ -+ dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], -+ sizeof(flc->flc) + desc_bytes(desc), -+ ctx->dir); -+ -+ /* -+ * Job Descriptor and Shared Descriptors -+ * must all fit into the 64-word Descriptor h/w Buffer -+ */ -+ if (rem_bytes >= DESC_QI_RFC4543_DEC_LEN) { -+ ctx->cdata.key_inline = true; -+ } else { -+ ctx->cdata.key_inline = false; -+ ctx->cdata.key_dma = ctx->key_dma; -+ } -+ -+ flc = &ctx->flc[DECRYPT]; -+ desc = flc->sh_desc; -+ cnstr_shdsc_rfc4543_decap(desc, &ctx->cdata, ivsize, ctx->authsize, -+ true); -+ flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ -+ dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], -+ sizeof(flc->flc) + desc_bytes(desc), -+ ctx->dir); -+ -+ return 0; -+} -+ -+static int rfc4543_setauthsize(struct crypto_aead *authenc, -+ unsigned int authsize) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(authenc); -+ -+ ctx->authsize = authsize; -+ rfc4543_set_sh_desc(authenc); -+ -+ return 0; -+} -+ -+static int rfc4543_setkey(struct crypto_aead *aead, -+ const u8 *key, unsigned int keylen) -+{ -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ struct device *dev = ctx->dev; -+ -+ if (keylen < 4) -+ return -EINVAL; -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -+#endif -+ -+ memcpy(ctx->key, key, keylen); -+ /* -+ * The last four bytes of the key material are used as the salt value -+ * in the nonce. Update the AES key length. -+ */ -+ ctx->cdata.keylen = keylen - 4; -+ dma_sync_single_for_device(dev, ctx->key_dma, ctx->cdata.keylen, -+ ctx->dir); -+ -+ return rfc4543_set_sh_desc(aead); -+} -+ -+static int ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, -+ const u8 *key, unsigned int keylen) -+{ -+ struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); -+ struct crypto_tfm *tfm = crypto_ablkcipher_tfm(ablkcipher); -+ const char *alg_name = crypto_tfm_alg_name(tfm); -+ struct device *dev = ctx->dev; -+ struct caam_flc *flc; -+ unsigned int ivsize = crypto_ablkcipher_ivsize(ablkcipher); -+ u32 *desc; -+ u32 ctx1_iv_off = 0; -+ const bool ctr_mode = ((ctx->cdata.algtype & OP_ALG_AAI_MASK) == -+ OP_ALG_AAI_CTR_MOD128); -+ const bool is_rfc3686 = (ctr_mode && strstr(alg_name, "rfc3686")); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -+#endif -+ /* -+ * AES-CTR needs to load IV in CONTEXT1 reg -+ * at an offset of 128bits (16bytes) -+ * CONTEXT1[255:128] = IV -+ */ -+ if (ctr_mode) -+ ctx1_iv_off = 16; -+ -+ /* -+ * RFC3686 specific: -+ * | CONTEXT1[255:128] = {NONCE, IV, COUNTER} -+ * | *key = {KEY, NONCE} -+ */ -+ if (is_rfc3686) { -+ ctx1_iv_off = 16 + CTR_RFC3686_NONCE_SIZE; -+ keylen -= CTR_RFC3686_NONCE_SIZE; -+ } -+ -+ ctx->cdata.keylen = keylen; -+ ctx->cdata.key_virt = key; -+ ctx->cdata.key_inline = true; -+ -+ /* ablkcipher_encrypt shared descriptor */ -+ flc = &ctx->flc[ENCRYPT]; -+ desc = flc->sh_desc; -+ cnstr_shdsc_ablkcipher_encap(desc, &ctx->cdata, ivsize, -+ is_rfc3686, ctx1_iv_off); -+ flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ -+ dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], -+ sizeof(flc->flc) + desc_bytes(desc), -+ ctx->dir); -+ -+ /* ablkcipher_decrypt shared descriptor */ -+ flc = &ctx->flc[DECRYPT]; -+ desc = flc->sh_desc; -+ cnstr_shdsc_ablkcipher_decap(desc, &ctx->cdata, ivsize, -+ is_rfc3686, ctx1_iv_off); -+ flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ -+ dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], -+ sizeof(flc->flc) + desc_bytes(desc), -+ ctx->dir); -+ -+ /* ablkcipher_givencrypt shared descriptor */ -+ flc = &ctx->flc[GIVENCRYPT]; -+ desc = flc->sh_desc; -+ cnstr_shdsc_ablkcipher_givencap(desc, &ctx->cdata, -+ ivsize, is_rfc3686, ctx1_iv_off); -+ flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ -+ dma_sync_single_for_device(dev, ctx->flc_dma[GIVENCRYPT], -+ sizeof(flc->flc) + desc_bytes(desc), -+ ctx->dir); -+ -+ return 0; -+} -+ -+static int xts_ablkcipher_setkey(struct crypto_ablkcipher *ablkcipher, -+ const u8 *key, unsigned int keylen) -+{ -+ struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); -+ struct device *dev = ctx->dev; -+ struct caam_flc *flc; -+ u32 *desc; -+ -+ if (keylen != 2 * AES_MIN_KEY_SIZE && keylen != 2 * AES_MAX_KEY_SIZE) { -+ dev_err(dev, "key size mismatch\n"); -+ crypto_ablkcipher_set_flags(ablkcipher, -+ CRYPTO_TFM_RES_BAD_KEY_LEN); -+ return -EINVAL; -+ } -+ -+ ctx->cdata.keylen = keylen; -+ ctx->cdata.key_virt = key; -+ ctx->cdata.key_inline = true; -+ -+ /* xts_ablkcipher_encrypt shared descriptor */ -+ flc = &ctx->flc[ENCRYPT]; -+ desc = flc->sh_desc; -+ cnstr_shdsc_xts_ablkcipher_encap(desc, &ctx->cdata); -+ flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ -+ dma_sync_single_for_device(dev, ctx->flc_dma[ENCRYPT], -+ sizeof(flc->flc) + desc_bytes(desc), -+ ctx->dir); -+ -+ /* xts_ablkcipher_decrypt shared descriptor */ -+ flc = &ctx->flc[DECRYPT]; -+ desc = flc->sh_desc; -+ cnstr_shdsc_xts_ablkcipher_decap(desc, &ctx->cdata); -+ flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ -+ dma_sync_single_for_device(dev, ctx->flc_dma[DECRYPT], -+ sizeof(flc->flc) + desc_bytes(desc), -+ ctx->dir); -+ -+ return 0; -+} -+ -+static struct ablkcipher_edesc *ablkcipher_edesc_alloc(struct ablkcipher_request -+ *req, bool encrypt) -+{ -+ struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); -+ struct caam_request *req_ctx = ablkcipher_request_ctx(req); -+ struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; -+ struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; -+ struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); -+ struct device *dev = ctx->dev; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; -+ int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; -+ struct ablkcipher_edesc *edesc; -+ dma_addr_t iv_dma; -+ bool in_contig; -+ int ivsize = crypto_ablkcipher_ivsize(ablkcipher); -+ int dst_sg_idx, qm_sg_ents; -+ struct dpaa2_sg_entry *sg_table; -+ enum optype op_type = encrypt ? ENCRYPT : DECRYPT; -+ -+ src_nents = sg_nents_for_len(req->src, req->nbytes); -+ if (unlikely(src_nents < 0)) { -+ dev_err(dev, "Insufficient bytes (%d) in src S/G\n", -+ req->nbytes); -+ return ERR_PTR(src_nents); -+ } -+ -+ if (unlikely(req->dst != req->src)) { -+ dst_nents = sg_nents_for_len(req->dst, req->nbytes); -+ if (unlikely(dst_nents < 0)) { -+ dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", -+ req->nbytes); -+ return ERR_PTR(dst_nents); -+ } -+ -+ mapped_src_nents = dma_map_sg(dev, req->src, src_nents, -+ DMA_TO_DEVICE); -+ if (unlikely(!mapped_src_nents)) { -+ dev_err(dev, "unable to map source\n"); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, -+ DMA_FROM_DEVICE); -+ if (unlikely(!mapped_dst_nents)) { -+ dev_err(dev, "unable to map destination\n"); -+ dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); -+ return ERR_PTR(-ENOMEM); -+ } -+ } else { -+ mapped_src_nents = dma_map_sg(dev, req->src, src_nents, -+ DMA_BIDIRECTIONAL); -+ if (unlikely(!mapped_src_nents)) { -+ dev_err(dev, "unable to map source\n"); -+ return ERR_PTR(-ENOMEM); -+ } -+ } -+ -+ iv_dma = dma_map_single(dev, req->info, ivsize, DMA_TO_DEVICE); -+ if (dma_mapping_error(dev, iv_dma)) { -+ dev_err(dev, "unable to map IV\n"); -+ caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, -+ 0, 0, 0, 0); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ if (mapped_src_nents == 1 && -+ iv_dma + ivsize == sg_dma_address(req->src)) { -+ in_contig = true; -+ qm_sg_ents = 0; -+ } else { -+ in_contig = false; -+ qm_sg_ents = 1 + mapped_src_nents; -+ } -+ dst_sg_idx = qm_sg_ents; -+ -+ qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; -+ if (unlikely(qm_sg_ents > CAAM_QI_MAX_ABLKCIPHER_SG)) { -+ dev_err(dev, "Insufficient S/G entries: %d > %lu\n", -+ qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG); -+ caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, -+ iv_dma, ivsize, op_type, 0, 0); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ /* allocate space for base edesc and link tables */ -+ edesc = qi_cache_zalloc(GFP_DMA | flags); -+ if (unlikely(!edesc)) { -+ dev_err(dev, "could not allocate extended descriptor\n"); -+ caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, -+ iv_dma, ivsize, op_type, 0, 0); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ edesc->src_nents = src_nents; -+ edesc->dst_nents = dst_nents; -+ edesc->iv_dma = iv_dma; -+ sg_table = &edesc->sgt[0]; -+ edesc->qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); -+ -+ if (!in_contig) { -+ dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); -+ sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0); -+ } -+ -+ if (mapped_dst_nents > 1) -+ sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + -+ dst_sg_idx, 0); -+ -+ edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes, -+ DMA_TO_DEVICE); -+ if (dma_mapping_error(dev, edesc->qm_sg_dma)) { -+ dev_err(dev, "unable to map S/G table\n"); -+ caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, -+ iv_dma, ivsize, op_type, 0, 0); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); -+ dpaa2_fl_set_final(in_fle, true); -+ dpaa2_fl_set_len(in_fle, req->nbytes + ivsize); -+ dpaa2_fl_set_len(out_fle, req->nbytes); -+ -+ if (!in_contig) { -+ dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); -+ dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); -+ } else { -+ dpaa2_fl_set_format(in_fle, dpaa2_fl_single); -+ dpaa2_fl_set_addr(in_fle, iv_dma); -+ } -+ -+ if (req->src == req->dst) { -+ if (!in_contig) { -+ dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); -+ dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + -+ sizeof(*sg_table)); -+ } else { -+ dpaa2_fl_set_format(out_fle, dpaa2_fl_single); -+ dpaa2_fl_set_addr(out_fle, sg_dma_address(req->src)); -+ } -+ } else if (mapped_dst_nents > 1) { -+ dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); -+ dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx * -+ sizeof(*sg_table)); -+ } else { -+ dpaa2_fl_set_format(out_fle, dpaa2_fl_single); -+ dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); -+ } -+ -+ return edesc; -+} -+ -+static struct ablkcipher_edesc *ablkcipher_giv_edesc_alloc( -+ struct skcipher_givcrypt_request *greq) -+{ -+ struct ablkcipher_request *req = &greq->creq; -+ struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); -+ struct caam_request *req_ctx = ablkcipher_request_ctx(req); -+ struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; -+ struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; -+ struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); -+ struct device *dev = ctx->dev; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; -+ int src_nents, mapped_src_nents, dst_nents, mapped_dst_nents; -+ struct ablkcipher_edesc *edesc; -+ dma_addr_t iv_dma; -+ bool out_contig; -+ int ivsize = crypto_ablkcipher_ivsize(ablkcipher); -+ struct dpaa2_sg_entry *sg_table; -+ int dst_sg_idx, qm_sg_ents; -+ -+ src_nents = sg_nents_for_len(req->src, req->nbytes); -+ if (unlikely(src_nents < 0)) { -+ dev_err(dev, "Insufficient bytes (%d) in src S/G\n", -+ req->nbytes); -+ return ERR_PTR(src_nents); -+ } -+ -+ if (unlikely(req->dst != req->src)) { -+ dst_nents = sg_nents_for_len(req->dst, req->nbytes); -+ if (unlikely(dst_nents < 0)) { -+ dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", -+ req->nbytes); -+ return ERR_PTR(dst_nents); -+ } -+ -+ mapped_src_nents = dma_map_sg(dev, req->src, src_nents, -+ DMA_TO_DEVICE); -+ if (unlikely(!mapped_src_nents)) { -+ dev_err(dev, "unable to map source\n"); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ mapped_dst_nents = dma_map_sg(dev, req->dst, dst_nents, -+ DMA_FROM_DEVICE); -+ if (unlikely(!mapped_dst_nents)) { -+ dev_err(dev, "unable to map destination\n"); -+ dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); -+ return ERR_PTR(-ENOMEM); -+ } -+ } else { -+ mapped_src_nents = dma_map_sg(dev, req->src, src_nents, -+ DMA_BIDIRECTIONAL); -+ if (unlikely(!mapped_src_nents)) { -+ dev_err(dev, "unable to map source\n"); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ dst_nents = src_nents; -+ mapped_dst_nents = src_nents; -+ } -+ -+ iv_dma = dma_map_single(dev, greq->giv, ivsize, DMA_FROM_DEVICE); -+ if (dma_mapping_error(dev, iv_dma)) { -+ dev_err(dev, "unable to map IV\n"); -+ caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, -+ 0, 0, 0, 0); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ qm_sg_ents = mapped_src_nents > 1 ? mapped_src_nents : 0; -+ dst_sg_idx = qm_sg_ents; -+ if (mapped_dst_nents == 1 && -+ iv_dma + ivsize == sg_dma_address(req->dst)) { -+ out_contig = true; -+ } else { -+ out_contig = false; -+ qm_sg_ents += 1 + mapped_dst_nents; -+ } -+ -+ if (unlikely(qm_sg_ents > CAAM_QI_MAX_ABLKCIPHER_SG)) { -+ dev_err(dev, "Insufficient S/G entries: %d > %lu\n", -+ qm_sg_ents, CAAM_QI_MAX_ABLKCIPHER_SG); -+ caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, -+ iv_dma, ivsize, GIVENCRYPT, 0, 0); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ /* allocate space for base edesc and link tables */ -+ edesc = qi_cache_zalloc(GFP_DMA | flags); -+ if (!edesc) { -+ dev_err(dev, "could not allocate extended descriptor\n"); -+ caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, -+ iv_dma, ivsize, GIVENCRYPT, 0, 0); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ edesc->src_nents = src_nents; -+ edesc->dst_nents = dst_nents; -+ edesc->iv_dma = iv_dma; -+ sg_table = &edesc->sgt[0]; -+ edesc->qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); -+ -+ if (mapped_src_nents > 1) -+ sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table, 0); -+ -+ if (!out_contig) { -+ dma_to_qm_sg_one(sg_table + dst_sg_idx, iv_dma, ivsize, 0); -+ sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + -+ dst_sg_idx + 1, 0); -+ } -+ -+ edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes, -+ DMA_TO_DEVICE); -+ if (dma_mapping_error(dev, edesc->qm_sg_dma)) { -+ dev_err(dev, "unable to map S/G table\n"); -+ caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, -+ iv_dma, ivsize, GIVENCRYPT, 0, 0); -+ qi_cache_free(edesc); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); -+ dpaa2_fl_set_final(in_fle, true); -+ dpaa2_fl_set_len(in_fle, req->nbytes); -+ dpaa2_fl_set_len(out_fle, ivsize + req->nbytes); -+ -+ if (mapped_src_nents > 1) { -+ dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); -+ dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); -+ } else { -+ dpaa2_fl_set_format(in_fle, dpaa2_fl_single); -+ dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); -+ } -+ -+ if (!out_contig) { -+ dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); -+ dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx * -+ sizeof(*sg_table)); -+ } else { -+ dpaa2_fl_set_format(out_fle, dpaa2_fl_single); -+ dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); -+ } -+ -+ return edesc; -+} -+ -+static void aead_unmap(struct device *dev, struct aead_edesc *edesc, -+ struct aead_request *req) -+{ -+ struct crypto_aead *aead = crypto_aead_reqtfm(req); -+ int ivsize = crypto_aead_ivsize(aead); -+ struct caam_request *caam_req = aead_request_ctx(req); -+ -+ caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, -+ edesc->iv_dma, ivsize, caam_req->op_type, -+ edesc->qm_sg_dma, edesc->qm_sg_bytes); -+ dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); -+} -+ -+static void tls_unmap(struct device *dev, struct tls_edesc *edesc, -+ struct aead_request *req) -+{ -+ struct crypto_aead *tls = crypto_aead_reqtfm(req); -+ int ivsize = crypto_aead_ivsize(tls); -+ struct caam_request *caam_req = aead_request_ctx(req); -+ -+ caam_unmap(dev, req->src, edesc->dst, edesc->src_nents, -+ edesc->dst_nents, edesc->iv_dma, ivsize, caam_req->op_type, -+ edesc->qm_sg_dma, edesc->qm_sg_bytes); -+} -+ -+static void ablkcipher_unmap(struct device *dev, -+ struct ablkcipher_edesc *edesc, -+ struct ablkcipher_request *req) -+{ -+ struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); -+ int ivsize = crypto_ablkcipher_ivsize(ablkcipher); -+ struct caam_request *caam_req = ablkcipher_request_ctx(req); -+ -+ caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, -+ edesc->iv_dma, ivsize, caam_req->op_type, -+ edesc->qm_sg_dma, edesc->qm_sg_bytes); -+} -+ -+static void aead_encrypt_done(void *cbk_ctx, u32 status) -+{ -+ struct crypto_async_request *areq = cbk_ctx; -+ struct aead_request *req = container_of(areq, struct aead_request, -+ base); -+ struct caam_request *req_ctx = to_caam_req(areq); -+ struct aead_edesc *edesc = req_ctx->edesc; -+ struct crypto_aead *aead = crypto_aead_reqtfm(req); -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ int ecode = 0; -+ -+#ifdef DEBUG -+ dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); -+#endif -+ -+ if (unlikely(status)) { -+ caam_qi2_strstatus(ctx->dev, status); -+ ecode = -EIO; -+ } -+ -+ aead_unmap(ctx->dev, edesc, req); -+ qi_cache_free(edesc); -+ aead_request_complete(req, ecode); -+} -+ -+static void aead_decrypt_done(void *cbk_ctx, u32 status) -+{ -+ struct crypto_async_request *areq = cbk_ctx; -+ struct aead_request *req = container_of(areq, struct aead_request, -+ base); -+ struct caam_request *req_ctx = to_caam_req(areq); -+ struct aead_edesc *edesc = req_ctx->edesc; -+ struct crypto_aead *aead = crypto_aead_reqtfm(req); -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ int ecode = 0; -+ -+#ifdef DEBUG -+ dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); -+#endif -+ -+ if (unlikely(status)) { -+ caam_qi2_strstatus(ctx->dev, status); -+ /* -+ * verify hw auth check passed else return -EBADMSG -+ */ -+ if ((status & JRSTA_CCBERR_ERRID_MASK) == -+ JRSTA_CCBERR_ERRID_ICVCHK) -+ ecode = -EBADMSG; -+ else -+ ecode = -EIO; -+ } -+ -+ aead_unmap(ctx->dev, edesc, req); -+ qi_cache_free(edesc); -+ aead_request_complete(req, ecode); -+} -+ -+static int aead_encrypt(struct aead_request *req) -+{ -+ struct aead_edesc *edesc; -+ struct crypto_aead *aead = crypto_aead_reqtfm(req); -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ struct caam_request *caam_req = aead_request_ctx(req); -+ int ret; -+ -+ /* allocate extended descriptor */ -+ edesc = aead_edesc_alloc(req, true); -+ if (IS_ERR(edesc)) -+ return PTR_ERR(edesc); -+ -+ caam_req->flc = &ctx->flc[ENCRYPT]; -+ caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; -+ caam_req->op_type = ENCRYPT; -+ caam_req->cbk = aead_encrypt_done; -+ caam_req->ctx = &req->base; -+ caam_req->edesc = edesc; -+ ret = dpaa2_caam_enqueue(ctx->dev, caam_req); -+ if (ret != -EINPROGRESS && -+ !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { -+ aead_unmap(ctx->dev, edesc, req); -+ qi_cache_free(edesc); -+ } -+ -+ return ret; -+} -+ -+static int aead_decrypt(struct aead_request *req) -+{ -+ struct aead_edesc *edesc; -+ struct crypto_aead *aead = crypto_aead_reqtfm(req); -+ struct caam_ctx *ctx = crypto_aead_ctx(aead); -+ struct caam_request *caam_req = aead_request_ctx(req); -+ int ret; -+ -+ /* allocate extended descriptor */ -+ edesc = aead_edesc_alloc(req, false); -+ if (IS_ERR(edesc)) -+ return PTR_ERR(edesc); -+ -+ caam_req->flc = &ctx->flc[DECRYPT]; -+ caam_req->flc_dma = ctx->flc_dma[DECRYPT]; -+ caam_req->op_type = DECRYPT; -+ caam_req->cbk = aead_decrypt_done; -+ caam_req->ctx = &req->base; -+ caam_req->edesc = edesc; -+ ret = dpaa2_caam_enqueue(ctx->dev, caam_req); -+ if (ret != -EINPROGRESS && -+ !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { -+ aead_unmap(ctx->dev, edesc, req); -+ qi_cache_free(edesc); -+ } -+ -+ return ret; -+} -+ -+static void tls_encrypt_done(void *cbk_ctx, u32 status) -+{ -+ struct crypto_async_request *areq = cbk_ctx; -+ struct aead_request *req = container_of(areq, struct aead_request, -+ base); -+ struct caam_request *req_ctx = to_caam_req(areq); -+ struct tls_edesc *edesc = req_ctx->edesc; -+ struct crypto_aead *tls = crypto_aead_reqtfm(req); -+ struct caam_ctx *ctx = crypto_aead_ctx(tls); -+ int ecode = 0; -+ -+#ifdef DEBUG -+ dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); -+#endif -+ -+ if (unlikely(status)) { -+ caam_qi2_strstatus(ctx->dev, status); -+ ecode = -EIO; -+ } -+ -+ tls_unmap(ctx->dev, edesc, req); -+ qi_cache_free(edesc); -+ aead_request_complete(req, ecode); -+} -+ -+static void tls_decrypt_done(void *cbk_ctx, u32 status) -+{ -+ struct crypto_async_request *areq = cbk_ctx; -+ struct aead_request *req = container_of(areq, struct aead_request, -+ base); -+ struct caam_request *req_ctx = to_caam_req(areq); -+ struct tls_edesc *edesc = req_ctx->edesc; -+ struct crypto_aead *tls = crypto_aead_reqtfm(req); -+ struct caam_ctx *ctx = crypto_aead_ctx(tls); -+ int ecode = 0; -+ -+#ifdef DEBUG -+ dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); -+#endif -+ -+ if (unlikely(status)) { -+ caam_qi2_strstatus(ctx->dev, status); -+ /* -+ * verify hw auth check passed else return -EBADMSG -+ */ -+ if ((status & JRSTA_CCBERR_ERRID_MASK) == -+ JRSTA_CCBERR_ERRID_ICVCHK) -+ ecode = -EBADMSG; -+ else -+ ecode = -EIO; -+ } -+ -+ tls_unmap(ctx->dev, edesc, req); -+ qi_cache_free(edesc); -+ aead_request_complete(req, ecode); -+} -+ -+static int tls_encrypt(struct aead_request *req) -+{ -+ struct tls_edesc *edesc; -+ struct crypto_aead *tls = crypto_aead_reqtfm(req); -+ struct caam_ctx *ctx = crypto_aead_ctx(tls); -+ struct caam_request *caam_req = aead_request_ctx(req); -+ int ret; -+ -+ /* allocate extended descriptor */ -+ edesc = tls_edesc_alloc(req, true); -+ if (IS_ERR(edesc)) -+ return PTR_ERR(edesc); -+ -+ caam_req->flc = &ctx->flc[ENCRYPT]; -+ caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; -+ caam_req->op_type = ENCRYPT; -+ caam_req->cbk = tls_encrypt_done; -+ caam_req->ctx = &req->base; -+ caam_req->edesc = edesc; -+ ret = dpaa2_caam_enqueue(ctx->dev, caam_req); -+ if (ret != -EINPROGRESS && -+ !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { -+ tls_unmap(ctx->dev, edesc, req); -+ qi_cache_free(edesc); -+ } -+ -+ return ret; -+} -+ -+static int tls_decrypt(struct aead_request *req) -+{ -+ struct tls_edesc *edesc; -+ struct crypto_aead *tls = crypto_aead_reqtfm(req); -+ struct caam_ctx *ctx = crypto_aead_ctx(tls); -+ struct caam_request *caam_req = aead_request_ctx(req); -+ int ret; -+ -+ /* allocate extended descriptor */ -+ edesc = tls_edesc_alloc(req, false); -+ if (IS_ERR(edesc)) -+ return PTR_ERR(edesc); -+ -+ caam_req->flc = &ctx->flc[DECRYPT]; -+ caam_req->flc_dma = ctx->flc_dma[DECRYPT]; -+ caam_req->op_type = DECRYPT; -+ caam_req->cbk = tls_decrypt_done; -+ caam_req->ctx = &req->base; -+ caam_req->edesc = edesc; -+ ret = dpaa2_caam_enqueue(ctx->dev, caam_req); -+ if (ret != -EINPROGRESS && -+ !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { -+ tls_unmap(ctx->dev, edesc, req); -+ qi_cache_free(edesc); -+ } -+ -+ return ret; -+} -+ -+static int ipsec_gcm_encrypt(struct aead_request *req) -+{ -+ if (req->assoclen < 8) -+ return -EINVAL; -+ -+ return aead_encrypt(req); -+} -+ -+static int ipsec_gcm_decrypt(struct aead_request *req) -+{ -+ if (req->assoclen < 8) -+ return -EINVAL; -+ -+ return aead_decrypt(req); -+} -+ -+static void ablkcipher_done(void *cbk_ctx, u32 status) -+{ -+ struct crypto_async_request *areq = cbk_ctx; -+ struct ablkcipher_request *req = ablkcipher_request_cast(areq); -+ struct caam_request *req_ctx = to_caam_req(areq); -+ struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); -+ struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); -+ struct ablkcipher_edesc *edesc = req_ctx->edesc; -+ int ecode = 0; -+ int ivsize = crypto_ablkcipher_ivsize(ablkcipher); -+ -+#ifdef DEBUG -+ dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); -+#endif -+ -+ if (unlikely(status)) { -+ caam_qi2_strstatus(ctx->dev, status); -+ ecode = -EIO; -+ } -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "dstiv @" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, req->info, -+ edesc->src_nents > 1 ? 100 : ivsize, 1); -+ caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, req->dst, -+ edesc->dst_nents > 1 ? 100 : req->nbytes, 1); -+#endif -+ -+ ablkcipher_unmap(ctx->dev, edesc, req); -+ qi_cache_free(edesc); -+ -+ /* -+ * The crypto API expects us to set the IV (req->info) to the last -+ * ciphertext block. This is used e.g. by the CTS mode. -+ */ -+ scatterwalk_map_and_copy(req->info, req->dst, req->nbytes - ivsize, -+ ivsize, 0); -+ -+ ablkcipher_request_complete(req, ecode); -+} -+ -+static int ablkcipher_encrypt(struct ablkcipher_request *req) -+{ -+ struct ablkcipher_edesc *edesc; -+ struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); -+ struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); -+ struct caam_request *caam_req = ablkcipher_request_ctx(req); -+ int ret; -+ -+ /* allocate extended descriptor */ -+ edesc = ablkcipher_edesc_alloc(req, true); -+ if (IS_ERR(edesc)) -+ return PTR_ERR(edesc); -+ -+ caam_req->flc = &ctx->flc[ENCRYPT]; -+ caam_req->flc_dma = ctx->flc_dma[ENCRYPT]; -+ caam_req->op_type = ENCRYPT; -+ caam_req->cbk = ablkcipher_done; -+ caam_req->ctx = &req->base; -+ caam_req->edesc = edesc; -+ ret = dpaa2_caam_enqueue(ctx->dev, caam_req); -+ if (ret != -EINPROGRESS && -+ !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { -+ ablkcipher_unmap(ctx->dev, edesc, req); -+ qi_cache_free(edesc); -+ } -+ -+ return ret; -+} -+ -+static int ablkcipher_givencrypt(struct skcipher_givcrypt_request *greq) -+{ -+ struct ablkcipher_request *req = &greq->creq; -+ struct ablkcipher_edesc *edesc; -+ struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); -+ struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); -+ struct caam_request *caam_req = ablkcipher_request_ctx(req); -+ int ret; -+ -+ /* allocate extended descriptor */ -+ edesc = ablkcipher_giv_edesc_alloc(greq); -+ if (IS_ERR(edesc)) -+ return PTR_ERR(edesc); -+ -+ caam_req->flc = &ctx->flc[GIVENCRYPT]; -+ caam_req->flc_dma = ctx->flc_dma[GIVENCRYPT]; -+ caam_req->op_type = GIVENCRYPT; -+ caam_req->cbk = ablkcipher_done; -+ caam_req->ctx = &req->base; -+ caam_req->edesc = edesc; -+ ret = dpaa2_caam_enqueue(ctx->dev, caam_req); -+ if (ret != -EINPROGRESS && -+ !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { -+ ablkcipher_unmap(ctx->dev, edesc, req); -+ qi_cache_free(edesc); -+ } -+ -+ return ret; -+} -+ -+static int ablkcipher_decrypt(struct ablkcipher_request *req) -+{ -+ struct ablkcipher_edesc *edesc; -+ struct crypto_ablkcipher *ablkcipher = crypto_ablkcipher_reqtfm(req); -+ struct caam_ctx *ctx = crypto_ablkcipher_ctx(ablkcipher); -+ struct caam_request *caam_req = ablkcipher_request_ctx(req); -+ int ret; -+ -+ /* allocate extended descriptor */ -+ edesc = ablkcipher_edesc_alloc(req, false); -+ if (IS_ERR(edesc)) -+ return PTR_ERR(edesc); -+ -+ caam_req->flc = &ctx->flc[DECRYPT]; -+ caam_req->flc_dma = ctx->flc_dma[DECRYPT]; -+ caam_req->op_type = DECRYPT; -+ caam_req->cbk = ablkcipher_done; -+ caam_req->ctx = &req->base; -+ caam_req->edesc = edesc; -+ ret = dpaa2_caam_enqueue(ctx->dev, caam_req); -+ if (ret != -EINPROGRESS && -+ !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) { -+ ablkcipher_unmap(ctx->dev, edesc, req); -+ qi_cache_free(edesc); -+ } -+ -+ return ret; -+} -+ -+struct caam_crypto_alg { -+ struct list_head entry; -+ struct crypto_alg crypto_alg; -+ struct caam_alg_entry caam; -+}; -+ -+static int caam_cra_init(struct crypto_tfm *tfm, bool uses_dkp) -+{ -+ struct crypto_alg *alg = tfm->__crt_alg; -+ struct caam_crypto_alg *caam_alg = container_of(alg, typeof(*caam_alg), -+ crypto_alg); -+ struct caam_ctx *ctx = crypto_tfm_ctx(tfm); -+ dma_addr_t dma_addr; -+ int i; -+ -+ /* copy descriptor header template value */ -+ ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | -+ caam_alg->caam.class1_alg_type; -+ ctx->adata.algtype = OP_TYPE_CLASS2_ALG | -+ caam_alg->caam.class2_alg_type; -+ -+ ctx->dev = caam_alg->caam.dev; -+ ctx->dir = uses_dkp ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; -+ -+ dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, -+ offsetof(struct caam_ctx, flc_dma), -+ ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); -+ if (dma_mapping_error(ctx->dev, dma_addr)) { -+ dev_err(ctx->dev, "unable to map key, shared descriptors\n"); -+ return -ENOMEM; -+ } -+ -+ for (i = 0; i < NUM_OP; i++) -+ ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); -+ ctx->key_dma = dma_addr + NUM_OP * sizeof(ctx->flc[0]); -+ -+ return 0; -+} -+ -+static int caam_cra_init_ablkcipher(struct crypto_tfm *tfm) -+{ -+ struct ablkcipher_tfm *ablkcipher_tfm = -+ crypto_ablkcipher_crt(__crypto_ablkcipher_cast(tfm)); -+ -+ ablkcipher_tfm->reqsize = sizeof(struct caam_request); -+ return caam_cra_init(tfm, false); -+} -+ -+static int caam_cra_init_aead(struct crypto_aead *tfm) -+{ -+ struct aead_alg *alg = crypto_aead_alg(tfm); -+ -+ crypto_aead_set_reqsize(tfm, sizeof(struct caam_request)); -+ return caam_cra_init(crypto_aead_tfm(tfm), -+ (alg->setkey == aead_setkey) || -+ (alg->setkey == tls_setkey)); -+} -+ -+static void caam_exit_common(struct caam_ctx *ctx) -+{ -+ dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], -+ offsetof(struct caam_ctx, flc_dma), ctx->dir, -+ DMA_ATTR_SKIP_CPU_SYNC); -+} -+ -+static void caam_cra_exit(struct crypto_tfm *tfm) -+{ -+ caam_exit_common(crypto_tfm_ctx(tfm)); -+} -+ -+static void caam_cra_exit_aead(struct crypto_aead *tfm) -+{ -+ caam_exit_common(crypto_aead_ctx(tfm)); -+} -+ -+#define template_ablkcipher template_u.ablkcipher -+struct caam_alg_template { -+ char name[CRYPTO_MAX_ALG_NAME]; -+ char driver_name[CRYPTO_MAX_ALG_NAME]; -+ unsigned int blocksize; -+ u32 type; -+ union { -+ struct ablkcipher_alg ablkcipher; -+ } template_u; -+ u32 class1_alg_type; -+ u32 class2_alg_type; -+}; -+ -+static struct caam_alg_template driver_algs[] = { -+ /* ablkcipher descriptor */ -+ { -+ .name = "cbc(aes)", -+ .driver_name = "cbc-aes-caam-qi2", -+ .blocksize = AES_BLOCK_SIZE, -+ .type = CRYPTO_ALG_TYPE_GIVCIPHER, -+ .template_ablkcipher = { -+ .setkey = ablkcipher_setkey, -+ .encrypt = ablkcipher_encrypt, -+ .decrypt = ablkcipher_decrypt, -+ .givencrypt = ablkcipher_givencrypt, -+ .geniv = "<built-in>", -+ .min_keysize = AES_MIN_KEY_SIZE, -+ .max_keysize = AES_MAX_KEY_SIZE, -+ .ivsize = AES_BLOCK_SIZE, -+ }, -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ }, -+ { -+ .name = "cbc(des3_ede)", -+ .driver_name = "cbc-3des-caam-qi2", -+ .blocksize = DES3_EDE_BLOCK_SIZE, -+ .type = CRYPTO_ALG_TYPE_GIVCIPHER, -+ .template_ablkcipher = { -+ .setkey = ablkcipher_setkey, -+ .encrypt = ablkcipher_encrypt, -+ .decrypt = ablkcipher_decrypt, -+ .givencrypt = ablkcipher_givencrypt, -+ .geniv = "<built-in>", -+ .min_keysize = DES3_EDE_KEY_SIZE, -+ .max_keysize = DES3_EDE_KEY_SIZE, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ }, -+ { -+ .name = "cbc(des)", -+ .driver_name = "cbc-des-caam-qi2", -+ .blocksize = DES_BLOCK_SIZE, -+ .type = CRYPTO_ALG_TYPE_GIVCIPHER, -+ .template_ablkcipher = { -+ .setkey = ablkcipher_setkey, -+ .encrypt = ablkcipher_encrypt, -+ .decrypt = ablkcipher_decrypt, -+ .givencrypt = ablkcipher_givencrypt, -+ .geniv = "<built-in>", -+ .min_keysize = DES_KEY_SIZE, -+ .max_keysize = DES_KEY_SIZE, -+ .ivsize = DES_BLOCK_SIZE, -+ }, -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ }, -+ { -+ .name = "ctr(aes)", -+ .driver_name = "ctr-aes-caam-qi2", -+ .blocksize = 1, -+ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, -+ .template_ablkcipher = { -+ .setkey = ablkcipher_setkey, -+ .encrypt = ablkcipher_encrypt, -+ .decrypt = ablkcipher_decrypt, -+ .geniv = "chainiv", -+ .min_keysize = AES_MIN_KEY_SIZE, -+ .max_keysize = AES_MAX_KEY_SIZE, -+ .ivsize = AES_BLOCK_SIZE, -+ }, -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, -+ }, -+ { -+ .name = "rfc3686(ctr(aes))", -+ .driver_name = "rfc3686-ctr-aes-caam-qi2", -+ .blocksize = 1, -+ .type = CRYPTO_ALG_TYPE_GIVCIPHER, -+ .template_ablkcipher = { -+ .setkey = ablkcipher_setkey, -+ .encrypt = ablkcipher_encrypt, -+ .decrypt = ablkcipher_decrypt, -+ .givencrypt = ablkcipher_givencrypt, -+ .geniv = "<built-in>", -+ .min_keysize = AES_MIN_KEY_SIZE + -+ CTR_RFC3686_NONCE_SIZE, -+ .max_keysize = AES_MAX_KEY_SIZE + -+ CTR_RFC3686_NONCE_SIZE, -+ .ivsize = CTR_RFC3686_IV_SIZE, -+ }, -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CTR_MOD128, -+ }, -+ { -+ .name = "xts(aes)", -+ .driver_name = "xts-aes-caam-qi2", -+ .blocksize = AES_BLOCK_SIZE, -+ .type = CRYPTO_ALG_TYPE_ABLKCIPHER, -+ .template_ablkcipher = { -+ .setkey = xts_ablkcipher_setkey, -+ .encrypt = ablkcipher_encrypt, -+ .decrypt = ablkcipher_decrypt, -+ .geniv = "eseqiv", -+ .min_keysize = 2 * AES_MIN_KEY_SIZE, -+ .max_keysize = 2 * AES_MAX_KEY_SIZE, -+ .ivsize = AES_BLOCK_SIZE, -+ }, -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_XTS, -+ } -+}; -+ -+static struct caam_aead_alg driver_aeads[] = { -+ { -+ .aead = { -+ .base = { -+ .cra_name = "rfc4106(gcm(aes))", -+ .cra_driver_name = "rfc4106-gcm-aes-caam-qi2", -+ .cra_blocksize = 1, -+ }, -+ .setkey = rfc4106_setkey, -+ .setauthsize = rfc4106_setauthsize, -+ .encrypt = ipsec_gcm_encrypt, -+ .decrypt = ipsec_gcm_decrypt, -+ .ivsize = 8, -+ .maxauthsize = AES_BLOCK_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "rfc4543(gcm(aes))", -+ .cra_driver_name = "rfc4543-gcm-aes-caam-qi2", -+ .cra_blocksize = 1, -+ }, -+ .setkey = rfc4543_setkey, -+ .setauthsize = rfc4543_setauthsize, -+ .encrypt = ipsec_gcm_encrypt, -+ .decrypt = ipsec_gcm_decrypt, -+ .ivsize = 8, -+ .maxauthsize = AES_BLOCK_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, -+ }, -+ }, -+ /* Galois Counter Mode */ -+ { -+ .aead = { -+ .base = { -+ .cra_name = "gcm(aes)", -+ .cra_driver_name = "gcm-aes-caam-qi2", -+ .cra_blocksize = 1, -+ }, -+ .setkey = gcm_setkey, -+ .setauthsize = gcm_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = 12, -+ .maxauthsize = AES_BLOCK_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_GCM, -+ } -+ }, -+ /* single-pass ipsec_esp descriptor */ -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(md5),cbc(aes))", -+ .cra_driver_name = "authenc-hmac-md5-" -+ "cbc-aes-caam-qi2", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = MD5_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_MD5 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(md5)," -+ "cbc(aes)))", -+ .cra_driver_name = "echainiv-authenc-hmac-md5-" -+ "cbc-aes-caam-qi2", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = MD5_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_MD5 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha1),cbc(aes))", -+ .cra_driver_name = "authenc-hmac-sha1-" -+ "cbc-aes-caam-qi2", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = SHA1_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA1 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha1)," -+ "cbc(aes)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha1-cbc-aes-caam-qi2", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = SHA1_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA1 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha224),cbc(aes))", -+ .cra_driver_name = "authenc-hmac-sha224-" -+ "cbc-aes-caam-qi2", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = SHA224_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA224 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha224)," -+ "cbc(aes)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha224-cbc-aes-caam-qi2", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = SHA224_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA224 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha256),cbc(aes))", -+ .cra_driver_name = "authenc-hmac-sha256-" -+ "cbc-aes-caam-qi2", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = SHA256_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA256 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha256)," -+ "cbc(aes)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha256-cbc-aes-" -+ "caam-qi2", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = SHA256_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA256 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha384),cbc(aes))", -+ .cra_driver_name = "authenc-hmac-sha384-" -+ "cbc-aes-caam-qi2", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = SHA384_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA384 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha384)," -+ "cbc(aes)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha384-cbc-aes-" -+ "caam-qi2", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = SHA384_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA384 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha512),cbc(aes))", -+ .cra_driver_name = "authenc-hmac-sha512-" -+ "cbc-aes-caam-qi2", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = SHA512_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA512 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha512)," -+ "cbc(aes)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha512-cbc-aes-" -+ "caam-qi2", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = SHA512_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA512 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(md5),cbc(des3_ede))", -+ .cra_driver_name = "authenc-hmac-md5-" -+ "cbc-des3_ede-caam-qi2", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = MD5_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_MD5 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(md5)," -+ "cbc(des3_ede)))", -+ .cra_driver_name = "echainiv-authenc-hmac-md5-" -+ "cbc-des3_ede-caam-qi2", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = MD5_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_MD5 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha1)," -+ "cbc(des3_ede))", -+ .cra_driver_name = "authenc-hmac-sha1-" -+ "cbc-des3_ede-caam-qi2", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = SHA1_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA1 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha1)," -+ "cbc(des3_ede)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha1-" -+ "cbc-des3_ede-caam-qi2", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = SHA1_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA1 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha224)," -+ "cbc(des3_ede))", -+ .cra_driver_name = "authenc-hmac-sha224-" -+ "cbc-des3_ede-caam-qi2", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = SHA224_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA224 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha224)," -+ "cbc(des3_ede)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha224-" -+ "cbc-des3_ede-caam-qi2", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = SHA224_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA224 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha256)," -+ "cbc(des3_ede))", -+ .cra_driver_name = "authenc-hmac-sha256-" -+ "cbc-des3_ede-caam-qi2", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = SHA256_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA256 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha256)," -+ "cbc(des3_ede)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha256-" -+ "cbc-des3_ede-caam-qi2", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = SHA256_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA256 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha384)," -+ "cbc(des3_ede))", -+ .cra_driver_name = "authenc-hmac-sha384-" -+ "cbc-des3_ede-caam-qi2", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = SHA384_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA384 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha384)," -+ "cbc(des3_ede)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha384-" -+ "cbc-des3_ede-caam-qi2", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = SHA384_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA384 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha512)," -+ "cbc(des3_ede))", -+ .cra_driver_name = "authenc-hmac-sha512-" -+ "cbc-des3_ede-caam-qi2", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = SHA512_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA512 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha512)," -+ "cbc(des3_ede)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha512-" -+ "cbc-des3_ede-caam-qi2", -+ .cra_blocksize = DES3_EDE_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES3_EDE_BLOCK_SIZE, -+ .maxauthsize = SHA512_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_3DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA512 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(md5),cbc(des))", -+ .cra_driver_name = "authenc-hmac-md5-" -+ "cbc-des-caam-qi2", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = MD5_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_MD5 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(md5)," -+ "cbc(des)))", -+ .cra_driver_name = "echainiv-authenc-hmac-md5-" -+ "cbc-des-caam-qi2", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = MD5_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_MD5 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha1),cbc(des))", -+ .cra_driver_name = "authenc-hmac-sha1-" -+ "cbc-des-caam-qi2", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = SHA1_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA1 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha1)," -+ "cbc(des)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha1-cbc-des-caam-qi2", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = SHA1_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA1 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha224),cbc(des))", -+ .cra_driver_name = "authenc-hmac-sha224-" -+ "cbc-des-caam-qi2", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = SHA224_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA224 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha224)," -+ "cbc(des)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha224-cbc-des-" -+ "caam-qi2", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = SHA224_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA224 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha256),cbc(des))", -+ .cra_driver_name = "authenc-hmac-sha256-" -+ "cbc-des-caam-qi2", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = SHA256_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA256 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha256)," -+ "cbc(des)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha256-cbc-desi-" -+ "caam-qi2", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = SHA256_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA256 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha384),cbc(des))", -+ .cra_driver_name = "authenc-hmac-sha384-" -+ "cbc-des-caam-qi2", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = SHA384_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA384 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha384)," -+ "cbc(des)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha384-cbc-des-" -+ "caam-qi2", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = SHA384_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA384 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha512),cbc(des))", -+ .cra_driver_name = "authenc-hmac-sha512-" -+ "cbc-des-caam-qi2", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = SHA512_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA512 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "echainiv(authenc(hmac(sha512)," -+ "cbc(des)))", -+ .cra_driver_name = "echainiv-authenc-" -+ "hmac-sha512-cbc-des-" -+ "caam-qi2", -+ .cra_blocksize = DES_BLOCK_SIZE, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = DES_BLOCK_SIZE, -+ .maxauthsize = SHA512_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_DES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA512 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .geniv = true, -+ } -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(md5)," -+ "rfc3686(ctr(aes)))", -+ .cra_driver_name = "authenc-hmac-md5-" -+ "rfc3686-ctr-aes-caam-qi2", -+ .cra_blocksize = 1, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = CTR_RFC3686_IV_SIZE, -+ .maxauthsize = MD5_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | -+ OP_ALG_AAI_CTR_MOD128, -+ .class2_alg_type = OP_ALG_ALGSEL_MD5 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .rfc3686 = true, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "seqiv(authenc(" -+ "hmac(md5),rfc3686(ctr(aes))))", -+ .cra_driver_name = "seqiv-authenc-hmac-md5-" -+ "rfc3686-ctr-aes-caam-qi2", -+ .cra_blocksize = 1, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = CTR_RFC3686_IV_SIZE, -+ .maxauthsize = MD5_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | -+ OP_ALG_AAI_CTR_MOD128, -+ .class2_alg_type = OP_ALG_ALGSEL_MD5 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .rfc3686 = true, -+ .geniv = true, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha1)," -+ "rfc3686(ctr(aes)))", -+ .cra_driver_name = "authenc-hmac-sha1-" -+ "rfc3686-ctr-aes-caam-qi2", -+ .cra_blocksize = 1, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = CTR_RFC3686_IV_SIZE, -+ .maxauthsize = SHA1_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | -+ OP_ALG_AAI_CTR_MOD128, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA1 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .rfc3686 = true, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "seqiv(authenc(" -+ "hmac(sha1),rfc3686(ctr(aes))))", -+ .cra_driver_name = "seqiv-authenc-hmac-sha1-" -+ "rfc3686-ctr-aes-caam-qi2", -+ .cra_blocksize = 1, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = CTR_RFC3686_IV_SIZE, -+ .maxauthsize = SHA1_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | -+ OP_ALG_AAI_CTR_MOD128, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA1 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .rfc3686 = true, -+ .geniv = true, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha224)," -+ "rfc3686(ctr(aes)))", -+ .cra_driver_name = "authenc-hmac-sha224-" -+ "rfc3686-ctr-aes-caam-qi2", -+ .cra_blocksize = 1, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = CTR_RFC3686_IV_SIZE, -+ .maxauthsize = SHA224_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | -+ OP_ALG_AAI_CTR_MOD128, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA224 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .rfc3686 = true, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "seqiv(authenc(" -+ "hmac(sha224),rfc3686(ctr(aes))))", -+ .cra_driver_name = "seqiv-authenc-hmac-sha224-" -+ "rfc3686-ctr-aes-caam-qi2", -+ .cra_blocksize = 1, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = CTR_RFC3686_IV_SIZE, -+ .maxauthsize = SHA224_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | -+ OP_ALG_AAI_CTR_MOD128, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA224 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .rfc3686 = true, -+ .geniv = true, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha256)," -+ "rfc3686(ctr(aes)))", -+ .cra_driver_name = "authenc-hmac-sha256-" -+ "rfc3686-ctr-aes-caam-qi2", -+ .cra_blocksize = 1, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = CTR_RFC3686_IV_SIZE, -+ .maxauthsize = SHA256_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | -+ OP_ALG_AAI_CTR_MOD128, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA256 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .rfc3686 = true, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "seqiv(authenc(hmac(sha256)," -+ "rfc3686(ctr(aes))))", -+ .cra_driver_name = "seqiv-authenc-hmac-sha256-" -+ "rfc3686-ctr-aes-caam-qi2", -+ .cra_blocksize = 1, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = CTR_RFC3686_IV_SIZE, -+ .maxauthsize = SHA256_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | -+ OP_ALG_AAI_CTR_MOD128, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA256 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .rfc3686 = true, -+ .geniv = true, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha384)," -+ "rfc3686(ctr(aes)))", -+ .cra_driver_name = "authenc-hmac-sha384-" -+ "rfc3686-ctr-aes-caam-qi2", -+ .cra_blocksize = 1, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = CTR_RFC3686_IV_SIZE, -+ .maxauthsize = SHA384_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | -+ OP_ALG_AAI_CTR_MOD128, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA384 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .rfc3686 = true, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "seqiv(authenc(hmac(sha384)," -+ "rfc3686(ctr(aes))))", -+ .cra_driver_name = "seqiv-authenc-hmac-sha384-" -+ "rfc3686-ctr-aes-caam-qi2", -+ .cra_blocksize = 1, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = CTR_RFC3686_IV_SIZE, -+ .maxauthsize = SHA384_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | -+ OP_ALG_AAI_CTR_MOD128, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA384 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .rfc3686 = true, -+ .geniv = true, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "authenc(hmac(sha512)," -+ "rfc3686(ctr(aes)))", -+ .cra_driver_name = "authenc-hmac-sha512-" -+ "rfc3686-ctr-aes-caam-qi2", -+ .cra_blocksize = 1, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = CTR_RFC3686_IV_SIZE, -+ .maxauthsize = SHA512_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | -+ OP_ALG_AAI_CTR_MOD128, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA512 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .rfc3686 = true, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "seqiv(authenc(hmac(sha512)," -+ "rfc3686(ctr(aes))))", -+ .cra_driver_name = "seqiv-authenc-hmac-sha512-" -+ "rfc3686-ctr-aes-caam-qi2", -+ .cra_blocksize = 1, -+ }, -+ .setkey = aead_setkey, -+ .setauthsize = aead_setauthsize, -+ .encrypt = aead_encrypt, -+ .decrypt = aead_decrypt, -+ .ivsize = CTR_RFC3686_IV_SIZE, -+ .maxauthsize = SHA512_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | -+ OP_ALG_AAI_CTR_MOD128, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA512 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ .rfc3686 = true, -+ .geniv = true, -+ }, -+ }, -+ { -+ .aead = { -+ .base = { -+ .cra_name = "tls10(hmac(sha1),cbc(aes))", -+ .cra_driver_name = "tls10-hmac-sha1-cbc-aes-caam-qi2", -+ .cra_blocksize = AES_BLOCK_SIZE, -+ }, -+ .setkey = tls_setkey, -+ .setauthsize = tls_setauthsize, -+ .encrypt = tls_encrypt, -+ .decrypt = tls_decrypt, -+ .ivsize = AES_BLOCK_SIZE, -+ .maxauthsize = SHA1_DIGEST_SIZE, -+ }, -+ .caam = { -+ .class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, -+ .class2_alg_type = OP_ALG_ALGSEL_SHA1 | -+ OP_ALG_AAI_HMAC_PRECOMP, -+ }, -+ }, -+}; -+ -+static struct caam_crypto_alg *caam_alg_alloc(struct caam_alg_template -+ *template) -+{ -+ struct caam_crypto_alg *t_alg; -+ struct crypto_alg *alg; -+ -+ t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); -+ if (!t_alg) -+ return ERR_PTR(-ENOMEM); -+ -+ alg = &t_alg->crypto_alg; -+ -+ snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", template->name); -+ snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", -+ template->driver_name); -+ alg->cra_module = THIS_MODULE; -+ alg->cra_exit = caam_cra_exit; -+ alg->cra_priority = CAAM_CRA_PRIORITY; -+ alg->cra_blocksize = template->blocksize; -+ alg->cra_alignmask = 0; -+ alg->cra_ctxsize = sizeof(struct caam_ctx); -+ alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY | -+ template->type; -+ switch (template->type) { -+ case CRYPTO_ALG_TYPE_GIVCIPHER: -+ alg->cra_init = caam_cra_init_ablkcipher; -+ alg->cra_type = &crypto_givcipher_type; -+ alg->cra_ablkcipher = template->template_ablkcipher; -+ break; -+ case CRYPTO_ALG_TYPE_ABLKCIPHER: -+ alg->cra_init = caam_cra_init_ablkcipher; -+ alg->cra_type = &crypto_ablkcipher_type; -+ alg->cra_ablkcipher = template->template_ablkcipher; -+ break; -+ } -+ -+ t_alg->caam.class1_alg_type = template->class1_alg_type; -+ t_alg->caam.class2_alg_type = template->class2_alg_type; -+ -+ return t_alg; -+} -+ -+static void caam_aead_alg_init(struct caam_aead_alg *t_alg) -+{ -+ struct aead_alg *alg = &t_alg->aead; -+ -+ alg->base.cra_module = THIS_MODULE; -+ alg->base.cra_priority = CAAM_CRA_PRIORITY; -+ alg->base.cra_ctxsize = sizeof(struct caam_ctx); -+ alg->base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY; -+ -+ alg->init = caam_cra_init_aead; -+ alg->exit = caam_cra_exit_aead; -+} -+ -+/* max hash key is max split key size */ -+#define CAAM_MAX_HASH_KEY_SIZE (SHA512_DIGEST_SIZE * 2) -+ -+#define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE -+#define CAAM_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE -+ -+#define DESC_HASH_MAX_USED_BYTES (DESC_AHASH_FINAL_LEN + \ -+ CAAM_MAX_HASH_KEY_SIZE) -+#define DESC_HASH_MAX_USED_LEN (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ) -+ -+/* caam context sizes for hashes: running digest + 8 */ -+#define HASH_MSG_LEN 8 -+#define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE) -+ -+enum hash_optype { -+ UPDATE = 0, -+ UPDATE_FIRST, -+ FINALIZE, -+ DIGEST, -+ HASH_NUM_OP -+}; -+ -+/** -+ * caam_hash_ctx - ahash per-session context -+ * @flc: Flow Contexts array -+ * @flc_dma: I/O virtual addresses of the Flow Contexts -+ * @key: virtual address of the authentication key -+ * @dev: dpseci device -+ * @ctx_len: size of Context Register -+ * @adata: hashing algorithm details -+ */ -+struct caam_hash_ctx { -+ struct caam_flc flc[HASH_NUM_OP]; -+ dma_addr_t flc_dma[HASH_NUM_OP]; -+ u8 key[CAAM_MAX_HASH_KEY_SIZE]; -+ struct device *dev; -+ int ctx_len; -+ struct alginfo adata; -+}; -+ -+/* ahash state */ -+struct caam_hash_state { -+ struct caam_request caam_req; -+ dma_addr_t buf_dma; -+ dma_addr_t ctx_dma; -+ u8 buf_0[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; -+ int buflen_0; -+ u8 buf_1[CAAM_MAX_HASH_BLOCK_SIZE] ____cacheline_aligned; -+ int buflen_1; -+ u8 caam_ctx[MAX_CTX_LEN] ____cacheline_aligned; -+ int (*update)(struct ahash_request *req); -+ int (*final)(struct ahash_request *req); -+ int (*finup)(struct ahash_request *req); -+ int current_buf; -+}; -+ -+struct caam_export_state { -+ u8 buf[CAAM_MAX_HASH_BLOCK_SIZE]; -+ u8 caam_ctx[MAX_CTX_LEN]; -+ int buflen; -+ int (*update)(struct ahash_request *req); -+ int (*final)(struct ahash_request *req); -+ int (*finup)(struct ahash_request *req); -+}; -+ -+static inline void switch_buf(struct caam_hash_state *state) -+{ -+ state->current_buf ^= 1; -+} -+ -+static inline u8 *current_buf(struct caam_hash_state *state) -+{ -+ return state->current_buf ? state->buf_1 : state->buf_0; -+} -+ -+static inline u8 *alt_buf(struct caam_hash_state *state) -+{ -+ return state->current_buf ? state->buf_0 : state->buf_1; -+} -+ -+static inline int *current_buflen(struct caam_hash_state *state) -+{ -+ return state->current_buf ? &state->buflen_1 : &state->buflen_0; -+} -+ -+static inline int *alt_buflen(struct caam_hash_state *state) -+{ -+ return state->current_buf ? &state->buflen_0 : &state->buflen_1; -+} -+ -+/* Map current buffer in state (if length > 0) and put it in link table */ -+static inline int buf_map_to_qm_sg(struct device *dev, -+ struct dpaa2_sg_entry *qm_sg, -+ struct caam_hash_state *state) -+{ -+ int buflen = *current_buflen(state); -+ -+ if (!buflen) -+ return 0; -+ -+ state->buf_dma = dma_map_single(dev, current_buf(state), buflen, -+ DMA_TO_DEVICE); -+ if (dma_mapping_error(dev, state->buf_dma)) { -+ dev_err(dev, "unable to map buf\n"); -+ state->buf_dma = 0; -+ return -ENOMEM; -+ } -+ -+ dma_to_qm_sg_one(qm_sg, state->buf_dma, buflen, 0); -+ -+ return 0; -+} -+ -+/* Map state->caam_ctx, and add it to link table */ -+static inline int ctx_map_to_qm_sg(struct device *dev, -+ struct caam_hash_state *state, int ctx_len, -+ struct dpaa2_sg_entry *qm_sg, u32 flag) -+{ -+ state->ctx_dma = dma_map_single(dev, state->caam_ctx, ctx_len, flag); -+ if (dma_mapping_error(dev, state->ctx_dma)) { -+ dev_err(dev, "unable to map ctx\n"); -+ state->ctx_dma = 0; -+ return -ENOMEM; -+ } -+ -+ dma_to_qm_sg_one(qm_sg, state->ctx_dma, ctx_len, 0); -+ -+ return 0; -+} -+ -+static int ahash_set_sh_desc(struct crypto_ahash *ahash) -+{ -+ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); -+ int digestsize = crypto_ahash_digestsize(ahash); -+ struct dpaa2_caam_priv *priv = dev_get_drvdata(ctx->dev); -+ struct caam_flc *flc; -+ u32 *desc; -+ -+ ctx->adata.key_virt = ctx->key; -+ ctx->adata.key_inline = true; -+ -+ /* ahash_update shared descriptor */ -+ flc = &ctx->flc[UPDATE]; -+ desc = flc->sh_desc; -+ cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len, -+ ctx->ctx_len, true, priv->sec_attr.era); -+ flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ -+ dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE], -+ desc_bytes(desc), DMA_BIDIRECTIONAL); -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, -+ "ahash update shdesc@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+ -+ /* ahash_update_first shared descriptor */ -+ flc = &ctx->flc[UPDATE_FIRST]; -+ desc = flc->sh_desc; -+ cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len, -+ ctx->ctx_len, false, priv->sec_attr.era); -+ flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ -+ dma_sync_single_for_device(ctx->dev, ctx->flc_dma[UPDATE_FIRST], -+ desc_bytes(desc), DMA_BIDIRECTIONAL); -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, -+ "ahash update first shdesc@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+ -+ /* ahash_final shared descriptor */ -+ flc = &ctx->flc[FINALIZE]; -+ desc = flc->sh_desc; -+ cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize, -+ ctx->ctx_len, true, priv->sec_attr.era); -+ flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ -+ dma_sync_single_for_device(ctx->dev, ctx->flc_dma[FINALIZE], -+ desc_bytes(desc), DMA_BIDIRECTIONAL); -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, -+ "ahash final shdesc@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+ -+ /* ahash_digest shared descriptor */ -+ flc = &ctx->flc[DIGEST]; -+ desc = flc->sh_desc; -+ cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize, -+ ctx->ctx_len, false, priv->sec_attr.era); -+ flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ -+ dma_sync_single_for_device(ctx->dev, ctx->flc_dma[DIGEST], -+ desc_bytes(desc), DMA_BIDIRECTIONAL); -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, -+ "ahash digest shdesc@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+ -+ return 0; -+} -+ -+/* Digest hash size if it is too large */ -+static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in, -+ u32 *keylen, u8 *key_out, u32 digestsize) -+{ -+ struct caam_request *req_ctx; -+ u32 *desc; -+ struct split_key_sh_result result; -+ dma_addr_t src_dma, dst_dma; -+ struct caam_flc *flc; -+ dma_addr_t flc_dma; -+ int ret = -ENOMEM; -+ struct dpaa2_fl_entry *in_fle, *out_fle; -+ -+ req_ctx = kzalloc(sizeof(*req_ctx), GFP_KERNEL | GFP_DMA); -+ if (!req_ctx) -+ return -ENOMEM; -+ -+ in_fle = &req_ctx->fd_flt[1]; -+ out_fle = &req_ctx->fd_flt[0]; -+ -+ flc = kzalloc(sizeof(*flc), GFP_KERNEL | GFP_DMA); -+ if (!flc) -+ goto err_flc; -+ -+ src_dma = dma_map_single(ctx->dev, (void *)key_in, *keylen, -+ DMA_TO_DEVICE); -+ if (dma_mapping_error(ctx->dev, src_dma)) { -+ dev_err(ctx->dev, "unable to map key input memory\n"); -+ goto err_src_dma; -+ } -+ dst_dma = dma_map_single(ctx->dev, (void *)key_out, digestsize, -+ DMA_FROM_DEVICE); -+ if (dma_mapping_error(ctx->dev, dst_dma)) { -+ dev_err(ctx->dev, "unable to map key output memory\n"); -+ goto err_dst_dma; -+ } -+ -+ desc = flc->sh_desc; -+ -+ init_sh_desc(desc, 0); -+ -+ /* descriptor to perform unkeyed hash on key_in */ -+ append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT | -+ OP_ALG_AS_INITFINAL); -+ append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 | -+ FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_MSG); -+ append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | -+ LDST_SRCDST_BYTE_CONTEXT); -+ -+ flc->flc[1] = cpu_to_caam32(desc_len(desc)); /* SDL */ -+ flc_dma = dma_map_single(ctx->dev, flc, sizeof(flc->flc) + -+ desc_bytes(desc), DMA_TO_DEVICE); -+ if (dma_mapping_error(ctx->dev, flc_dma)) { -+ dev_err(ctx->dev, "unable to map shared descriptor\n"); -+ goto err_flc_dma; -+ } -+ -+ dpaa2_fl_set_final(in_fle, true); -+ dpaa2_fl_set_format(in_fle, dpaa2_fl_single); -+ dpaa2_fl_set_addr(in_fle, src_dma); -+ dpaa2_fl_set_len(in_fle, *keylen); -+ dpaa2_fl_set_format(out_fle, dpaa2_fl_single); -+ dpaa2_fl_set_addr(out_fle, dst_dma); -+ dpaa2_fl_set_len(out_fle, digestsize); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "key_in@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, key_in, *keylen, 1); -+ print_hex_dump(KERN_ERR, "shdesc@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -+#endif -+ -+ result.err = 0; -+ init_completion(&result.completion); -+ result.dev = ctx->dev; -+ -+ req_ctx->flc = flc; -+ req_ctx->flc_dma = flc_dma; -+ req_ctx->cbk = split_key_sh_done; -+ req_ctx->ctx = &result; -+ -+ ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); -+ if (ret == -EINPROGRESS) { -+ /* in progress */ -+ wait_for_completion(&result.completion); -+ ret = result.err; -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, -+ "digested key@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, key_in, digestsize, -+ 1); -+#endif -+ } -+ -+ dma_unmap_single(ctx->dev, flc_dma, sizeof(flc->flc) + desc_bytes(desc), -+ DMA_TO_DEVICE); -+err_flc_dma: -+ dma_unmap_single(ctx->dev, dst_dma, digestsize, DMA_FROM_DEVICE); -+err_dst_dma: -+ dma_unmap_single(ctx->dev, src_dma, *keylen, DMA_TO_DEVICE); -+err_src_dma: -+ kfree(flc); -+err_flc: -+ kfree(req_ctx); -+ -+ *keylen = digestsize; -+ -+ return ret; -+} -+ -+static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key, -+ unsigned int keylen) -+{ -+ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); -+ unsigned int blocksize = crypto_tfm_alg_blocksize(&ahash->base); -+ unsigned int digestsize = crypto_ahash_digestsize(ahash); -+ int ret; -+ u8 *hashed_key = NULL; -+ -+#ifdef DEBUG -+ dev_err(ctx->dev, "keylen %d blocksize %d\n", keylen, blocksize); -+#endif -+ -+ if (keylen > blocksize) { -+ hashed_key = kmalloc_array(digestsize, sizeof(*hashed_key), -+ GFP_KERNEL | GFP_DMA); -+ if (!hashed_key) -+ return -ENOMEM; -+ ret = hash_digest_key(ctx, key, &keylen, hashed_key, -+ digestsize); -+ if (ret) -+ goto bad_free_key; -+ key = hashed_key; -+ } -+ -+ ctx->adata.keylen = keylen; -+ ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & -+ OP_ALG_ALGSEL_MASK); -+ if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE) -+ goto bad_free_key; -+ -+ memcpy(ctx->key, key, keylen); -+ -+ kfree(hashed_key); -+ return ahash_set_sh_desc(ahash); -+bad_free_key: -+ kfree(hashed_key); -+ crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN); -+ return -EINVAL; -+} -+ -+static inline void ahash_unmap(struct device *dev, struct ahash_edesc *edesc, -+ struct ahash_request *req, int dst_len) -+{ -+ struct caam_hash_state *state = ahash_request_ctx(req); -+ -+ if (edesc->src_nents) -+ dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); -+ if (edesc->dst_dma) -+ dma_unmap_single(dev, edesc->dst_dma, dst_len, DMA_FROM_DEVICE); -+ -+ if (edesc->qm_sg_bytes) -+ dma_unmap_single(dev, edesc->qm_sg_dma, edesc->qm_sg_bytes, -+ DMA_TO_DEVICE); -+ -+ if (state->buf_dma) { -+ dma_unmap_single(dev, state->buf_dma, *current_buflen(state), -+ DMA_TO_DEVICE); -+ state->buf_dma = 0; -+ } -+} -+ -+static inline void ahash_unmap_ctx(struct device *dev, -+ struct ahash_edesc *edesc, -+ struct ahash_request *req, int dst_len, -+ u32 flag) -+{ -+ struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); -+ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); -+ struct caam_hash_state *state = ahash_request_ctx(req); -+ -+ if (state->ctx_dma) { -+ dma_unmap_single(dev, state->ctx_dma, ctx->ctx_len, flag); -+ state->ctx_dma = 0; -+ } -+ ahash_unmap(dev, edesc, req, dst_len); -+} -+ -+static void ahash_done(void *cbk_ctx, u32 status) -+{ -+ struct crypto_async_request *areq = cbk_ctx; -+ struct ahash_request *req = ahash_request_cast(areq); -+ struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); -+ struct caam_hash_state *state = ahash_request_ctx(req); -+ struct ahash_edesc *edesc = state->caam_req.edesc; -+ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); -+ int digestsize = crypto_ahash_digestsize(ahash); -+ int ecode = 0; -+ -+#ifdef DEBUG -+ dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); -+#endif -+ -+ if (unlikely(status)) { -+ caam_qi2_strstatus(ctx->dev, status); -+ ecode = -EIO; -+ } -+ -+ ahash_unmap(ctx->dev, edesc, req, digestsize); -+ qi_cache_free(edesc); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "ctx@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, -+ ctx->ctx_len, 1); -+ if (req->result) -+ print_hex_dump(KERN_ERR, "result@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, req->result, -+ digestsize, 1); -+#endif -+ -+ req->base.complete(&req->base, ecode); -+} -+ -+static void ahash_done_bi(void *cbk_ctx, u32 status) -+{ -+ struct crypto_async_request *areq = cbk_ctx; -+ struct ahash_request *req = ahash_request_cast(areq); -+ struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); -+ struct caam_hash_state *state = ahash_request_ctx(req); -+ struct ahash_edesc *edesc = state->caam_req.edesc; -+ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); -+ int ecode = 0; -+#ifdef DEBUG -+ int digestsize = crypto_ahash_digestsize(ahash); -+ -+ dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); -+#endif -+ -+ if (unlikely(status)) { -+ caam_qi2_strstatus(ctx->dev, status); -+ ecode = -EIO; -+ } -+ -+ ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); -+ switch_buf(state); -+ qi_cache_free(edesc); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "ctx@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, -+ ctx->ctx_len, 1); -+ if (req->result) -+ print_hex_dump(KERN_ERR, "result@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, req->result, -+ digestsize, 1); -+#endif -+ -+ req->base.complete(&req->base, ecode); -+} -+ -+static void ahash_done_ctx_src(void *cbk_ctx, u32 status) -+{ -+ struct crypto_async_request *areq = cbk_ctx; -+ struct ahash_request *req = ahash_request_cast(areq); -+ struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); -+ struct caam_hash_state *state = ahash_request_ctx(req); -+ struct ahash_edesc *edesc = state->caam_req.edesc; -+ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); -+ int digestsize = crypto_ahash_digestsize(ahash); -+ int ecode = 0; -+ -+#ifdef DEBUG -+ dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); -+#endif -+ -+ if (unlikely(status)) { -+ caam_qi2_strstatus(ctx->dev, status); -+ ecode = -EIO; -+ } -+ -+ ahash_unmap_ctx(ctx->dev, edesc, req, digestsize, DMA_TO_DEVICE); -+ qi_cache_free(edesc); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "ctx@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, -+ ctx->ctx_len, 1); -+ if (req->result) -+ print_hex_dump(KERN_ERR, "result@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, req->result, -+ digestsize, 1); -+#endif -+ -+ req->base.complete(&req->base, ecode); -+} -+ -+static void ahash_done_ctx_dst(void *cbk_ctx, u32 status) -+{ -+ struct crypto_async_request *areq = cbk_ctx; -+ struct ahash_request *req = ahash_request_cast(areq); -+ struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); -+ struct caam_hash_state *state = ahash_request_ctx(req); -+ struct ahash_edesc *edesc = state->caam_req.edesc; -+ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); -+ int ecode = 0; -+#ifdef DEBUG -+ int digestsize = crypto_ahash_digestsize(ahash); -+ -+ dev_err(ctx->dev, "%s %d: err 0x%x\n", __func__, __LINE__, status); -+#endif -+ -+ if (unlikely(status)) { -+ caam_qi2_strstatus(ctx->dev, status); -+ ecode = -EIO; -+ } -+ -+ ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE); -+ switch_buf(state); -+ qi_cache_free(edesc); -+ -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "ctx@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, -+ ctx->ctx_len, 1); -+ if (req->result) -+ print_hex_dump(KERN_ERR, "result@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, req->result, -+ digestsize, 1); -+#endif -+ -+ req->base.complete(&req->base, ecode); -+} -+ -+static int ahash_update_ctx(struct ahash_request *req) -+{ -+ struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); -+ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); -+ struct caam_hash_state *state = ahash_request_ctx(req); -+ struct caam_request *req_ctx = &state->caam_req; -+ struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; -+ struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; -+ u8 *buf = current_buf(state); -+ int *buflen = current_buflen(state); -+ u8 *next_buf = alt_buf(state); -+ int *next_buflen = alt_buflen(state), last_buflen; -+ int in_len = *buflen + req->nbytes, to_hash; -+ int src_nents, mapped_nents, qm_sg_bytes, qm_sg_src_index; -+ struct ahash_edesc *edesc; -+ int ret = 0; -+ -+ last_buflen = *next_buflen; -+ *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); -+ to_hash = in_len - *next_buflen; -+ -+ if (to_hash) { -+ struct dpaa2_sg_entry *sg_table; -+ -+ src_nents = sg_nents_for_len(req->src, -+ req->nbytes - (*next_buflen)); -+ if (src_nents < 0) { -+ dev_err(ctx->dev, "Invalid number of src SG.\n"); -+ return src_nents; -+ } -+ -+ if (src_nents) { -+ mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, -+ DMA_TO_DEVICE); -+ if (!mapped_nents) { -+ dev_err(ctx->dev, "unable to DMA map source\n"); -+ return -ENOMEM; -+ } -+ } else { -+ mapped_nents = 0; -+ } -+ -+ /* allocate space for base edesc and link tables */ -+ edesc = qi_cache_zalloc(GFP_DMA | flags); -+ if (!edesc) { -+ dma_unmap_sg(ctx->dev, req->src, src_nents, -+ DMA_TO_DEVICE); -+ return -ENOMEM; -+ } -+ -+ edesc->src_nents = src_nents; -+ qm_sg_src_index = 1 + (*buflen ? 1 : 0); -+ qm_sg_bytes = (qm_sg_src_index + mapped_nents) * -+ sizeof(*sg_table); -+ sg_table = &edesc->sgt[0]; -+ -+ ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, -+ DMA_BIDIRECTIONAL); -+ if (ret) -+ goto unmap_ctx; -+ -+ ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); -+ if (ret) -+ goto unmap_ctx; -+ -+ if (mapped_nents) { -+ sg_to_qm_sg_last(req->src, mapped_nents, -+ sg_table + qm_sg_src_index, 0); -+ if (*next_buflen) -+ scatterwalk_map_and_copy(next_buf, req->src, -+ to_hash - *buflen, -+ *next_buflen, 0); -+ } else { -+ dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, -+ true); -+ } -+ -+ edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, -+ qm_sg_bytes, DMA_TO_DEVICE); -+ if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { -+ dev_err(ctx->dev, "unable to map S/G table\n"); -+ ret = -ENOMEM; -+ goto unmap_ctx; -+ } -+ edesc->qm_sg_bytes = qm_sg_bytes; -+ -+ memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); -+ dpaa2_fl_set_final(in_fle, true); -+ dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); -+ dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); -+ dpaa2_fl_set_len(in_fle, ctx->ctx_len + to_hash); -+ dpaa2_fl_set_format(out_fle, dpaa2_fl_single); -+ dpaa2_fl_set_addr(out_fle, state->ctx_dma); -+ dpaa2_fl_set_len(out_fle, ctx->ctx_len); -+ -+ req_ctx->flc = &ctx->flc[UPDATE]; -+ req_ctx->flc_dma = ctx->flc_dma[UPDATE]; -+ req_ctx->cbk = ahash_done_bi; -+ req_ctx->ctx = &req->base; -+ req_ctx->edesc = edesc; -+ -+ ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); -+ if (ret != -EINPROGRESS && -+ !(ret == -EBUSY && -+ req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) -+ goto unmap_ctx; -+ } else if (*next_buflen) { -+ scatterwalk_map_and_copy(buf + *buflen, req->src, 0, -+ req->nbytes, 0); -+ *buflen = *next_buflen; -+ *next_buflen = last_buflen; -+ } -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "buf@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); -+ print_hex_dump(KERN_ERR, "next buf@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, next_buf, -+ *next_buflen, 1); -+#endif -+ -+ return ret; -+unmap_ctx: -+ ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); -+ qi_cache_free(edesc); -+ return ret; -+} -+ -+static int ahash_final_ctx(struct ahash_request *req) -+{ -+ struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); -+ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); -+ struct caam_hash_state *state = ahash_request_ctx(req); -+ struct caam_request *req_ctx = &state->caam_req; -+ struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; -+ struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; -+ int buflen = *current_buflen(state); -+ int qm_sg_bytes, qm_sg_src_index; -+ int digestsize = crypto_ahash_digestsize(ahash); -+ struct ahash_edesc *edesc; -+ struct dpaa2_sg_entry *sg_table; -+ int ret; -+ -+ /* allocate space for base edesc and link tables */ -+ edesc = qi_cache_zalloc(GFP_DMA | flags); -+ if (!edesc) -+ return -ENOMEM; -+ -+ qm_sg_src_index = 1 + (buflen ? 1 : 0); -+ qm_sg_bytes = qm_sg_src_index * sizeof(*sg_table); -+ sg_table = &edesc->sgt[0]; -+ -+ ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, -+ DMA_TO_DEVICE); -+ if (ret) -+ goto unmap_ctx; -+ -+ ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); -+ if (ret) -+ goto unmap_ctx; -+ -+ dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, true); -+ -+ edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, -+ DMA_TO_DEVICE); -+ if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { -+ dev_err(ctx->dev, "unable to map S/G table\n"); -+ ret = -ENOMEM; -+ goto unmap_ctx; -+ } -+ edesc->qm_sg_bytes = qm_sg_bytes; -+ -+ edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize, -+ DMA_FROM_DEVICE); -+ if (dma_mapping_error(ctx->dev, edesc->dst_dma)) { -+ dev_err(ctx->dev, "unable to map dst\n"); -+ edesc->dst_dma = 0; -+ ret = -ENOMEM; -+ goto unmap_ctx; -+ } -+ -+ memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); -+ dpaa2_fl_set_final(in_fle, true); -+ dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); -+ dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); -+ dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen); -+ dpaa2_fl_set_format(out_fle, dpaa2_fl_single); -+ dpaa2_fl_set_addr(out_fle, edesc->dst_dma); -+ dpaa2_fl_set_len(out_fle, digestsize); -+ -+ req_ctx->flc = &ctx->flc[FINALIZE]; -+ req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; -+ req_ctx->cbk = ahash_done_ctx_src; -+ req_ctx->ctx = &req->base; -+ req_ctx->edesc = edesc; -+ -+ ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); -+ if (ret == -EINPROGRESS || -+ (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) -+ return ret; -+ -+unmap_ctx: -+ ahash_unmap_ctx(ctx->dev, edesc, req, digestsize, DMA_FROM_DEVICE); -+ qi_cache_free(edesc); -+ return ret; -+} -+ -+static int ahash_finup_ctx(struct ahash_request *req) -+{ -+ struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); -+ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); -+ struct caam_hash_state *state = ahash_request_ctx(req); -+ struct caam_request *req_ctx = &state->caam_req; -+ struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; -+ struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; -+ int buflen = *current_buflen(state); -+ int qm_sg_bytes, qm_sg_src_index; -+ int src_nents, mapped_nents; -+ int digestsize = crypto_ahash_digestsize(ahash); -+ struct ahash_edesc *edesc; -+ struct dpaa2_sg_entry *sg_table; -+ int ret; -+ -+ src_nents = sg_nents_for_len(req->src, req->nbytes); -+ if (src_nents < 0) { -+ dev_err(ctx->dev, "Invalid number of src SG.\n"); -+ return src_nents; -+ } -+ -+ if (src_nents) { -+ mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, -+ DMA_TO_DEVICE); -+ if (!mapped_nents) { -+ dev_err(ctx->dev, "unable to DMA map source\n"); -+ return -ENOMEM; -+ } -+ } else { -+ mapped_nents = 0; -+ } -+ -+ /* allocate space for base edesc and link tables */ -+ edesc = qi_cache_zalloc(GFP_DMA | flags); -+ if (!edesc) { -+ dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); -+ return -ENOMEM; -+ } -+ -+ edesc->src_nents = src_nents; -+ qm_sg_src_index = 1 + (buflen ? 1 : 0); -+ qm_sg_bytes = (qm_sg_src_index + mapped_nents) * sizeof(*sg_table); -+ sg_table = &edesc->sgt[0]; -+ -+ ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, -+ DMA_TO_DEVICE); -+ if (ret) -+ goto unmap_ctx; -+ -+ ret = buf_map_to_qm_sg(ctx->dev, sg_table + 1, state); -+ if (ret) -+ goto unmap_ctx; -+ -+ sg_to_qm_sg_last(req->src, mapped_nents, sg_table + qm_sg_src_index, 0); -+ -+ edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, -+ DMA_TO_DEVICE); -+ if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { -+ dev_err(ctx->dev, "unable to map S/G table\n"); -+ ret = -ENOMEM; -+ goto unmap_ctx; -+ } -+ edesc->qm_sg_bytes = qm_sg_bytes; -+ -+ edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize, -+ DMA_FROM_DEVICE); -+ if (dma_mapping_error(ctx->dev, edesc->dst_dma)) { -+ dev_err(ctx->dev, "unable to map dst\n"); -+ edesc->dst_dma = 0; -+ ret = -ENOMEM; -+ goto unmap_ctx; -+ } -+ -+ memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); -+ dpaa2_fl_set_final(in_fle, true); -+ dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); -+ dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); -+ dpaa2_fl_set_len(in_fle, ctx->ctx_len + buflen + req->nbytes); -+ dpaa2_fl_set_format(out_fle, dpaa2_fl_single); -+ dpaa2_fl_set_addr(out_fle, edesc->dst_dma); -+ dpaa2_fl_set_len(out_fle, digestsize); -+ -+ req_ctx->flc = &ctx->flc[FINALIZE]; -+ req_ctx->flc_dma = ctx->flc_dma[FINALIZE]; -+ req_ctx->cbk = ahash_done_ctx_src; -+ req_ctx->ctx = &req->base; -+ req_ctx->edesc = edesc; -+ -+ ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); -+ if (ret == -EINPROGRESS || -+ (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) -+ return ret; -+ -+unmap_ctx: -+ ahash_unmap_ctx(ctx->dev, edesc, req, digestsize, DMA_FROM_DEVICE); -+ qi_cache_free(edesc); -+ return ret; -+} -+ -+static int ahash_digest(struct ahash_request *req) -+{ -+ struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); -+ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); -+ struct caam_hash_state *state = ahash_request_ctx(req); -+ struct caam_request *req_ctx = &state->caam_req; -+ struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; -+ struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; -+ int digestsize = crypto_ahash_digestsize(ahash); -+ int src_nents, mapped_nents; -+ struct ahash_edesc *edesc; -+ int ret = -ENOMEM; -+ -+ state->buf_dma = 0; -+ -+ src_nents = sg_nents_for_len(req->src, req->nbytes); -+ if (src_nents < 0) { -+ dev_err(ctx->dev, "Invalid number of src SG.\n"); -+ return src_nents; -+ } -+ -+ if (src_nents) { -+ mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, -+ DMA_TO_DEVICE); -+ if (!mapped_nents) { -+ dev_err(ctx->dev, "unable to map source for DMA\n"); -+ return ret; -+ } -+ } else { -+ mapped_nents = 0; -+ } -+ -+ /* allocate space for base edesc and link tables */ -+ edesc = qi_cache_zalloc(GFP_DMA | flags); -+ if (!edesc) { -+ dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); -+ return ret; -+ } -+ -+ edesc->src_nents = src_nents; -+ memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); -+ -+ if (mapped_nents > 1) { -+ int qm_sg_bytes; -+ struct dpaa2_sg_entry *sg_table = &edesc->sgt[0]; -+ -+ qm_sg_bytes = mapped_nents * sizeof(*sg_table); -+ sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0); -+ edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, -+ qm_sg_bytes, DMA_TO_DEVICE); -+ if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { -+ dev_err(ctx->dev, "unable to map S/G table\n"); -+ goto unmap; -+ } -+ edesc->qm_sg_bytes = qm_sg_bytes; -+ dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); -+ dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); -+ } else { -+ dpaa2_fl_set_format(in_fle, dpaa2_fl_single); -+ dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); -+ } -+ -+ edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize, -+ DMA_FROM_DEVICE); -+ if (dma_mapping_error(ctx->dev, edesc->dst_dma)) { -+ dev_err(ctx->dev, "unable to map dst\n"); -+ edesc->dst_dma = 0; -+ goto unmap; -+ } -+ -+ dpaa2_fl_set_final(in_fle, true); -+ dpaa2_fl_set_len(in_fle, req->nbytes); -+ dpaa2_fl_set_format(out_fle, dpaa2_fl_single); -+ dpaa2_fl_set_addr(out_fle, edesc->dst_dma); -+ dpaa2_fl_set_len(out_fle, digestsize); -+ -+ req_ctx->flc = &ctx->flc[DIGEST]; -+ req_ctx->flc_dma = ctx->flc_dma[DIGEST]; -+ req_ctx->cbk = ahash_done; -+ req_ctx->ctx = &req->base; -+ req_ctx->edesc = edesc; -+ ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); -+ if (ret == -EINPROGRESS || -+ (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) -+ return ret; -+ -+unmap: -+ ahash_unmap(ctx->dev, edesc, req, digestsize); -+ qi_cache_free(edesc); -+ return ret; -+} -+ -+static int ahash_final_no_ctx(struct ahash_request *req) -+{ -+ struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); -+ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); -+ struct caam_hash_state *state = ahash_request_ctx(req); -+ struct caam_request *req_ctx = &state->caam_req; -+ struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; -+ struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; -+ u8 *buf = current_buf(state); -+ int buflen = *current_buflen(state); -+ int digestsize = crypto_ahash_digestsize(ahash); -+ struct ahash_edesc *edesc; -+ int ret = -ENOMEM; -+ -+ /* allocate space for base edesc and link tables */ -+ edesc = qi_cache_zalloc(GFP_DMA | flags); -+ if (!edesc) -+ return ret; -+ -+ state->buf_dma = dma_map_single(ctx->dev, buf, buflen, DMA_TO_DEVICE); -+ if (dma_mapping_error(ctx->dev, state->buf_dma)) { -+ dev_err(ctx->dev, "unable to map src\n"); -+ goto unmap; -+ } -+ -+ edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize, -+ DMA_FROM_DEVICE); -+ if (dma_mapping_error(ctx->dev, edesc->dst_dma)) { -+ dev_err(ctx->dev, "unable to map dst\n"); -+ edesc->dst_dma = 0; -+ goto unmap; -+ } -+ -+ memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); -+ dpaa2_fl_set_final(in_fle, true); -+ dpaa2_fl_set_format(in_fle, dpaa2_fl_single); -+ dpaa2_fl_set_addr(in_fle, state->buf_dma); -+ dpaa2_fl_set_len(in_fle, buflen); -+ dpaa2_fl_set_format(out_fle, dpaa2_fl_single); -+ dpaa2_fl_set_addr(out_fle, edesc->dst_dma); -+ dpaa2_fl_set_len(out_fle, digestsize); -+ -+ req_ctx->flc = &ctx->flc[DIGEST]; -+ req_ctx->flc_dma = ctx->flc_dma[DIGEST]; -+ req_ctx->cbk = ahash_done; -+ req_ctx->ctx = &req->base; -+ req_ctx->edesc = edesc; -+ -+ ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); -+ if (ret == -EINPROGRESS || -+ (ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) -+ return ret; -+ -+unmap: -+ ahash_unmap(ctx->dev, edesc, req, digestsize); -+ qi_cache_free(edesc); -+ return ret; -+} -+ -+static int ahash_update_no_ctx(struct ahash_request *req) -+{ -+ struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); -+ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); -+ struct caam_hash_state *state = ahash_request_ctx(req); -+ struct caam_request *req_ctx = &state->caam_req; -+ struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; -+ struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; -+ u8 *buf = current_buf(state); -+ int *buflen = current_buflen(state); -+ u8 *next_buf = alt_buf(state); -+ int *next_buflen = alt_buflen(state); -+ int in_len = *buflen + req->nbytes, to_hash; -+ int qm_sg_bytes, src_nents, mapped_nents; -+ struct ahash_edesc *edesc; -+ int ret = 0; -+ -+ *next_buflen = in_len & (crypto_tfm_alg_blocksize(&ahash->base) - 1); -+ to_hash = in_len - *next_buflen; -+ -+ if (to_hash) { -+ struct dpaa2_sg_entry *sg_table; -+ -+ src_nents = sg_nents_for_len(req->src, -+ req->nbytes - *next_buflen); -+ if (src_nents < 0) { -+ dev_err(ctx->dev, "Invalid number of src SG.\n"); -+ return src_nents; -+ } -+ -+ if (src_nents) { -+ mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, -+ DMA_TO_DEVICE); -+ if (!mapped_nents) { -+ dev_err(ctx->dev, "unable to DMA map source\n"); -+ return -ENOMEM; -+ } -+ } else { -+ mapped_nents = 0; -+ } -+ -+ /* allocate space for base edesc and link tables */ -+ edesc = qi_cache_zalloc(GFP_DMA | flags); -+ if (!edesc) { -+ dma_unmap_sg(ctx->dev, req->src, src_nents, -+ DMA_TO_DEVICE); -+ return -ENOMEM; -+ } -+ -+ edesc->src_nents = src_nents; -+ qm_sg_bytes = (1 + mapped_nents) * sizeof(*sg_table); -+ sg_table = &edesc->sgt[0]; -+ -+ ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); -+ if (ret) -+ goto unmap_ctx; -+ -+ sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0); -+ -+ if (*next_buflen) -+ scatterwalk_map_and_copy(next_buf, req->src, -+ to_hash - *buflen, -+ *next_buflen, 0); -+ -+ edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, -+ qm_sg_bytes, DMA_TO_DEVICE); -+ if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { -+ dev_err(ctx->dev, "unable to map S/G table\n"); -+ ret = -ENOMEM; -+ goto unmap_ctx; -+ } -+ edesc->qm_sg_bytes = qm_sg_bytes; -+ -+ state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, -+ ctx->ctx_len, DMA_FROM_DEVICE); -+ if (dma_mapping_error(ctx->dev, state->ctx_dma)) { -+ dev_err(ctx->dev, "unable to map ctx\n"); -+ state->ctx_dma = 0; -+ ret = -ENOMEM; -+ goto unmap_ctx; -+ } -+ -+ memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); -+ dpaa2_fl_set_final(in_fle, true); -+ dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); -+ dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); -+ dpaa2_fl_set_len(in_fle, to_hash); -+ dpaa2_fl_set_format(out_fle, dpaa2_fl_single); -+ dpaa2_fl_set_addr(out_fle, state->ctx_dma); -+ dpaa2_fl_set_len(out_fle, ctx->ctx_len); -+ -+ req_ctx->flc = &ctx->flc[UPDATE_FIRST]; -+ req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; -+ req_ctx->cbk = ahash_done_ctx_dst; -+ req_ctx->ctx = &req->base; -+ req_ctx->edesc = edesc; -+ -+ ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); -+ if (ret != -EINPROGRESS && -+ !(ret == -EBUSY && -+ req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) -+ goto unmap_ctx; -+ -+ state->update = ahash_update_ctx; -+ state->finup = ahash_finup_ctx; -+ state->final = ahash_final_ctx; -+ } else if (*next_buflen) { -+ scatterwalk_map_and_copy(buf + *buflen, req->src, 0, -+ req->nbytes, 0); -+ *buflen = *next_buflen; -+ *next_buflen = 0; -+ } -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "buf@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); -+ print_hex_dump(KERN_ERR, "next buf@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, next_buf, -+ *next_buflen, 1); -+#endif -+ -+ return ret; -+unmap_ctx: -+ ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE); -+ qi_cache_free(edesc); -+ return ret; -+} -+ -+static int ahash_finup_no_ctx(struct ahash_request *req) -+{ -+ struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); -+ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); -+ struct caam_hash_state *state = ahash_request_ctx(req); -+ struct caam_request *req_ctx = &state->caam_req; -+ struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; -+ struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; -+ int buflen = *current_buflen(state); -+ int qm_sg_bytes, src_nents, mapped_nents; -+ int digestsize = crypto_ahash_digestsize(ahash); -+ struct ahash_edesc *edesc; -+ struct dpaa2_sg_entry *sg_table; -+ int ret; -+ -+ src_nents = sg_nents_for_len(req->src, req->nbytes); -+ if (src_nents < 0) { -+ dev_err(ctx->dev, "Invalid number of src SG.\n"); -+ return src_nents; -+ } -+ -+ if (src_nents) { -+ mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, -+ DMA_TO_DEVICE); -+ if (!mapped_nents) { -+ dev_err(ctx->dev, "unable to DMA map source\n"); -+ return -ENOMEM; -+ } -+ } else { -+ mapped_nents = 0; -+ } -+ -+ /* allocate space for base edesc and link tables */ -+ edesc = qi_cache_zalloc(GFP_DMA | flags); -+ if (!edesc) { -+ dma_unmap_sg(ctx->dev, req->src, src_nents, DMA_TO_DEVICE); -+ return -ENOMEM; -+ } -+ -+ edesc->src_nents = src_nents; -+ qm_sg_bytes = (2 + mapped_nents) * sizeof(*sg_table); -+ sg_table = &edesc->sgt[0]; -+ -+ ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); -+ if (ret) -+ goto unmap; -+ -+ sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0); -+ -+ edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, -+ DMA_TO_DEVICE); -+ if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { -+ dev_err(ctx->dev, "unable to map S/G table\n"); -+ ret = -ENOMEM; -+ goto unmap; -+ } -+ edesc->qm_sg_bytes = qm_sg_bytes; -+ -+ edesc->dst_dma = dma_map_single(ctx->dev, req->result, digestsize, -+ DMA_FROM_DEVICE); -+ if (dma_mapping_error(ctx->dev, edesc->dst_dma)) { -+ dev_err(ctx->dev, "unable to map dst\n"); -+ edesc->dst_dma = 0; -+ ret = -ENOMEM; -+ goto unmap; -+ } -+ -+ memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); -+ dpaa2_fl_set_final(in_fle, true); -+ dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); -+ dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); -+ dpaa2_fl_set_len(in_fle, buflen + req->nbytes); -+ dpaa2_fl_set_format(out_fle, dpaa2_fl_single); -+ dpaa2_fl_set_addr(out_fle, edesc->dst_dma); -+ dpaa2_fl_set_len(out_fle, digestsize); -+ -+ req_ctx->flc = &ctx->flc[DIGEST]; -+ req_ctx->flc_dma = ctx->flc_dma[DIGEST]; -+ req_ctx->cbk = ahash_done; -+ req_ctx->ctx = &req->base; -+ req_ctx->edesc = edesc; -+ ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); -+ if (ret != -EINPROGRESS && -+ !(ret == -EBUSY && req->base.flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) -+ goto unmap; -+ -+ return ret; -+unmap: -+ ahash_unmap(ctx->dev, edesc, req, digestsize); -+ qi_cache_free(edesc); -+ return -ENOMEM; -+} -+ -+static int ahash_update_first(struct ahash_request *req) -+{ -+ struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); -+ struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); -+ struct caam_hash_state *state = ahash_request_ctx(req); -+ struct caam_request *req_ctx = &state->caam_req; -+ struct dpaa2_fl_entry *in_fle = &req_ctx->fd_flt[1]; -+ struct dpaa2_fl_entry *out_fle = &req_ctx->fd_flt[0]; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; -+ u8 *next_buf = alt_buf(state); -+ int *next_buflen = alt_buflen(state); -+ int to_hash; -+ int src_nents, mapped_nents; -+ struct ahash_edesc *edesc; -+ int ret = 0; -+ -+ *next_buflen = req->nbytes & (crypto_tfm_alg_blocksize(&ahash->base) - -+ 1); -+ to_hash = req->nbytes - *next_buflen; -+ -+ if (to_hash) { -+ struct dpaa2_sg_entry *sg_table; -+ -+ src_nents = sg_nents_for_len(req->src, -+ req->nbytes - (*next_buflen)); -+ if (src_nents < 0) { -+ dev_err(ctx->dev, "Invalid number of src SG.\n"); -+ return src_nents; -+ } -+ -+ if (src_nents) { -+ mapped_nents = dma_map_sg(ctx->dev, req->src, src_nents, -+ DMA_TO_DEVICE); -+ if (!mapped_nents) { -+ dev_err(ctx->dev, "unable to map source for DMA\n"); -+ return -ENOMEM; -+ } -+ } else { -+ mapped_nents = 0; -+ } -+ -+ /* allocate space for base edesc and link tables */ -+ edesc = qi_cache_zalloc(GFP_DMA | flags); -+ if (!edesc) { -+ dma_unmap_sg(ctx->dev, req->src, src_nents, -+ DMA_TO_DEVICE); -+ return -ENOMEM; -+ } -+ -+ edesc->src_nents = src_nents; -+ sg_table = &edesc->sgt[0]; -+ -+ memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); -+ dpaa2_fl_set_final(in_fle, true); -+ dpaa2_fl_set_len(in_fle, to_hash); -+ -+ if (mapped_nents > 1) { -+ int qm_sg_bytes; -+ -+ sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0); -+ qm_sg_bytes = mapped_nents * sizeof(*sg_table); -+ edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, -+ qm_sg_bytes, -+ DMA_TO_DEVICE); -+ if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { -+ dev_err(ctx->dev, "unable to map S/G table\n"); -+ ret = -ENOMEM; -+ goto unmap_ctx; -+ } -+ edesc->qm_sg_bytes = qm_sg_bytes; -+ dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); -+ dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); -+ } else { -+ dpaa2_fl_set_format(in_fle, dpaa2_fl_single); -+ dpaa2_fl_set_addr(in_fle, sg_dma_address(req->src)); -+ } -+ -+ if (*next_buflen) -+ scatterwalk_map_and_copy(next_buf, req->src, to_hash, -+ *next_buflen, 0); -+ -+ state->ctx_dma = dma_map_single(ctx->dev, state->caam_ctx, -+ ctx->ctx_len, DMA_FROM_DEVICE); -+ if (dma_mapping_error(ctx->dev, state->ctx_dma)) { -+ dev_err(ctx->dev, "unable to map ctx\n"); -+ state->ctx_dma = 0; -+ ret = -ENOMEM; -+ goto unmap_ctx; -+ } -+ -+ dpaa2_fl_set_format(out_fle, dpaa2_fl_single); -+ dpaa2_fl_set_addr(out_fle, state->ctx_dma); -+ dpaa2_fl_set_len(out_fle, ctx->ctx_len); -+ -+ req_ctx->flc = &ctx->flc[UPDATE_FIRST]; -+ req_ctx->flc_dma = ctx->flc_dma[UPDATE_FIRST]; -+ req_ctx->cbk = ahash_done_ctx_dst; -+ req_ctx->ctx = &req->base; -+ req_ctx->edesc = edesc; -+ -+ ret = dpaa2_caam_enqueue(ctx->dev, req_ctx); -+ if (ret != -EINPROGRESS && -+ !(ret == -EBUSY && req->base.flags & -+ CRYPTO_TFM_REQ_MAY_BACKLOG)) -+ goto unmap_ctx; -+ -+ state->update = ahash_update_ctx; -+ state->finup = ahash_finup_ctx; -+ state->final = ahash_final_ctx; -+ } else if (*next_buflen) { -+ state->update = ahash_update_no_ctx; -+ state->finup = ahash_finup_no_ctx; -+ state->final = ahash_final_no_ctx; -+ scatterwalk_map_and_copy(next_buf, req->src, 0, -+ req->nbytes, 0); -+ switch_buf(state); -+ } -+#ifdef DEBUG -+ print_hex_dump(KERN_ERR, "next buf@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen, 1); -+#endif -+ -+ return ret; -+unmap_ctx: -+ ahash_unmap_ctx(ctx->dev, edesc, req, ctx->ctx_len, DMA_TO_DEVICE); -+ qi_cache_free(edesc); -+ return ret; -+} -+ -+static int ahash_finup_first(struct ahash_request *req) -+{ -+ return ahash_digest(req); -+} -+ -+static int ahash_init(struct ahash_request *req) -+{ -+ struct caam_hash_state *state = ahash_request_ctx(req); -+ -+ state->update = ahash_update_first; -+ state->finup = ahash_finup_first; -+ state->final = ahash_final_no_ctx; -+ -+ state->ctx_dma = 0; -+ state->current_buf = 0; -+ state->buf_dma = 0; -+ state->buflen_0 = 0; -+ state->buflen_1 = 0; -+ -+ return 0; -+} -+ -+static int ahash_update(struct ahash_request *req) -+{ -+ struct caam_hash_state *state = ahash_request_ctx(req); -+ -+ return state->update(req); -+} -+ -+static int ahash_finup(struct ahash_request *req) -+{ -+ struct caam_hash_state *state = ahash_request_ctx(req); -+ -+ return state->finup(req); -+} -+ -+static int ahash_final(struct ahash_request *req) -+{ -+ struct caam_hash_state *state = ahash_request_ctx(req); -+ -+ return state->final(req); -+} -+ -+static int ahash_export(struct ahash_request *req, void *out) -+{ -+ struct caam_hash_state *state = ahash_request_ctx(req); -+ struct caam_export_state *export = out; -+ int len; -+ u8 *buf; -+ -+ if (state->current_buf) { -+ buf = state->buf_1; -+ len = state->buflen_1; -+ } else { -+ buf = state->buf_0; -+ len = state->buflen_0; -+ } -+ -+ memcpy(export->buf, buf, len); -+ memcpy(export->caam_ctx, state->caam_ctx, sizeof(export->caam_ctx)); -+ export->buflen = len; -+ export->update = state->update; -+ export->final = state->final; -+ export->finup = state->finup; -+ -+ return 0; -+} -+ -+static int ahash_import(struct ahash_request *req, const void *in) -+{ -+ struct caam_hash_state *state = ahash_request_ctx(req); -+ const struct caam_export_state *export = in; -+ -+ memset(state, 0, sizeof(*state)); -+ memcpy(state->buf_0, export->buf, export->buflen); -+ memcpy(state->caam_ctx, export->caam_ctx, sizeof(state->caam_ctx)); -+ state->buflen_0 = export->buflen; -+ state->update = export->update; -+ state->final = export->final; -+ state->finup = export->finup; -+ -+ return 0; -+} -+ -+struct caam_hash_template { -+ char name[CRYPTO_MAX_ALG_NAME]; -+ char driver_name[CRYPTO_MAX_ALG_NAME]; -+ char hmac_name[CRYPTO_MAX_ALG_NAME]; -+ char hmac_driver_name[CRYPTO_MAX_ALG_NAME]; -+ unsigned int blocksize; -+ struct ahash_alg template_ahash; -+ u32 alg_type; -+}; -+ -+/* ahash descriptors */ -+static struct caam_hash_template driver_hash[] = { -+ { -+ .name = "sha1", -+ .driver_name = "sha1-caam-qi2", -+ .hmac_name = "hmac(sha1)", -+ .hmac_driver_name = "hmac-sha1-caam-qi2", -+ .blocksize = SHA1_BLOCK_SIZE, -+ .template_ahash = { -+ .init = ahash_init, -+ .update = ahash_update, -+ .final = ahash_final, -+ .finup = ahash_finup, -+ .digest = ahash_digest, -+ .export = ahash_export, -+ .import = ahash_import, -+ .setkey = ahash_setkey, -+ .halg = { -+ .digestsize = SHA1_DIGEST_SIZE, -+ .statesize = sizeof(struct caam_export_state), -+ }, -+ }, -+ .alg_type = OP_ALG_ALGSEL_SHA1, -+ }, { -+ .name = "sha224", -+ .driver_name = "sha224-caam-qi2", -+ .hmac_name = "hmac(sha224)", -+ .hmac_driver_name = "hmac-sha224-caam-qi2", -+ .blocksize = SHA224_BLOCK_SIZE, -+ .template_ahash = { -+ .init = ahash_init, -+ .update = ahash_update, -+ .final = ahash_final, -+ .finup = ahash_finup, -+ .digest = ahash_digest, -+ .export = ahash_export, -+ .import = ahash_import, -+ .setkey = ahash_setkey, -+ .halg = { -+ .digestsize = SHA224_DIGEST_SIZE, -+ .statesize = sizeof(struct caam_export_state), -+ }, -+ }, -+ .alg_type = OP_ALG_ALGSEL_SHA224, -+ }, { -+ .name = "sha256", -+ .driver_name = "sha256-caam-qi2", -+ .hmac_name = "hmac(sha256)", -+ .hmac_driver_name = "hmac-sha256-caam-qi2", -+ .blocksize = SHA256_BLOCK_SIZE, -+ .template_ahash = { -+ .init = ahash_init, -+ .update = ahash_update, -+ .final = ahash_final, -+ .finup = ahash_finup, -+ .digest = ahash_digest, -+ .export = ahash_export, -+ .import = ahash_import, -+ .setkey = ahash_setkey, -+ .halg = { -+ .digestsize = SHA256_DIGEST_SIZE, -+ .statesize = sizeof(struct caam_export_state), -+ }, -+ }, -+ .alg_type = OP_ALG_ALGSEL_SHA256, -+ }, { -+ .name = "sha384", -+ .driver_name = "sha384-caam-qi2", -+ .hmac_name = "hmac(sha384)", -+ .hmac_driver_name = "hmac-sha384-caam-qi2", -+ .blocksize = SHA384_BLOCK_SIZE, -+ .template_ahash = { -+ .init = ahash_init, -+ .update = ahash_update, -+ .final = ahash_final, -+ .finup = ahash_finup, -+ .digest = ahash_digest, -+ .export = ahash_export, -+ .import = ahash_import, -+ .setkey = ahash_setkey, -+ .halg = { -+ .digestsize = SHA384_DIGEST_SIZE, -+ .statesize = sizeof(struct caam_export_state), -+ }, -+ }, -+ .alg_type = OP_ALG_ALGSEL_SHA384, -+ }, { -+ .name = "sha512", -+ .driver_name = "sha512-caam-qi2", -+ .hmac_name = "hmac(sha512)", -+ .hmac_driver_name = "hmac-sha512-caam-qi2", -+ .blocksize = SHA512_BLOCK_SIZE, -+ .template_ahash = { -+ .init = ahash_init, -+ .update = ahash_update, -+ .final = ahash_final, -+ .finup = ahash_finup, -+ .digest = ahash_digest, -+ .export = ahash_export, -+ .import = ahash_import, -+ .setkey = ahash_setkey, -+ .halg = { -+ .digestsize = SHA512_DIGEST_SIZE, -+ .statesize = sizeof(struct caam_export_state), -+ }, -+ }, -+ .alg_type = OP_ALG_ALGSEL_SHA512, -+ }, { -+ .name = "md5", -+ .driver_name = "md5-caam-qi2", -+ .hmac_name = "hmac(md5)", -+ .hmac_driver_name = "hmac-md5-caam-qi2", -+ .blocksize = MD5_BLOCK_WORDS * 4, -+ .template_ahash = { -+ .init = ahash_init, -+ .update = ahash_update, -+ .final = ahash_final, -+ .finup = ahash_finup, -+ .digest = ahash_digest, -+ .export = ahash_export, -+ .import = ahash_import, -+ .setkey = ahash_setkey, -+ .halg = { -+ .digestsize = MD5_DIGEST_SIZE, -+ .statesize = sizeof(struct caam_export_state), -+ }, -+ }, -+ .alg_type = OP_ALG_ALGSEL_MD5, -+ } -+}; -+ -+struct caam_hash_alg { -+ struct list_head entry; -+ struct device *dev; -+ int alg_type; -+ struct ahash_alg ahash_alg; -+}; -+ -+static int caam_hash_cra_init(struct crypto_tfm *tfm) -+{ -+ struct crypto_ahash *ahash = __crypto_ahash_cast(tfm); -+ struct crypto_alg *base = tfm->__crt_alg; -+ struct hash_alg_common *halg = -+ container_of(base, struct hash_alg_common, base); -+ struct ahash_alg *alg = -+ container_of(halg, struct ahash_alg, halg); -+ struct caam_hash_alg *caam_hash = -+ container_of(alg, struct caam_hash_alg, ahash_alg); -+ struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); -+ /* Sizes for MDHA running digests: MD5, SHA1, 224, 256, 384, 512 */ -+ static const u8 runninglen[] = { HASH_MSG_LEN + MD5_DIGEST_SIZE, -+ HASH_MSG_LEN + SHA1_DIGEST_SIZE, -+ HASH_MSG_LEN + 32, -+ HASH_MSG_LEN + SHA256_DIGEST_SIZE, -+ HASH_MSG_LEN + 64, -+ HASH_MSG_LEN + SHA512_DIGEST_SIZE }; -+ dma_addr_t dma_addr; -+ int i; -+ -+ ctx->dev = caam_hash->dev; -+ -+ dma_addr = dma_map_single_attrs(ctx->dev, ctx->flc, sizeof(ctx->flc), -+ DMA_BIDIRECTIONAL, -+ DMA_ATTR_SKIP_CPU_SYNC); -+ if (dma_mapping_error(ctx->dev, dma_addr)) { -+ dev_err(ctx->dev, "unable to map shared descriptors\n"); -+ return -ENOMEM; -+ } -+ -+ for (i = 0; i < HASH_NUM_OP; i++) -+ ctx->flc_dma[i] = dma_addr + i * sizeof(ctx->flc[i]); -+ -+ /* copy descriptor header template value */ -+ ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; -+ -+ ctx->ctx_len = runninglen[(ctx->adata.algtype & -+ OP_ALG_ALGSEL_SUBMASK) >> -+ OP_ALG_ALGSEL_SHIFT]; -+ -+ crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), -+ sizeof(struct caam_hash_state)); -+ -+ return ahash_set_sh_desc(ahash); -+} -+ -+static void caam_hash_cra_exit(struct crypto_tfm *tfm) -+{ -+ struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); -+ -+ dma_unmap_single_attrs(ctx->dev, ctx->flc_dma[0], sizeof(ctx->flc), -+ DMA_BIDIRECTIONAL, DMA_ATTR_SKIP_CPU_SYNC); -+} -+ -+static struct caam_hash_alg *caam_hash_alloc(struct device *dev, -+ struct caam_hash_template *template, bool keyed) -+{ -+ struct caam_hash_alg *t_alg; -+ struct ahash_alg *halg; -+ struct crypto_alg *alg; -+ -+ t_alg = kzalloc(sizeof(*t_alg), GFP_KERNEL); -+ if (!t_alg) -+ return ERR_PTR(-ENOMEM); -+ -+ t_alg->ahash_alg = template->template_ahash; -+ halg = &t_alg->ahash_alg; -+ alg = &halg->halg.base; -+ -+ if (keyed) { -+ snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", -+ template->hmac_name); -+ snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", -+ template->hmac_driver_name); -+ } else { -+ snprintf(alg->cra_name, CRYPTO_MAX_ALG_NAME, "%s", -+ template->name); -+ snprintf(alg->cra_driver_name, CRYPTO_MAX_ALG_NAME, "%s", -+ template->driver_name); -+ t_alg->ahash_alg.setkey = NULL; -+ } -+ alg->cra_module = THIS_MODULE; -+ alg->cra_init = caam_hash_cra_init; -+ alg->cra_exit = caam_hash_cra_exit; -+ alg->cra_ctxsize = sizeof(struct caam_hash_ctx); -+ alg->cra_priority = CAAM_CRA_PRIORITY; -+ alg->cra_blocksize = template->blocksize; -+ alg->cra_alignmask = 0; -+ alg->cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_TYPE_AHASH; -+ alg->cra_type = &crypto_ahash_type; -+ -+ t_alg->alg_type = template->alg_type; -+ t_alg->dev = dev; -+ -+ return t_alg; -+} -+ -+static void dpaa2_caam_fqdan_cb(struct dpaa2_io_notification_ctx *nctx) -+{ -+ struct dpaa2_caam_priv_per_cpu *ppriv; -+ -+ ppriv = container_of(nctx, struct dpaa2_caam_priv_per_cpu, nctx); -+ napi_schedule_irqoff(&ppriv->napi); -+} -+ -+static int __cold dpaa2_dpseci_dpio_setup(struct dpaa2_caam_priv *priv) -+{ -+ struct device *dev = priv->dev; -+ struct dpaa2_io_notification_ctx *nctx; -+ struct dpaa2_caam_priv_per_cpu *ppriv; -+ int err, i = 0, cpu; -+ -+ for_each_online_cpu(cpu) { -+ ppriv = per_cpu_ptr(priv->ppriv, cpu); -+ ppriv->priv = priv; -+ nctx = &ppriv->nctx; -+ nctx->is_cdan = 0; -+ nctx->id = ppriv->rsp_fqid; -+ nctx->desired_cpu = cpu; -+ nctx->cb = dpaa2_caam_fqdan_cb; -+ -+ /* Register notification callbacks */ -+ err = dpaa2_io_service_register(NULL, nctx); -+ if (unlikely(err)) { -+ dev_dbg(dev, "No affine DPIO for cpu %d\n", cpu); -+ nctx->cb = NULL; -+ /* -+ * If no affine DPIO for this core, there's probably -+ * none available for next cores either. Signal we want -+ * to retry later, in case the DPIO devices weren't -+ * probed yet. -+ */ -+ err = -EPROBE_DEFER; -+ goto err; -+ } -+ -+ ppriv->store = dpaa2_io_store_create(DPAA2_CAAM_STORE_SIZE, -+ dev); -+ if (unlikely(!ppriv->store)) { -+ dev_err(dev, "dpaa2_io_store_create() failed\n"); -+ goto err; -+ } -+ -+ if (++i == priv->num_pairs) -+ break; -+ } -+ -+ return 0; -+ -+err: -+ for_each_online_cpu(cpu) { -+ ppriv = per_cpu_ptr(priv->ppriv, cpu); -+ if (!ppriv->nctx.cb) -+ break; -+ dpaa2_io_service_deregister(NULL, &ppriv->nctx); -+ } -+ -+ for_each_online_cpu(cpu) { -+ ppriv = per_cpu_ptr(priv->ppriv, cpu); -+ if (!ppriv->store) -+ break; -+ dpaa2_io_store_destroy(ppriv->store); -+ } -+ -+ return err; -+} -+ -+static void __cold dpaa2_dpseci_dpio_free(struct dpaa2_caam_priv *priv) -+{ -+ struct dpaa2_caam_priv_per_cpu *ppriv; -+ int i = 0, cpu; -+ -+ for_each_online_cpu(cpu) { -+ ppriv = per_cpu_ptr(priv->ppriv, cpu); -+ dpaa2_io_service_deregister(NULL, &ppriv->nctx); -+ dpaa2_io_store_destroy(ppriv->store); -+ -+ if (++i == priv->num_pairs) -+ return; -+ } -+} -+ -+static int dpaa2_dpseci_bind(struct dpaa2_caam_priv *priv) -+{ -+ struct dpseci_rx_queue_cfg rx_queue_cfg; -+ struct device *dev = priv->dev; -+ struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); -+ struct dpaa2_caam_priv_per_cpu *ppriv; -+ int err = 0, i = 0, cpu; -+ -+ /* Configure Rx queues */ -+ for_each_online_cpu(cpu) { -+ ppriv = per_cpu_ptr(priv->ppriv, cpu); -+ -+ rx_queue_cfg.options = DPSECI_QUEUE_OPT_DEST | -+ DPSECI_QUEUE_OPT_USER_CTX; -+ rx_queue_cfg.order_preservation_en = 0; -+ rx_queue_cfg.dest_cfg.dest_type = DPSECI_DEST_DPIO; -+ rx_queue_cfg.dest_cfg.dest_id = ppriv->nctx.dpio_id; -+ /* -+ * Rx priority (WQ) doesn't really matter, since we use -+ * pull mode, i.e. volatile dequeues from specific FQs -+ */ -+ rx_queue_cfg.dest_cfg.priority = 0; -+ rx_queue_cfg.user_ctx = ppriv->nctx.qman64; -+ -+ err = dpseci_set_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, -+ &rx_queue_cfg); -+ if (err) { -+ dev_err(dev, "dpseci_set_rx_queue() failed with err %d\n", -+ err); -+ return err; -+ } -+ -+ if (++i == priv->num_pairs) -+ break; -+ } -+ -+ return err; -+} -+ -+static void dpaa2_dpseci_congestion_free(struct dpaa2_caam_priv *priv) -+{ -+ struct device *dev = priv->dev; -+ -+ if (!priv->cscn_mem) -+ return; -+ -+ dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); -+ kfree(priv->cscn_mem); -+} -+ -+static void dpaa2_dpseci_free(struct dpaa2_caam_priv *priv) -+{ -+ struct device *dev = priv->dev; -+ struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); -+ -+ dpaa2_dpseci_congestion_free(priv); -+ dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); -+} -+ -+static void dpaa2_caam_process_fd(struct dpaa2_caam_priv *priv, -+ const struct dpaa2_fd *fd) -+{ -+ struct caam_request *req; -+ u32 fd_err; -+ -+ if (dpaa2_fd_get_format(fd) != dpaa2_fd_list) { -+ dev_err(priv->dev, "Only Frame List FD format is supported!\n"); -+ return; -+ } -+ -+ fd_err = dpaa2_fd_get_ctrl(fd) & FD_CTRL_ERR_MASK; -+ if (unlikely(fd_err)) -+ dev_err(priv->dev, "FD error: %08x\n", fd_err); -+ -+ /* -+ * FD[ADDR] is guaranteed to be valid, irrespective of errors reported -+ * in FD[ERR] or FD[FRC]. -+ */ -+ req = dpaa2_caam_iova_to_virt(priv, dpaa2_fd_get_addr(fd)); -+ dma_unmap_single(priv->dev, req->fd_flt_dma, sizeof(req->fd_flt), -+ DMA_BIDIRECTIONAL); -+ req->cbk(req->ctx, dpaa2_fd_get_frc(fd)); -+} -+ -+static int dpaa2_caam_pull_fq(struct dpaa2_caam_priv_per_cpu *ppriv) -+{ -+ int err; -+ -+ /* Retry while portal is busy */ -+ do { -+ err = dpaa2_io_service_pull_fq(NULL, ppriv->rsp_fqid, -+ ppriv->store); -+ } while (err == -EBUSY); -+ -+ if (unlikely(err)) -+ dev_err(ppriv->priv->dev, "dpaa2_io_service_pull err %d", err); -+ -+ return err; -+} -+ -+static int dpaa2_caam_store_consume(struct dpaa2_caam_priv_per_cpu *ppriv) -+{ -+ struct dpaa2_dq *dq; -+ int cleaned = 0, is_last; -+ -+ do { -+ dq = dpaa2_io_store_next(ppriv->store, &is_last); -+ if (unlikely(!dq)) { -+ if (unlikely(!is_last)) { -+ dev_dbg(ppriv->priv->dev, -+ "FQ %d returned no valid frames\n", -+ ppriv->rsp_fqid); -+ /* -+ * MUST retry until we get some sort of -+ * valid response token (be it "empty dequeue" -+ * or a valid frame). -+ */ -+ continue; -+ } -+ break; -+ } -+ -+ /* Process FD */ -+ dpaa2_caam_process_fd(ppriv->priv, dpaa2_dq_fd(dq)); -+ cleaned++; -+ } while (!is_last); -+ -+ return cleaned; -+} -+ -+static int dpaa2_dpseci_poll(struct napi_struct *napi, int budget) -+{ -+ struct dpaa2_caam_priv_per_cpu *ppriv; -+ struct dpaa2_caam_priv *priv; -+ int err, cleaned = 0, store_cleaned; -+ -+ ppriv = container_of(napi, struct dpaa2_caam_priv_per_cpu, napi); -+ priv = ppriv->priv; -+ -+ if (unlikely(dpaa2_caam_pull_fq(ppriv))) -+ return 0; -+ -+ do { -+ store_cleaned = dpaa2_caam_store_consume(ppriv); -+ cleaned += store_cleaned; -+ -+ if (store_cleaned == 0 || -+ cleaned > budget - DPAA2_CAAM_STORE_SIZE) -+ break; -+ -+ /* Try to dequeue some more */ -+ err = dpaa2_caam_pull_fq(ppriv); -+ if (unlikely(err)) -+ break; -+ } while (1); -+ -+ if (cleaned < budget) { -+ napi_complete_done(napi, cleaned); -+ err = dpaa2_io_service_rearm(NULL, &ppriv->nctx); -+ if (unlikely(err)) -+ dev_err(priv->dev, "Notification rearm failed: %d\n", -+ err); -+ } -+ -+ return cleaned; -+} -+ -+static int dpaa2_dpseci_congestion_setup(struct dpaa2_caam_priv *priv, -+ u16 token) -+{ -+ struct dpseci_congestion_notification_cfg cong_notif_cfg = { 0 }; -+ struct device *dev = priv->dev; -+ int err; -+ -+ /* -+ * Congestion group feature supported starting with DPSECI API v5.1 -+ * and only when object has been created with this capability. -+ */ -+ if ((DPSECI_VER(priv->major_ver, priv->minor_ver) < DPSECI_VER(5, 1)) || -+ !(priv->dpseci_attr.options & DPSECI_OPT_HAS_CG)) -+ return 0; -+ -+ priv->cscn_mem = kzalloc(DPAA2_CSCN_SIZE + DPAA2_CSCN_ALIGN, -+ GFP_KERNEL | GFP_DMA); -+ if (!priv->cscn_mem) -+ return -ENOMEM; -+ -+ priv->cscn_mem_aligned = PTR_ALIGN(priv->cscn_mem, DPAA2_CSCN_ALIGN); -+ priv->cscn_dma = dma_map_single(dev, priv->cscn_mem_aligned, -+ DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); -+ if (dma_mapping_error(dev, priv->cscn_dma)) { -+ dev_err(dev, "Error mapping CSCN memory area\n"); -+ err = -ENOMEM; -+ goto err_dma_map; -+ } -+ -+ cong_notif_cfg.units = DPSECI_CONGESTION_UNIT_BYTES; -+ cong_notif_cfg.threshold_entry = DPAA2_SEC_CONG_ENTRY_THRESH; -+ cong_notif_cfg.threshold_exit = DPAA2_SEC_CONG_EXIT_THRESH; -+ cong_notif_cfg.message_ctx = (u64)priv; -+ cong_notif_cfg.message_iova = priv->cscn_dma; -+ cong_notif_cfg.notification_mode = DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER | -+ DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT | -+ DPSECI_CGN_MODE_COHERENT_WRITE; -+ -+ err = dpseci_set_congestion_notification(priv->mc_io, 0, token, -+ &cong_notif_cfg); -+ if (err) { -+ dev_err(dev, "dpseci_set_congestion_notification failed\n"); -+ goto err_set_cong; -+ } -+ -+ return 0; -+ -+err_set_cong: -+ dma_unmap_single(dev, priv->cscn_dma, DPAA2_CSCN_SIZE, DMA_FROM_DEVICE); -+err_dma_map: -+ kfree(priv->cscn_mem); -+ -+ return err; -+} -+ -+static int __cold dpaa2_dpseci_setup(struct fsl_mc_device *ls_dev) -+{ -+ struct device *dev = &ls_dev->dev; -+ struct dpaa2_caam_priv *priv; -+ struct dpaa2_caam_priv_per_cpu *ppriv; -+ int err, cpu; -+ u8 i; -+ -+ priv = dev_get_drvdata(dev); -+ -+ priv->dev = dev; -+ priv->dpsec_id = ls_dev->obj_desc.id; -+ -+ /* Get a handle for the DPSECI this interface is associate with */ -+ err = dpseci_open(priv->mc_io, 0, priv->dpsec_id, &ls_dev->mc_handle); -+ if (err) { -+ dev_err(dev, "dpsec_open() failed: %d\n", err); -+ goto err_open; -+ } -+ -+ dev_info(dev, "Opened dpseci object successfully\n"); -+ -+ err = dpseci_get_api_version(priv->mc_io, 0, &priv->major_ver, -+ &priv->minor_ver); -+ if (err) { -+ dev_err(dev, "dpseci_get_api_version() failed\n"); -+ goto err_get_vers; -+ } -+ -+ err = dpseci_get_attributes(priv->mc_io, 0, ls_dev->mc_handle, -+ &priv->dpseci_attr); -+ if (err) { -+ dev_err(dev, "dpseci_get_attributes() failed\n"); -+ goto err_get_vers; -+ } -+ -+ err = dpseci_get_sec_attr(priv->mc_io, 0, ls_dev->mc_handle, -+ &priv->sec_attr); -+ if (err) { -+ dev_err(dev, "dpseci_get_sec_attr() failed\n"); -+ goto err_get_vers; -+ } -+ -+ err = dpaa2_dpseci_congestion_setup(priv, ls_dev->mc_handle); -+ if (err) { -+ dev_err(dev, "setup_congestion() failed\n"); -+ goto err_get_vers; -+ } -+ -+ priv->num_pairs = min(priv->dpseci_attr.num_rx_queues, -+ priv->dpseci_attr.num_tx_queues); -+ if (priv->num_pairs > num_online_cpus()) { -+ dev_warn(dev, "%d queues won't be used\n", -+ priv->num_pairs - num_online_cpus()); -+ priv->num_pairs = num_online_cpus(); -+ } -+ -+ for (i = 0; i < priv->dpseci_attr.num_rx_queues; i++) { -+ err = dpseci_get_rx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, -+ &priv->rx_queue_attr[i]); -+ if (err) { -+ dev_err(dev, "dpseci_get_rx_queue() failed\n"); -+ goto err_get_rx_queue; -+ } -+ } -+ -+ for (i = 0; i < priv->dpseci_attr.num_tx_queues; i++) { -+ err = dpseci_get_tx_queue(priv->mc_io, 0, ls_dev->mc_handle, i, -+ &priv->tx_queue_attr[i]); -+ if (err) { -+ dev_err(dev, "dpseci_get_tx_queue() failed\n"); -+ goto err_get_rx_queue; -+ } -+ } -+ -+ i = 0; -+ for_each_online_cpu(cpu) { -+ dev_info(dev, "prio %d: rx queue %d, tx queue %d\n", i, -+ priv->rx_queue_attr[i].fqid, -+ priv->tx_queue_attr[i].fqid); -+ -+ ppriv = per_cpu_ptr(priv->ppriv, cpu); -+ ppriv->req_fqid = priv->tx_queue_attr[i].fqid; -+ ppriv->rsp_fqid = priv->rx_queue_attr[i].fqid; -+ ppriv->prio = i; -+ -+ ppriv->net_dev.dev = *dev; -+ INIT_LIST_HEAD(&ppriv->net_dev.napi_list); -+ netif_napi_add(&ppriv->net_dev, &ppriv->napi, dpaa2_dpseci_poll, -+ DPAA2_CAAM_NAPI_WEIGHT); -+ if (++i == priv->num_pairs) -+ break; -+ } -+ -+ return 0; -+ -+err_get_rx_queue: -+ dpaa2_dpseci_congestion_free(priv); -+err_get_vers: -+ dpseci_close(priv->mc_io, 0, ls_dev->mc_handle); -+err_open: -+ return err; -+} -+ -+static int dpaa2_dpseci_enable(struct dpaa2_caam_priv *priv) -+{ -+ struct device *dev = priv->dev; -+ struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); -+ struct dpaa2_caam_priv_per_cpu *ppriv; -+ int err, i; -+ -+ for (i = 0; i < priv->num_pairs; i++) { -+ ppriv = per_cpu_ptr(priv->ppriv, i); -+ napi_enable(&ppriv->napi); -+ } -+ -+ err = dpseci_enable(priv->mc_io, 0, ls_dev->mc_handle); -+ if (err) { -+ dev_err(dev, "dpseci_enable() failed\n"); -+ return err; -+ } -+ -+ dev_info(dev, "DPSECI version %d.%d\n", -+ priv->major_ver, -+ priv->minor_ver); -+ -+ return 0; -+} -+ -+static int __cold dpaa2_dpseci_disable(struct dpaa2_caam_priv *priv) -+{ -+ struct device *dev = priv->dev; -+ struct dpaa2_caam_priv_per_cpu *ppriv; -+ struct fsl_mc_device *ls_dev = to_fsl_mc_device(dev); -+ int i, err = 0, enabled; -+ -+ err = dpseci_disable(priv->mc_io, 0, ls_dev->mc_handle); -+ if (err) { -+ dev_err(dev, "dpseci_disable() failed\n"); -+ return err; -+ } -+ -+ err = dpseci_is_enabled(priv->mc_io, 0, ls_dev->mc_handle, &enabled); -+ if (err) { -+ dev_err(dev, "dpseci_is_enabled() failed\n"); -+ return err; -+ } -+ -+ dev_dbg(dev, "disable: %s\n", enabled ? "false" : "true"); -+ -+ for (i = 0; i < priv->num_pairs; i++) { -+ ppriv = per_cpu_ptr(priv->ppriv, i); -+ napi_disable(&ppriv->napi); -+ netif_napi_del(&ppriv->napi); -+ } -+ -+ return 0; -+} -+ -+static struct list_head alg_list; -+static struct list_head hash_list; -+ -+static int dpaa2_caam_probe(struct fsl_mc_device *dpseci_dev) -+{ -+ struct device *dev; -+ struct dpaa2_caam_priv *priv; -+ int i, err = 0; -+ bool registered = false; -+ -+ /* -+ * There is no way to get CAAM endianness - there is no direct register -+ * space access and MC f/w does not provide this attribute. -+ * All DPAA2-based SoCs have little endian CAAM, thus hard-code this -+ * property. -+ */ -+ caam_little_end = true; -+ -+ caam_imx = false; -+ -+ dev = &dpseci_dev->dev; -+ -+ priv = kzalloc(sizeof(*priv), GFP_KERNEL); -+ if (!priv) -+ return -ENOMEM; -+ -+ dev_set_drvdata(dev, priv); -+ -+ priv->domain = iommu_get_domain_for_dev(dev); -+ -+ qi_cache = kmem_cache_create("dpaa2_caamqicache", CAAM_QI_MEMCACHE_SIZE, -+ 0, SLAB_CACHE_DMA, NULL); -+ if (!qi_cache) { -+ dev_err(dev, "Can't allocate SEC cache\n"); -+ err = -ENOMEM; -+ goto err_qicache; -+ } -+ -+ err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49)); -+ if (err) { -+ dev_err(dev, "dma_set_mask_and_coherent() failed\n"); -+ goto err_dma_mask; -+ } -+ -+ /* Obtain a MC portal */ -+ err = fsl_mc_portal_allocate(dpseci_dev, 0, &priv->mc_io); -+ if (err) { -+ if (err == -ENXIO) -+ err = -EPROBE_DEFER; -+ else -+ dev_err(dev, "MC portal allocation failed\n"); -+ -+ goto err_dma_mask; -+ } -+ -+ priv->ppriv = alloc_percpu(*priv->ppriv); -+ if (!priv->ppriv) { -+ dev_err(dev, "alloc_percpu() failed\n"); -+ goto err_alloc_ppriv; -+ } -+ -+ /* DPSECI initialization */ -+ err = dpaa2_dpseci_setup(dpseci_dev); -+ if (err < 0) { -+ dev_err(dev, "dpaa2_dpseci_setup() failed\n"); -+ goto err_dpseci_setup; -+ } -+ -+ /* DPIO */ -+ err = dpaa2_dpseci_dpio_setup(priv); -+ if (err) { -+ dev_err(dev, "dpaa2_dpseci_dpio_setup() failed\n"); -+ goto err_dpio_setup; -+ } -+ -+ /* DPSECI binding to DPIO */ -+ err = dpaa2_dpseci_bind(priv); -+ if (err) { -+ dev_err(dev, "dpaa2_dpseci_bind() failed\n"); -+ goto err_bind; -+ } -+ -+ /* DPSECI enable */ -+ err = dpaa2_dpseci_enable(priv); -+ if (err) { -+ dev_err(dev, "dpaa2_dpseci_enable() failed"); -+ goto err_bind; -+ } -+ -+ /* register crypto algorithms the device supports */ -+ INIT_LIST_HEAD(&alg_list); -+ for (i = 0; i < ARRAY_SIZE(driver_algs); i++) { -+ struct caam_crypto_alg *t_alg; -+ struct caam_alg_template *alg = driver_algs + i; -+ u32 alg_sel = alg->class1_alg_type & OP_ALG_ALGSEL_MASK; -+ -+ /* Skip DES algorithms if not supported by device */ -+ if (!priv->sec_attr.des_acc_num && -+ ((alg_sel == OP_ALG_ALGSEL_3DES) || -+ (alg_sel == OP_ALG_ALGSEL_DES))) -+ continue; -+ -+ /* Skip AES algorithms if not supported by device */ -+ if (!priv->sec_attr.aes_acc_num && -+ (alg_sel == OP_ALG_ALGSEL_AES)) -+ continue; -+ -+ t_alg = caam_alg_alloc(alg); -+ if (IS_ERR(t_alg)) { -+ err = PTR_ERR(t_alg); -+ dev_warn(dev, "%s alg allocation failed: %d\n", -+ alg->driver_name, err); -+ continue; -+ } -+ t_alg->caam.dev = dev; -+ -+ err = crypto_register_alg(&t_alg->crypto_alg); -+ if (err) { -+ dev_warn(dev, "%s alg registration failed: %d\n", -+ t_alg->crypto_alg.cra_driver_name, err); -+ kfree(t_alg); -+ continue; -+ } -+ -+ list_add_tail(&t_alg->entry, &alg_list); -+ registered = true; -+ } -+ -+ for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { -+ struct caam_aead_alg *t_alg = driver_aeads + i; -+ u32 c1_alg_sel = t_alg->caam.class1_alg_type & -+ OP_ALG_ALGSEL_MASK; -+ u32 c2_alg_sel = t_alg->caam.class2_alg_type & -+ OP_ALG_ALGSEL_MASK; -+ -+ /* Skip DES algorithms if not supported by device */ -+ if (!priv->sec_attr.des_acc_num && -+ ((c1_alg_sel == OP_ALG_ALGSEL_3DES) || -+ (c1_alg_sel == OP_ALG_ALGSEL_DES))) -+ continue; -+ -+ /* Skip AES algorithms if not supported by device */ -+ if (!priv->sec_attr.aes_acc_num && -+ (c1_alg_sel == OP_ALG_ALGSEL_AES)) -+ continue; -+ -+ /* -+ * Skip algorithms requiring message digests -+ * if MD not supported by device. -+ */ -+ if (!priv->sec_attr.md_acc_num && c2_alg_sel) -+ continue; -+ -+ t_alg->caam.dev = dev; -+ caam_aead_alg_init(t_alg); -+ -+ err = crypto_register_aead(&t_alg->aead); -+ if (err) { -+ dev_warn(dev, "%s alg registration failed: %d\n", -+ t_alg->aead.base.cra_driver_name, err); -+ continue; -+ } -+ -+ t_alg->registered = true; -+ registered = true; -+ } -+ if (registered) -+ dev_info(dev, "algorithms registered in /proc/crypto\n"); -+ -+ /* register hash algorithms the device supports */ -+ INIT_LIST_HEAD(&hash_list); -+ -+ /* -+ * Skip registration of any hashing algorithms if MD block -+ * is not present. -+ */ -+ if (!priv->sec_attr.md_acc_num) -+ return 0; -+ -+ for (i = 0; i < ARRAY_SIZE(driver_hash); i++) { -+ struct caam_hash_alg *t_alg; -+ struct caam_hash_template *alg = driver_hash + i; -+ -+ /* register hmac version */ -+ t_alg = caam_hash_alloc(dev, alg, true); -+ if (IS_ERR(t_alg)) { -+ err = PTR_ERR(t_alg); -+ dev_warn(dev, "%s hash alg allocation failed: %d\n", -+ alg->driver_name, err); -+ continue; -+ } -+ -+ err = crypto_register_ahash(&t_alg->ahash_alg); -+ if (err) { -+ dev_warn(dev, "%s alg registration failed: %d\n", -+ t_alg->ahash_alg.halg.base.cra_driver_name, -+ err); -+ kfree(t_alg); -+ } else { -+ list_add_tail(&t_alg->entry, &hash_list); -+ } -+ -+ /* register unkeyed version */ -+ t_alg = caam_hash_alloc(dev, alg, false); -+ if (IS_ERR(t_alg)) { -+ err = PTR_ERR(t_alg); -+ dev_warn(dev, "%s alg allocation failed: %d\n", -+ alg->driver_name, err); -+ continue; -+ } -+ -+ err = crypto_register_ahash(&t_alg->ahash_alg); -+ if (err) { -+ dev_warn(dev, "%s alg registration failed: %d\n", -+ t_alg->ahash_alg.halg.base.cra_driver_name, -+ err); -+ kfree(t_alg); -+ } else { -+ list_add_tail(&t_alg->entry, &hash_list); -+ } -+ } -+ if (!list_empty(&hash_list)) -+ dev_info(dev, "hash algorithms registered in /proc/crypto\n"); -+ -+ return err; -+ -+err_bind: -+ dpaa2_dpseci_dpio_free(priv); -+err_dpio_setup: -+ dpaa2_dpseci_free(priv); -+err_dpseci_setup: -+ free_percpu(priv->ppriv); -+err_alloc_ppriv: -+ fsl_mc_portal_free(priv->mc_io); -+err_dma_mask: -+ kmem_cache_destroy(qi_cache); -+err_qicache: -+ dev_set_drvdata(dev, NULL); -+ -+ return err; -+} -+ -+static int __cold dpaa2_caam_remove(struct fsl_mc_device *ls_dev) -+{ -+ struct device *dev; -+ struct dpaa2_caam_priv *priv; -+ int i; -+ -+ dev = &ls_dev->dev; -+ priv = dev_get_drvdata(dev); -+ -+ for (i = 0; i < ARRAY_SIZE(driver_aeads); i++) { -+ struct caam_aead_alg *t_alg = driver_aeads + i; -+ -+ if (t_alg->registered) -+ crypto_unregister_aead(&t_alg->aead); -+ } -+ -+ if (alg_list.next) { -+ struct caam_crypto_alg *t_alg, *n; -+ -+ list_for_each_entry_safe(t_alg, n, &alg_list, entry) { -+ crypto_unregister_alg(&t_alg->crypto_alg); -+ list_del(&t_alg->entry); -+ kfree(t_alg); -+ } -+ } -+ -+ if (hash_list.next) { -+ struct caam_hash_alg *t_hash_alg, *p; -+ -+ list_for_each_entry_safe(t_hash_alg, p, &hash_list, entry) { -+ crypto_unregister_ahash(&t_hash_alg->ahash_alg); -+ list_del(&t_hash_alg->entry); -+ kfree(t_hash_alg); -+ } -+ } -+ -+ dpaa2_dpseci_disable(priv); -+ dpaa2_dpseci_dpio_free(priv); -+ dpaa2_dpseci_free(priv); -+ free_percpu(priv->ppriv); -+ fsl_mc_portal_free(priv->mc_io); -+ dev_set_drvdata(dev, NULL); -+ kmem_cache_destroy(qi_cache); -+ -+ return 0; -+} -+ -+int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req) -+{ -+ struct dpaa2_fd fd; -+ struct dpaa2_caam_priv *priv = dev_get_drvdata(dev); -+ int err = 0, i, id; -+ -+ if (IS_ERR(req)) -+ return PTR_ERR(req); -+ -+ if (priv->cscn_mem) { -+ dma_sync_single_for_cpu(priv->dev, priv->cscn_dma, -+ DPAA2_CSCN_SIZE, -+ DMA_FROM_DEVICE); -+ if (unlikely(dpaa2_cscn_state_congested(priv->cscn_mem_aligned))) { -+ dev_dbg_ratelimited(dev, "Dropping request\n"); -+ return -EBUSY; -+ } -+ } -+ -+ dpaa2_fl_set_flc(&req->fd_flt[1], req->flc_dma); -+ -+ req->fd_flt_dma = dma_map_single(dev, req->fd_flt, sizeof(req->fd_flt), -+ DMA_BIDIRECTIONAL); -+ if (dma_mapping_error(dev, req->fd_flt_dma)) { -+ dev_err(dev, "DMA mapping error for QI enqueue request\n"); -+ goto err_out; -+ } -+ -+ memset(&fd, 0, sizeof(fd)); -+ dpaa2_fd_set_format(&fd, dpaa2_fd_list); -+ dpaa2_fd_set_addr(&fd, req->fd_flt_dma); -+ dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1])); -+ dpaa2_fd_set_flc(&fd, req->flc_dma); -+ -+ /* -+ * There is no guarantee that preemption is disabled here, -+ * thus take action. -+ */ -+ preempt_disable(); -+ id = smp_processor_id() % priv->dpseci_attr.num_tx_queues; -+ for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) { -+ err = dpaa2_io_service_enqueue_fq(NULL, -+ priv->tx_queue_attr[id].fqid, -+ &fd); -+ if (err != -EBUSY) -+ break; -+ } -+ preempt_enable(); -+ -+ if (unlikely(err < 0)) { -+ dev_err(dev, "Error enqueuing frame: %d\n", err); -+ goto err_out; -+ } -+ -+ return -EINPROGRESS; -+ -+err_out: -+ dma_unmap_single(dev, req->fd_flt_dma, sizeof(req->fd_flt), -+ DMA_BIDIRECTIONAL); -+ return -EIO; -+} -+EXPORT_SYMBOL(dpaa2_caam_enqueue); -+ -+const struct fsl_mc_device_id dpaa2_caam_match_id_table[] = { -+ { -+ .vendor = FSL_MC_VENDOR_FREESCALE, -+ .obj_type = "dpseci", -+ }, -+ { .vendor = 0x0 } -+}; -+ -+static struct fsl_mc_driver dpaa2_caam_driver = { -+ .driver = { -+ .name = KBUILD_MODNAME, -+ .owner = THIS_MODULE, -+ }, -+ .probe = dpaa2_caam_probe, -+ .remove = dpaa2_caam_remove, -+ .match_id_table = dpaa2_caam_match_id_table -+}; -+ -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_AUTHOR("Freescale Semiconductor, Inc"); -+MODULE_DESCRIPTION("Freescale DPAA2 CAAM Driver"); -+ -+module_fsl_mc_driver(dpaa2_caam_driver); ---- /dev/null -+++ b/drivers/crypto/caam/caamalg_qi2.h -@@ -0,0 +1,283 @@ -+/* -+ * Copyright 2015-2016 Freescale Semiconductor Inc. -+ * Copyright 2017 NXP -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the names of the above-listed copyright holders nor the -+ * names of any contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE -+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -+ * POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#ifndef _CAAMALG_QI2_H_ -+#define _CAAMALG_QI2_H_ -+ -+#include "../../../drivers/staging/fsl-mc/include/dpaa2-io.h" -+#include "../../../drivers/staging/fsl-mc/include/dpaa2-fd.h" -+#include <linux/threads.h> -+#include "dpseci.h" -+#include "desc_constr.h" -+ -+#define DPAA2_CAAM_STORE_SIZE 16 -+/* NAPI weight *must* be a multiple of the store size. */ -+#define DPAA2_CAAM_NAPI_WEIGHT 64 -+ -+/* The congestion entrance threshold was chosen so that on LS2088 -+ * we support the maximum throughput for the available memory -+ */ -+#define DPAA2_SEC_CONG_ENTRY_THRESH (128 * 1024 * 1024) -+#define DPAA2_SEC_CONG_EXIT_THRESH (DPAA2_SEC_CONG_ENTRY_THRESH * 9 / 10) -+ -+/** -+ * dpaa2_caam_priv - driver private data -+ * @dpseci_id: DPSECI object unique ID -+ * @major_ver: DPSECI major version -+ * @minor_ver: DPSECI minor version -+ * @dpseci_attr: DPSECI attributes -+ * @sec_attr: SEC engine attributes -+ * @rx_queue_attr: array of Rx queue attributes -+ * @tx_queue_attr: array of Tx queue attributes -+ * @cscn_mem: pointer to memory region containing the -+ * dpaa2_cscn struct; it's size is larger than -+ * sizeof(struct dpaa2_cscn) to accommodate alignment -+ * @cscn_mem_aligned: pointer to struct dpaa2_cscn; it is computed -+ * as PTR_ALIGN(cscn_mem, DPAA2_CSCN_ALIGN) -+ * @cscn_dma: dma address used by the QMAN to write CSCN messages -+ * @dev: device associated with the DPSECI object -+ * @mc_io: pointer to MC portal's I/O object -+ * @domain: IOMMU domain -+ * @ppriv: per CPU pointers to privata data -+ */ -+struct dpaa2_caam_priv { -+ int dpsec_id; -+ -+ u16 major_ver; -+ u16 minor_ver; -+ -+ struct dpseci_attr dpseci_attr; -+ struct dpseci_sec_attr sec_attr; -+ struct dpseci_rx_queue_attr rx_queue_attr[DPSECI_PRIO_NUM]; -+ struct dpseci_tx_queue_attr tx_queue_attr[DPSECI_PRIO_NUM]; -+ int num_pairs; -+ -+ /* congestion */ -+ void *cscn_mem; -+ void *cscn_mem_aligned; -+ dma_addr_t cscn_dma; -+ -+ struct device *dev; -+ struct fsl_mc_io *mc_io; -+ struct iommu_domain *domain; -+ -+ struct dpaa2_caam_priv_per_cpu __percpu *ppriv; -+}; -+ -+/** -+ * dpaa2_caam_priv_per_cpu - per CPU private data -+ * @napi: napi structure -+ * @net_dev: netdev used by napi -+ * @req_fqid: (virtual) request (Tx / enqueue) FQID -+ * @rsp_fqid: (virtual) response (Rx / dequeue) FQID -+ * @prio: internal queue number - index for dpaa2_caam_priv.*_queue_attr -+ * @nctx: notification context of response FQ -+ * @store: where dequeued frames are stored -+ * @priv: backpointer to dpaa2_caam_priv -+ */ -+struct dpaa2_caam_priv_per_cpu { -+ struct napi_struct napi; -+ struct net_device net_dev; -+ int req_fqid; -+ int rsp_fqid; -+ int prio; -+ struct dpaa2_io_notification_ctx nctx; -+ struct dpaa2_io_store *store; -+ struct dpaa2_caam_priv *priv; -+}; -+ -+/* -+ * The CAAM QI hardware constructs a job descriptor which points -+ * to shared descriptor (as pointed by context_a of FQ to CAAM). -+ * When the job descriptor is executed by deco, the whole job -+ * descriptor together with shared descriptor gets loaded in -+ * deco buffer which is 64 words long (each 32-bit). -+ * -+ * The job descriptor constructed by QI hardware has layout: -+ * -+ * HEADER (1 word) -+ * Shdesc ptr (1 or 2 words) -+ * SEQ_OUT_PTR (1 word) -+ * Out ptr (1 or 2 words) -+ * Out length (1 word) -+ * SEQ_IN_PTR (1 word) -+ * In ptr (1 or 2 words) -+ * In length (1 word) -+ * -+ * The shdesc ptr is used to fetch shared descriptor contents -+ * into deco buffer. -+ * -+ * Apart from shdesc contents, the total number of words that -+ * get loaded in deco buffer are '8' or '11'. The remaining words -+ * in deco buffer can be used for storing shared descriptor. -+ */ -+#define MAX_SDLEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) / CAAM_CMD_SZ) -+ -+/* Length of a single buffer in the QI driver memory cache */ -+#define CAAM_QI_MEMCACHE_SIZE 512 -+ -+/* -+ * aead_edesc - s/w-extended aead descriptor -+ * @src_nents: number of segments in input scatterlist -+ * @dst_nents: number of segments in output scatterlist -+ * @iv_dma: dma address of iv for checking continuity and link table -+ * @qm_sg_bytes: length of dma mapped h/w link table -+ * @qm_sg_dma: bus physical mapped address of h/w link table -+ * @assoclen: associated data length, in CAAM endianness -+ * @assoclen_dma: bus physical mapped address of req->assoclen -+ * @sgt: the h/w link table -+ */ -+struct aead_edesc { -+ int src_nents; -+ int dst_nents; -+ dma_addr_t iv_dma; -+ int qm_sg_bytes; -+ dma_addr_t qm_sg_dma; -+ unsigned int assoclen; -+ dma_addr_t assoclen_dma; -+#define CAAM_QI_MAX_AEAD_SG \ -+ ((CAAM_QI_MEMCACHE_SIZE - offsetof(struct aead_edesc, sgt)) / \ -+ sizeof(struct dpaa2_sg_entry)) -+ struct dpaa2_sg_entry sgt[0]; -+}; -+ -+/* -+ * tls_edesc - s/w-extended tls descriptor -+ * @src_nents: number of segments in input scatterlist -+ * @dst_nents: number of segments in output scatterlist -+ * @iv_dma: dma address of iv for checking continuity and link table -+ * @qm_sg_bytes: length of dma mapped h/w link table -+ * @qm_sg_dma: bus physical mapped address of h/w link table -+ * @tmp: array of scatterlists used by 'scatterwalk_ffwd' -+ * @dst: pointer to output scatterlist, usefull for unmapping -+ * @sgt: the h/w link table -+ */ -+struct tls_edesc { -+ int src_nents; -+ int dst_nents; -+ dma_addr_t iv_dma; -+ int qm_sg_bytes; -+ dma_addr_t qm_sg_dma; -+ struct scatterlist tmp[2]; -+ struct scatterlist *dst; -+ struct dpaa2_sg_entry sgt[0]; -+}; -+ -+/* -+ * ablkcipher_edesc - s/w-extended ablkcipher descriptor -+ * @src_nents: number of segments in input scatterlist -+ * @dst_nents: number of segments in output scatterlist -+ * @iv_dma: dma address of iv for checking continuity and link table -+ * @qm_sg_bytes: length of dma mapped qm_sg space -+ * @qm_sg_dma: I/O virtual address of h/w link table -+ * @sgt: the h/w link table -+ */ -+struct ablkcipher_edesc { -+ int src_nents; -+ int dst_nents; -+ dma_addr_t iv_dma; -+ int qm_sg_bytes; -+ dma_addr_t qm_sg_dma; -+#define CAAM_QI_MAX_ABLKCIPHER_SG \ -+ ((CAAM_QI_MEMCACHE_SIZE - offsetof(struct ablkcipher_edesc, sgt)) / \ -+ sizeof(struct dpaa2_sg_entry)) -+ struct dpaa2_sg_entry sgt[0]; -+}; -+ -+/* -+ * ahash_edesc - s/w-extended ahash descriptor -+ * @dst_dma: I/O virtual address of req->result -+ * @qm_sg_dma: I/O virtual address of h/w link table -+ * @src_nents: number of segments in input scatterlist -+ * @qm_sg_bytes: length of dma mapped qm_sg space -+ * @sgt: pointer to h/w link table -+ */ -+struct ahash_edesc { -+ dma_addr_t dst_dma; -+ dma_addr_t qm_sg_dma; -+ int src_nents; -+ int qm_sg_bytes; -+ struct dpaa2_sg_entry sgt[0]; -+}; -+ -+/** -+ * caam_flc - Flow Context (FLC) -+ * @flc: Flow Context options -+ * @sh_desc: Shared Descriptor -+ */ -+struct caam_flc { -+ u32 flc[16]; -+ u32 sh_desc[MAX_SDLEN]; -+} ____cacheline_aligned; -+ -+enum optype { -+ ENCRYPT = 0, -+ DECRYPT, -+ GIVENCRYPT, -+ NUM_OP -+}; -+ -+/** -+ * caam_request - the request structure the driver application should fill while -+ * submitting a job to driver. -+ * @fd_flt: Frame list table defining input and output -+ * fd_flt[0] - FLE pointing to output buffer -+ * fd_flt[1] - FLE pointing to input buffer -+ * @fd_flt_dma: DMA address for the frame list table -+ * @flc: Flow Context -+ * @flc_dma: I/O virtual address of Flow Context -+ * @op_type: operation type -+ * @cbk: Callback function to invoke when job is completed -+ * @ctx: arbit context attached with request by the application -+ * @edesc: extended descriptor; points to one of {ablkcipher,aead}_edesc -+ */ -+struct caam_request { -+ struct dpaa2_fl_entry fd_flt[2]; -+ dma_addr_t fd_flt_dma; -+ struct caam_flc *flc; -+ dma_addr_t flc_dma; -+ enum optype op_type; -+ void (*cbk)(void *ctx, u32 err); -+ void *ctx; -+ void *edesc; -+}; -+ -+/** -+ * dpaa2_caam_enqueue() - enqueue a crypto request -+ * @dev: device associated with the DPSECI object -+ * @req: pointer to caam_request -+ */ -+int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req); -+ -+#endif /* _CAAMALG_QI2_H_ */ ---- a/drivers/crypto/caam/caamhash.c -+++ b/drivers/crypto/caam/caamhash.c -@@ -62,6 +62,7 @@ - #include "error.h" - #include "sg_sw_sec4.h" - #include "key_gen.h" -+#include "caamhash_desc.h" - - #define CAAM_CRA_PRIORITY 3000 - -@@ -71,14 +72,6 @@ - #define CAAM_MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE - #define CAAM_MAX_HASH_DIGEST_SIZE SHA512_DIGEST_SIZE - --/* length of descriptors text */ --#define DESC_AHASH_BASE (4 * CAAM_CMD_SZ) --#define DESC_AHASH_UPDATE_LEN (6 * CAAM_CMD_SZ) --#define DESC_AHASH_UPDATE_FIRST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ) --#define DESC_AHASH_FINAL_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ) --#define DESC_AHASH_FINUP_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ) --#define DESC_AHASH_DIGEST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ) -- - #define DESC_HASH_MAX_USED_BYTES (DESC_AHASH_FINAL_LEN + \ - CAAM_MAX_HASH_KEY_SIZE) - #define DESC_HASH_MAX_USED_LEN (DESC_HASH_MAX_USED_BYTES / CAAM_CMD_SZ) -@@ -103,20 +96,15 @@ struct caam_hash_ctx { - u32 sh_desc_update_first[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; - u32 sh_desc_fin[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; - u32 sh_desc_digest[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; -- u32 sh_desc_finup[DESC_HASH_MAX_USED_LEN] ____cacheline_aligned; - dma_addr_t sh_desc_update_dma ____cacheline_aligned; - dma_addr_t sh_desc_update_first_dma; - dma_addr_t sh_desc_fin_dma; - dma_addr_t sh_desc_digest_dma; -- dma_addr_t sh_desc_finup_dma; -+ enum dma_data_direction dir; - struct device *jrdev; -- u32 alg_type; -- u32 alg_op; - u8 key[CAAM_MAX_HASH_KEY_SIZE]; -- dma_addr_t key_dma; - int ctx_len; -- unsigned int split_key_len; -- unsigned int split_key_pad_len; -+ struct alginfo adata; - }; - - /* ahash state */ -@@ -143,6 +131,31 @@ struct caam_export_state { - int (*finup)(struct ahash_request *req); - }; - -+static inline void switch_buf(struct caam_hash_state *state) -+{ -+ state->current_buf ^= 1; -+} -+ -+static inline u8 *current_buf(struct caam_hash_state *state) -+{ -+ return state->current_buf ? state->buf_1 : state->buf_0; -+} -+ -+static inline u8 *alt_buf(struct caam_hash_state *state) -+{ -+ return state->current_buf ? state->buf_0 : state->buf_1; -+} -+ -+static inline int *current_buflen(struct caam_hash_state *state) -+{ -+ return state->current_buf ? &state->buflen_1 : &state->buflen_0; -+} -+ -+static inline int *alt_buflen(struct caam_hash_state *state) -+{ -+ return state->current_buf ? &state->buflen_0 : &state->buflen_1; -+} -+ - /* Common job descriptor seq in/out ptr routines */ - - /* Map state->caam_ctx, and append seq_out_ptr command that points to it */ -@@ -175,40 +188,31 @@ static inline dma_addr_t map_seq_out_ptr - return dst_dma; - } - --/* Map current buffer in state and put it in link table */ --static inline dma_addr_t buf_map_to_sec4_sg(struct device *jrdev, -- struct sec4_sg_entry *sec4_sg, -- u8 *buf, int buflen) -+/* Map current buffer in state (if length > 0) and put it in link table */ -+static inline int buf_map_to_sec4_sg(struct device *jrdev, -+ struct sec4_sg_entry *sec4_sg, -+ struct caam_hash_state *state) - { -- dma_addr_t buf_dma; -+ int buflen = *current_buflen(state); - -- buf_dma = dma_map_single(jrdev, buf, buflen, DMA_TO_DEVICE); -- dma_to_sec4_sg_one(sec4_sg, buf_dma, buflen, 0); -+ if (!buflen) -+ return 0; - -- return buf_dma; --} -+ state->buf_dma = dma_map_single(jrdev, current_buf(state), buflen, -+ DMA_TO_DEVICE); -+ if (dma_mapping_error(jrdev, state->buf_dma)) { -+ dev_err(jrdev, "unable to map buf\n"); -+ state->buf_dma = 0; -+ return -ENOMEM; -+ } - --/* -- * Only put buffer in link table if it contains data, which is possible, -- * since a buffer has previously been used, and needs to be unmapped, -- */ --static inline dma_addr_t --try_buf_map_to_sec4_sg(struct device *jrdev, struct sec4_sg_entry *sec4_sg, -- u8 *buf, dma_addr_t buf_dma, int buflen, -- int last_buflen) --{ -- if (buf_dma && !dma_mapping_error(jrdev, buf_dma)) -- dma_unmap_single(jrdev, buf_dma, last_buflen, DMA_TO_DEVICE); -- if (buflen) -- buf_dma = buf_map_to_sec4_sg(jrdev, sec4_sg, buf, buflen); -- else -- buf_dma = 0; -+ dma_to_sec4_sg_one(sec4_sg, state->buf_dma, buflen, 0); - -- return buf_dma; -+ return 0; - } - - /* Map state->caam_ctx, and add it to link table */ --static inline int ctx_map_to_sec4_sg(u32 *desc, struct device *jrdev, -+static inline int ctx_map_to_sec4_sg(struct device *jrdev, - struct caam_hash_state *state, int ctx_len, - struct sec4_sg_entry *sec4_sg, u32 flag) - { -@@ -224,124 +228,22 @@ static inline int ctx_map_to_sec4_sg(u32 - return 0; - } - --/* Common shared descriptor commands */ --static inline void append_key_ahash(u32 *desc, struct caam_hash_ctx *ctx) --{ -- append_key_as_imm(desc, ctx->key, ctx->split_key_pad_len, -- ctx->split_key_len, CLASS_2 | -- KEY_DEST_MDHA_SPLIT | KEY_ENC); --} -- --/* Append key if it has been set */ --static inline void init_sh_desc_key_ahash(u32 *desc, struct caam_hash_ctx *ctx) --{ -- u32 *key_jump_cmd; -- -- init_sh_desc(desc, HDR_SHARE_SERIAL); -- -- if (ctx->split_key_len) { -- /* Skip if already shared */ -- key_jump_cmd = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -- JUMP_COND_SHRD); -- -- append_key_ahash(desc, ctx); -- -- set_jump_tgt_here(desc, key_jump_cmd); -- } -- -- /* Propagate errors from shared to job descriptor */ -- append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); --} -- --/* -- * For ahash read data from seqin following state->caam_ctx, -- * and write resulting class2 context to seqout, which may be state->caam_ctx -- * or req->result -- */ --static inline void ahash_append_load_str(u32 *desc, int digestsize) --{ -- /* Calculate remaining bytes to read */ -- append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -- -- /* Read remaining bytes */ -- append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 | -- FIFOLD_TYPE_MSG | KEY_VLF); -- -- /* Store class2 context bytes */ -- append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | -- LDST_SRCDST_BYTE_CONTEXT); --} -- --/* -- * For ahash update, final and finup, import context, read and write to seqout -- */ --static inline void ahash_ctx_data_to_out(u32 *desc, u32 op, u32 state, -- int digestsize, -- struct caam_hash_ctx *ctx) --{ -- init_sh_desc_key_ahash(desc, ctx); -- -- /* Import context from software */ -- append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | -- LDST_CLASS_2_CCB | ctx->ctx_len); -- -- /* Class 2 operation */ -- append_operation(desc, op | state | OP_ALG_ENCRYPT); -- -- /* -- * Load from buf and/or src and write to req->result or state->context -- */ -- ahash_append_load_str(desc, digestsize); --} -- --/* For ahash firsts and digest, read and write to seqout */ --static inline void ahash_data_to_out(u32 *desc, u32 op, u32 state, -- int digestsize, struct caam_hash_ctx *ctx) --{ -- init_sh_desc_key_ahash(desc, ctx); -- -- /* Class 2 operation */ -- append_operation(desc, op | state | OP_ALG_ENCRYPT); -- -- /* -- * Load from buf and/or src and write to req->result or state->context -- */ -- ahash_append_load_str(desc, digestsize); --} -- - static int ahash_set_sh_desc(struct crypto_ahash *ahash) - { - struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); - int digestsize = crypto_ahash_digestsize(ahash); - struct device *jrdev = ctx->jrdev; -- u32 have_key = 0; -+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); - u32 *desc; - -- if (ctx->split_key_len) -- have_key = OP_ALG_AAI_HMAC_PRECOMP; -+ ctx->adata.key_virt = ctx->key; - - /* ahash_update shared descriptor */ - desc = ctx->sh_desc_update; -- -- init_sh_desc(desc, HDR_SHARE_SERIAL); -- -- /* Import context from software */ -- append_cmd(desc, CMD_SEQ_LOAD | LDST_SRCDST_BYTE_CONTEXT | -- LDST_CLASS_2_CCB | ctx->ctx_len); -- -- /* Class 2 operation */ -- append_operation(desc, ctx->alg_type | OP_ALG_AS_UPDATE | -- OP_ALG_ENCRYPT); -- -- /* Load data and write to result or context */ -- ahash_append_load_str(desc, ctx->ctx_len); -- -- ctx->sh_desc_update_dma = dma_map_single(jrdev, desc, desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_update_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } -+ cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_UPDATE, ctx->ctx_len, -+ ctx->ctx_len, true, ctrlpriv->era); -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma, -+ desc_bytes(desc), ctx->dir); - #ifdef DEBUG - print_hex_dump(KERN_ERR, - "ahash update shdesc@"__stringify(__LINE__)": ", -@@ -350,17 +252,10 @@ static int ahash_set_sh_desc(struct cryp - - /* ahash_update_first shared descriptor */ - desc = ctx->sh_desc_update_first; -- -- ahash_data_to_out(desc, have_key | ctx->alg_type, OP_ALG_AS_INIT, -- ctx->ctx_len, ctx); -- -- ctx->sh_desc_update_first_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_update_first_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } -+ cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INIT, ctx->ctx_len, -+ ctx->ctx_len, false, ctrlpriv->era); -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma, -+ desc_bytes(desc), ctx->dir); - #ifdef DEBUG - print_hex_dump(KERN_ERR, - "ahash update first shdesc@"__stringify(__LINE__)": ", -@@ -369,53 +264,22 @@ static int ahash_set_sh_desc(struct cryp - - /* ahash_final shared descriptor */ - desc = ctx->sh_desc_fin; -- -- ahash_ctx_data_to_out(desc, have_key | ctx->alg_type, -- OP_ALG_AS_FINALIZE, digestsize, ctx); -- -- ctx->sh_desc_fin_dma = dma_map_single(jrdev, desc, desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_fin_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } -+ cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_FINALIZE, digestsize, -+ ctx->ctx_len, true, ctrlpriv->era); -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma, -+ desc_bytes(desc), ctx->dir); - #ifdef DEBUG - print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); - #endif - -- /* ahash_finup shared descriptor */ -- desc = ctx->sh_desc_finup; -- -- ahash_ctx_data_to_out(desc, have_key | ctx->alg_type, -- OP_ALG_AS_FINALIZE, digestsize, ctx); -- -- ctx->sh_desc_finup_dma = dma_map_single(jrdev, desc, desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_finup_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, "ahash finup shdesc@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, desc, -- desc_bytes(desc), 1); --#endif -- - /* ahash_digest shared descriptor */ - desc = ctx->sh_desc_digest; -- -- ahash_data_to_out(desc, have_key | ctx->alg_type, OP_ALG_AS_INITFINAL, -- digestsize, ctx); -- -- ctx->sh_desc_digest_dma = dma_map_single(jrdev, desc, -- desc_bytes(desc), -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->sh_desc_digest_dma)) { -- dev_err(jrdev, "unable to map shared descriptor\n"); -- return -ENOMEM; -- } -+ cnstr_shdsc_ahash(desc, &ctx->adata, OP_ALG_AS_INITFINAL, digestsize, -+ ctx->ctx_len, false, ctrlpriv->era); -+ dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma, -+ desc_bytes(desc), ctx->dir); - #ifdef DEBUG - print_hex_dump(KERN_ERR, - "ahash digest shdesc@"__stringify(__LINE__)": ", -@@ -426,14 +290,6 @@ static int ahash_set_sh_desc(struct cryp - return 0; - } - --static int gen_split_hash_key(struct caam_hash_ctx *ctx, const u8 *key_in, -- u32 keylen) --{ -- return gen_split_key(ctx->jrdev, ctx->key, ctx->split_key_len, -- ctx->split_key_pad_len, key_in, keylen, -- ctx->alg_op); --} -- - /* Digest hash size if it is too large */ - static int hash_digest_key(struct caam_hash_ctx *ctx, const u8 *key_in, - u32 *keylen, u8 *key_out, u32 digestsize) -@@ -469,7 +325,7 @@ static int hash_digest_key(struct caam_h - } - - /* Job descriptor to perform unkeyed hash on key_in */ -- append_operation(desc, ctx->alg_type | OP_ALG_ENCRYPT | -+ append_operation(desc, ctx->adata.algtype | OP_ALG_ENCRYPT | - OP_ALG_AS_INITFINAL); - append_seq_in_ptr(desc, src_dma, *keylen, 0); - append_seq_fifo_load(desc, *keylen, FIFOLD_CLASS_CLASS2 | -@@ -513,12 +369,10 @@ static int hash_digest_key(struct caam_h - static int ahash_setkey(struct crypto_ahash *ahash, - const u8 *key, unsigned int keylen) - { -- /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ -- static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; - struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); -- struct device *jrdev = ctx->jrdev; - int blocksize = crypto_tfm_alg_blocksize(&ahash->base); - int digestsize = crypto_ahash_digestsize(ahash); -+ struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); - int ret; - u8 *hashed_key = NULL; - -@@ -539,43 +393,29 @@ static int ahash_setkey(struct crypto_ah - key = hashed_key; - } - -- /* Pick class 2 key length from algorithm submask */ -- ctx->split_key_len = mdpadlen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >> -- OP_ALG_ALGSEL_SHIFT] * 2; -- ctx->split_key_pad_len = ALIGN(ctx->split_key_len, 16); -- --#ifdef DEBUG -- printk(KERN_ERR "split_key_len %d split_key_pad_len %d\n", -- ctx->split_key_len, ctx->split_key_pad_len); -- print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); --#endif -+ /* -+ * If DKP is supported, use it in the shared descriptor to generate -+ * the split key. -+ */ -+ if (ctrlpriv->era >= 6) { -+ ctx->adata.key_inline = true; -+ ctx->adata.keylen = keylen; -+ ctx->adata.keylen_pad = split_key_len(ctx->adata.algtype & -+ OP_ALG_ALGSEL_MASK); - -- ret = gen_split_hash_key(ctx, key, keylen); -- if (ret) -- goto bad_free_key; -+ if (ctx->adata.keylen_pad > CAAM_MAX_HASH_KEY_SIZE) -+ goto bad_free_key; - -- ctx->key_dma = dma_map_single(jrdev, ctx->key, ctx->split_key_pad_len, -- DMA_TO_DEVICE); -- if (dma_mapping_error(jrdev, ctx->key_dma)) { -- dev_err(jrdev, "unable to map key i/o memory\n"); -- ret = -ENOMEM; -- goto error_free_key; -+ memcpy(ctx->key, key, keylen); -+ } else { -+ ret = gen_split_key(ctx->jrdev, ctx->key, &ctx->adata, key, -+ keylen, CAAM_MAX_HASH_KEY_SIZE); -+ if (ret) -+ goto bad_free_key; - } --#ifdef DEBUG -- print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", -- DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, -- ctx->split_key_pad_len, 1); --#endif - -- ret = ahash_set_sh_desc(ahash); -- if (ret) { -- dma_unmap_single(jrdev, ctx->key_dma, ctx->split_key_pad_len, -- DMA_TO_DEVICE); -- } -- error_free_key: - kfree(hashed_key); -- return ret; -+ return ahash_set_sh_desc(ahash); - bad_free_key: - kfree(hashed_key); - crypto_ahash_set_flags(ahash, CRYPTO_TFM_RES_BAD_KEY_LEN); -@@ -604,6 +444,8 @@ static inline void ahash_unmap(struct de - struct ahash_edesc *edesc, - struct ahash_request *req, int dst_len) - { -+ struct caam_hash_state *state = ahash_request_ctx(req); -+ - if (edesc->src_nents) - dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); - if (edesc->dst_dma) -@@ -612,6 +454,12 @@ static inline void ahash_unmap(struct de - if (edesc->sec4_sg_bytes) - dma_unmap_single(dev, edesc->sec4_sg_dma, - edesc->sec4_sg_bytes, DMA_TO_DEVICE); -+ -+ if (state->buf_dma) { -+ dma_unmap_single(dev, state->buf_dma, *current_buflen(state), -+ DMA_TO_DEVICE); -+ state->buf_dma = 0; -+ } - } - - static inline void ahash_unmap_ctx(struct device *dev, -@@ -643,8 +491,7 @@ static void ahash_done(struct device *jr - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); - #endif - -- edesc = (struct ahash_edesc *)((char *)desc - -- offsetof(struct ahash_edesc, hw_desc)); -+ edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); - if (err) - caam_jr_strstatus(jrdev, err); - -@@ -671,19 +518,19 @@ static void ahash_done_bi(struct device - struct ahash_edesc *edesc; - struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); - struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); --#ifdef DEBUG - struct caam_hash_state *state = ahash_request_ctx(req); -+#ifdef DEBUG - int digestsize = crypto_ahash_digestsize(ahash); - - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); - #endif - -- edesc = (struct ahash_edesc *)((char *)desc - -- offsetof(struct ahash_edesc, hw_desc)); -+ edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); - if (err) - caam_jr_strstatus(jrdev, err); - - ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_BIDIRECTIONAL); -+ switch_buf(state); - kfree(edesc); - - #ifdef DEBUG -@@ -713,8 +560,7 @@ static void ahash_done_ctx_src(struct de - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); - #endif - -- edesc = (struct ahash_edesc *)((char *)desc - -- offsetof(struct ahash_edesc, hw_desc)); -+ edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); - if (err) - caam_jr_strstatus(jrdev, err); - -@@ -741,19 +587,19 @@ static void ahash_done_ctx_dst(struct de - struct ahash_edesc *edesc; - struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); - struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); --#ifdef DEBUG - struct caam_hash_state *state = ahash_request_ctx(req); -+#ifdef DEBUG - int digestsize = crypto_ahash_digestsize(ahash); - - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); - #endif - -- edesc = (struct ahash_edesc *)((char *)desc - -- offsetof(struct ahash_edesc, hw_desc)); -+ edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); - if (err) - caam_jr_strstatus(jrdev, err); - - ahash_unmap_ctx(jrdev, edesc, req, ctx->ctx_len, DMA_FROM_DEVICE); -+ switch_buf(state); - kfree(edesc); - - #ifdef DEBUG -@@ -835,13 +681,12 @@ static int ahash_update_ctx(struct ahash - struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); - struct caam_hash_state *state = ahash_request_ctx(req); - struct device *jrdev = ctx->jrdev; -- gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | -- CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; -- u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; -- int *buflen = state->current_buf ? &state->buflen_1 : &state->buflen_0; -- u8 *next_buf = state->current_buf ? state->buf_0 : state->buf_1; -- int *next_buflen = state->current_buf ? &state->buflen_0 : -- &state->buflen_1, last_buflen; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; -+ u8 *buf = current_buf(state); -+ int *buflen = current_buflen(state); -+ u8 *next_buf = alt_buf(state); -+ int *next_buflen = alt_buflen(state), last_buflen; - int in_len = *buflen + req->nbytes, to_hash; - u32 *desc; - int src_nents, mapped_nents, sec4_sg_bytes, sec4_sg_src_index; -@@ -890,15 +735,14 @@ static int ahash_update_ctx(struct ahash - edesc->src_nents = src_nents; - edesc->sec4_sg_bytes = sec4_sg_bytes; - -- ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, -+ ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len, - edesc->sec4_sg, DMA_BIDIRECTIONAL); - if (ret) - goto unmap_ctx; - -- state->buf_dma = try_buf_map_to_sec4_sg(jrdev, -- edesc->sec4_sg + 1, -- buf, state->buf_dma, -- *buflen, last_buflen); -+ ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state); -+ if (ret) -+ goto unmap_ctx; - - if (mapped_nents) { - sg_to_sec4_sg_last(req->src, mapped_nents, -@@ -909,12 +753,10 @@ static int ahash_update_ctx(struct ahash - to_hash - *buflen, - *next_buflen, 0); - } else { -- (edesc->sec4_sg + sec4_sg_src_index - 1)->len |= -- cpu_to_caam32(SEC4_SG_LEN_FIN); -+ sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - -+ 1); - } - -- state->current_buf = !state->current_buf; -- - desc = edesc->hw_desc; - - edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, -@@ -969,12 +811,9 @@ static int ahash_final_ctx(struct ahash_ - struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); - struct caam_hash_state *state = ahash_request_ctx(req); - struct device *jrdev = ctx->jrdev; -- gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | -- CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; -- u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; -- int buflen = state->current_buf ? state->buflen_1 : state->buflen_0; -- int last_buflen = state->current_buf ? state->buflen_0 : -- state->buflen_1; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; -+ int buflen = *current_buflen(state); - u32 *desc; - int sec4_sg_bytes, sec4_sg_src_index; - int digestsize = crypto_ahash_digestsize(ahash); -@@ -994,18 +833,17 @@ static int ahash_final_ctx(struct ahash_ - desc = edesc->hw_desc; - - edesc->sec4_sg_bytes = sec4_sg_bytes; -- edesc->src_nents = 0; - -- ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, -+ ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len, - edesc->sec4_sg, DMA_TO_DEVICE); - if (ret) - goto unmap_ctx; - -- state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, -- buf, state->buf_dma, buflen, -- last_buflen); -- (edesc->sec4_sg + sec4_sg_src_index - 1)->len |= -- cpu_to_caam32(SEC4_SG_LEN_FIN); -+ ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state); -+ if (ret) -+ goto unmap_ctx; -+ -+ sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - 1); - - edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, - sec4_sg_bytes, DMA_TO_DEVICE); -@@ -1048,12 +886,9 @@ static int ahash_finup_ctx(struct ahash_ - struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); - struct caam_hash_state *state = ahash_request_ctx(req); - struct device *jrdev = ctx->jrdev; -- gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | -- CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; -- u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; -- int buflen = state->current_buf ? state->buflen_1 : state->buflen_0; -- int last_buflen = state->current_buf ? state->buflen_0 : -- state->buflen_1; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; -+ int buflen = *current_buflen(state); - u32 *desc; - int sec4_sg_src_index; - int src_nents, mapped_nents; -@@ -1082,7 +917,7 @@ static int ahash_finup_ctx(struct ahash_ - - /* allocate space for base edesc and hw desc commands, link tables */ - edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents, -- ctx->sh_desc_finup, ctx->sh_desc_finup_dma, -+ ctx->sh_desc_fin, ctx->sh_desc_fin_dma, - flags); - if (!edesc) { - dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); -@@ -1093,14 +928,14 @@ static int ahash_finup_ctx(struct ahash_ - - edesc->src_nents = src_nents; - -- ret = ctx_map_to_sec4_sg(desc, jrdev, state, ctx->ctx_len, -+ ret = ctx_map_to_sec4_sg(jrdev, state, ctx->ctx_len, - edesc->sec4_sg, DMA_TO_DEVICE); - if (ret) - goto unmap_ctx; - -- state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, -- buf, state->buf_dma, buflen, -- last_buflen); -+ ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg + 1, state); -+ if (ret) -+ goto unmap_ctx; - - ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, - sec4_sg_src_index, ctx->ctx_len + buflen, -@@ -1136,15 +971,18 @@ static int ahash_digest(struct ahash_req - { - struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); - struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); -+ struct caam_hash_state *state = ahash_request_ctx(req); - struct device *jrdev = ctx->jrdev; -- gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | -- CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; - u32 *desc; - int digestsize = crypto_ahash_digestsize(ahash); - int src_nents, mapped_nents; - struct ahash_edesc *edesc; - int ret; - -+ state->buf_dma = 0; -+ - src_nents = sg_nents_for_len(req->src, req->nbytes); - if (src_nents < 0) { - dev_err(jrdev, "Invalid number of src SG.\n"); -@@ -1215,10 +1053,10 @@ static int ahash_final_no_ctx(struct aha - struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); - struct caam_hash_state *state = ahash_request_ctx(req); - struct device *jrdev = ctx->jrdev; -- gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | -- CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; -- u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; -- int buflen = state->current_buf ? state->buflen_1 : state->buflen_0; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; -+ u8 *buf = current_buf(state); -+ int buflen = *current_buflen(state); - u32 *desc; - int digestsize = crypto_ahash_digestsize(ahash); - struct ahash_edesc *edesc; -@@ -1246,7 +1084,6 @@ static int ahash_final_no_ctx(struct aha - dev_err(jrdev, "unable to map dst\n"); - goto unmap; - } -- edesc->src_nents = 0; - - #ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", -@@ -1276,13 +1113,12 @@ static int ahash_update_no_ctx(struct ah - struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); - struct caam_hash_state *state = ahash_request_ctx(req); - struct device *jrdev = ctx->jrdev; -- gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | -- CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; -- u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; -- int *buflen = state->current_buf ? &state->buflen_1 : &state->buflen_0; -- u8 *next_buf = state->current_buf ? state->buf_0 : state->buf_1; -- int *next_buflen = state->current_buf ? &state->buflen_0 : -- &state->buflen_1; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; -+ u8 *buf = current_buf(state); -+ int *buflen = current_buflen(state); -+ u8 *next_buf = alt_buf(state); -+ int *next_buflen = alt_buflen(state); - int in_len = *buflen + req->nbytes, to_hash; - int sec4_sg_bytes, src_nents, mapped_nents; - struct ahash_edesc *edesc; -@@ -1329,10 +1165,11 @@ static int ahash_update_no_ctx(struct ah - - edesc->src_nents = src_nents; - edesc->sec4_sg_bytes = sec4_sg_bytes; -- edesc->dst_dma = 0; - -- state->buf_dma = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, -- buf, *buflen); -+ ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state); -+ if (ret) -+ goto unmap_ctx; -+ - sg_to_sec4_sg_last(req->src, mapped_nents, - edesc->sec4_sg + 1, 0); - -@@ -1342,8 +1179,6 @@ static int ahash_update_no_ctx(struct ah - *next_buflen, 0); - } - -- state->current_buf = !state->current_buf; -- - desc = edesc->hw_desc; - - edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, -@@ -1403,12 +1238,9 @@ static int ahash_finup_no_ctx(struct aha - struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); - struct caam_hash_state *state = ahash_request_ctx(req); - struct device *jrdev = ctx->jrdev; -- gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | -- CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; -- u8 *buf = state->current_buf ? state->buf_1 : state->buf_0; -- int buflen = state->current_buf ? state->buflen_1 : state->buflen_0; -- int last_buflen = state->current_buf ? state->buflen_0 : -- state->buflen_1; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; -+ int buflen = *current_buflen(state); - u32 *desc; - int sec4_sg_bytes, sec4_sg_src_index, src_nents, mapped_nents; - int digestsize = crypto_ahash_digestsize(ahash); -@@ -1450,9 +1282,9 @@ static int ahash_finup_no_ctx(struct aha - edesc->src_nents = src_nents; - edesc->sec4_sg_bytes = sec4_sg_bytes; - -- state->buf_dma = try_buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, buf, -- state->buf_dma, buflen, -- last_buflen); -+ ret = buf_map_to_sec4_sg(jrdev, edesc->sec4_sg, state); -+ if (ret) -+ goto unmap; - - ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 1, buflen, - req->nbytes); -@@ -1496,11 +1328,10 @@ static int ahash_update_first(struct aha - struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); - struct caam_hash_state *state = ahash_request_ctx(req); - struct device *jrdev = ctx->jrdev; -- gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | -- CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; -- u8 *next_buf = state->current_buf ? state->buf_1 : state->buf_0; -- int *next_buflen = state->current_buf ? -- &state->buflen_1 : &state->buflen_0; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; -+ u8 *next_buf = alt_buf(state); -+ int *next_buflen = alt_buflen(state); - int to_hash; - u32 *desc; - int src_nents, mapped_nents; -@@ -1545,7 +1376,6 @@ static int ahash_update_first(struct aha - } - - edesc->src_nents = src_nents; -- edesc->dst_dma = 0; - - ret = ahash_edesc_add_src(ctx, edesc, req, mapped_nents, 0, 0, - to_hash); -@@ -1582,6 +1412,7 @@ static int ahash_update_first(struct aha - state->final = ahash_final_no_ctx; - scatterwalk_map_and_copy(next_buf, req->src, 0, - req->nbytes, 0); -+ switch_buf(state); - } - #ifdef DEBUG - print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", -@@ -1688,7 +1519,6 @@ struct caam_hash_template { - unsigned int blocksize; - struct ahash_alg template_ahash; - u32 alg_type; -- u32 alg_op; - }; - - /* ahash descriptors */ -@@ -1714,7 +1544,6 @@ static struct caam_hash_template driver_ - }, - }, - .alg_type = OP_ALG_ALGSEL_SHA1, -- .alg_op = OP_ALG_ALGSEL_SHA1 | OP_ALG_AAI_HMAC, - }, { - .name = "sha224", - .driver_name = "sha224-caam", -@@ -1736,7 +1565,6 @@ static struct caam_hash_template driver_ - }, - }, - .alg_type = OP_ALG_ALGSEL_SHA224, -- .alg_op = OP_ALG_ALGSEL_SHA224 | OP_ALG_AAI_HMAC, - }, { - .name = "sha256", - .driver_name = "sha256-caam", -@@ -1758,7 +1586,6 @@ static struct caam_hash_template driver_ - }, - }, - .alg_type = OP_ALG_ALGSEL_SHA256, -- .alg_op = OP_ALG_ALGSEL_SHA256 | OP_ALG_AAI_HMAC, - }, { - .name = "sha384", - .driver_name = "sha384-caam", -@@ -1780,7 +1607,6 @@ static struct caam_hash_template driver_ - }, - }, - .alg_type = OP_ALG_ALGSEL_SHA384, -- .alg_op = OP_ALG_ALGSEL_SHA384 | OP_ALG_AAI_HMAC, - }, { - .name = "sha512", - .driver_name = "sha512-caam", -@@ -1802,7 +1628,6 @@ static struct caam_hash_template driver_ - }, - }, - .alg_type = OP_ALG_ALGSEL_SHA512, -- .alg_op = OP_ALG_ALGSEL_SHA512 | OP_ALG_AAI_HMAC, - }, { - .name = "md5", - .driver_name = "md5-caam", -@@ -1824,14 +1649,12 @@ static struct caam_hash_template driver_ - }, - }, - .alg_type = OP_ALG_ALGSEL_MD5, -- .alg_op = OP_ALG_ALGSEL_MD5 | OP_ALG_AAI_HMAC, - }, - }; - - struct caam_hash_alg { - struct list_head entry; - int alg_type; -- int alg_op; - struct ahash_alg ahash_alg; - }; - -@@ -1853,6 +1676,8 @@ static int caam_hash_cra_init(struct cry - HASH_MSG_LEN + SHA256_DIGEST_SIZE, - HASH_MSG_LEN + 64, - HASH_MSG_LEN + SHA512_DIGEST_SIZE }; -+ dma_addr_t dma_addr; -+ struct caam_drv_private *priv; - - /* - * Get a Job ring from Job Ring driver to ensure in-order -@@ -1863,11 +1688,34 @@ static int caam_hash_cra_init(struct cry - pr_err("Job Ring Device allocation for transform failed\n"); - return PTR_ERR(ctx->jrdev); - } -+ -+ priv = dev_get_drvdata(ctx->jrdev->parent); -+ ctx->dir = priv->era >= 6 ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; -+ -+ dma_addr = dma_map_single_attrs(ctx->jrdev, ctx->sh_desc_update, -+ offsetof(struct caam_hash_ctx, -+ sh_desc_update_dma), -+ ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); -+ if (dma_mapping_error(ctx->jrdev, dma_addr)) { -+ dev_err(ctx->jrdev, "unable to map shared descriptors\n"); -+ caam_jr_free(ctx->jrdev); -+ return -ENOMEM; -+ } -+ -+ ctx->sh_desc_update_dma = dma_addr; -+ ctx->sh_desc_update_first_dma = dma_addr + -+ offsetof(struct caam_hash_ctx, -+ sh_desc_update_first); -+ ctx->sh_desc_fin_dma = dma_addr + offsetof(struct caam_hash_ctx, -+ sh_desc_fin); -+ ctx->sh_desc_digest_dma = dma_addr + offsetof(struct caam_hash_ctx, -+ sh_desc_digest); -+ - /* copy descriptor header template value */ -- ctx->alg_type = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; -- ctx->alg_op = OP_TYPE_CLASS2_ALG | caam_hash->alg_op; -+ ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam_hash->alg_type; - -- ctx->ctx_len = runninglen[(ctx->alg_op & OP_ALG_ALGSEL_SUBMASK) >> -+ ctx->ctx_len = runninglen[(ctx->adata.algtype & -+ OP_ALG_ALGSEL_SUBMASK) >> - OP_ALG_ALGSEL_SHIFT]; - - crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), -@@ -1879,30 +1727,10 @@ static void caam_hash_cra_exit(struct cr - { - struct caam_hash_ctx *ctx = crypto_tfm_ctx(tfm); - -- if (ctx->sh_desc_update_dma && -- !dma_mapping_error(ctx->jrdev, ctx->sh_desc_update_dma)) -- dma_unmap_single(ctx->jrdev, ctx->sh_desc_update_dma, -- desc_bytes(ctx->sh_desc_update), -- DMA_TO_DEVICE); -- if (ctx->sh_desc_update_first_dma && -- !dma_mapping_error(ctx->jrdev, ctx->sh_desc_update_first_dma)) -- dma_unmap_single(ctx->jrdev, ctx->sh_desc_update_first_dma, -- desc_bytes(ctx->sh_desc_update_first), -- DMA_TO_DEVICE); -- if (ctx->sh_desc_fin_dma && -- !dma_mapping_error(ctx->jrdev, ctx->sh_desc_fin_dma)) -- dma_unmap_single(ctx->jrdev, ctx->sh_desc_fin_dma, -- desc_bytes(ctx->sh_desc_fin), DMA_TO_DEVICE); -- if (ctx->sh_desc_digest_dma && -- !dma_mapping_error(ctx->jrdev, ctx->sh_desc_digest_dma)) -- dma_unmap_single(ctx->jrdev, ctx->sh_desc_digest_dma, -- desc_bytes(ctx->sh_desc_digest), -- DMA_TO_DEVICE); -- if (ctx->sh_desc_finup_dma && -- !dma_mapping_error(ctx->jrdev, ctx->sh_desc_finup_dma)) -- dma_unmap_single(ctx->jrdev, ctx->sh_desc_finup_dma, -- desc_bytes(ctx->sh_desc_finup), DMA_TO_DEVICE); -- -+ dma_unmap_single_attrs(ctx->jrdev, ctx->sh_desc_update_dma, -+ offsetof(struct caam_hash_ctx, -+ sh_desc_update_dma), -+ ctx->dir, DMA_ATTR_SKIP_CPU_SYNC); - caam_jr_free(ctx->jrdev); - } - -@@ -1961,7 +1789,6 @@ caam_hash_alloc(struct caam_hash_templat - alg->cra_type = &crypto_ahash_type; - - t_alg->alg_type = template->alg_type; -- t_alg->alg_op = template->alg_op; - - return t_alg; - } ---- /dev/null -+++ b/drivers/crypto/caam/caamhash_desc.c -@@ -0,0 +1,108 @@ -+/* -+ * Shared descriptors for ahash algorithms -+ * -+ * Copyright 2017 NXP -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the names of the above-listed copyright holders nor the -+ * names of any contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE -+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -+ * POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include "compat.h" -+#include "desc_constr.h" -+#include "caamhash_desc.h" -+ -+/** -+ * cnstr_shdsc_ahash - ahash shared descriptor -+ * @desc: pointer to buffer used for descriptor construction -+ * @adata: pointer to authentication transform definitions. -+ * A split key is required for SEC Era < 6; the size of the split key -+ * is specified in this case. -+ * Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, SHA224, -+ * SHA256, SHA384, SHA512}. -+ * @state: algorithm state OP_ALG_AS_{INIT, FINALIZE, INITFINALIZE, UPDATE} -+ * @digestsize: algorithm's digest size -+ * @ctx_len: size of Context Register -+ * @import_ctx: true if previous Context Register needs to be restored -+ * must be true for ahash update and final -+ * must be false for for ahash first and digest -+ * @era: SEC Era -+ */ -+void cnstr_shdsc_ahash(u32 * const desc, struct alginfo *adata, u32 state, -+ int digestsize, int ctx_len, bool import_ctx, int era) -+{ -+ u32 op = adata->algtype; -+ -+ init_sh_desc(desc, HDR_SHARE_SERIAL); -+ -+ /* Append key if it has been set; ahash update excluded */ -+ if (state != OP_ALG_AS_UPDATE && adata->keylen) { -+ u32 *skip_key_load; -+ -+ /* Skip key loading if already shared */ -+ skip_key_load = append_jump(desc, JUMP_JSL | JUMP_TEST_ALL | -+ JUMP_COND_SHRD); -+ -+ if (era < 6) -+ append_key_as_imm(desc, adata->key_virt, -+ adata->keylen_pad, -+ adata->keylen, CLASS_2 | -+ KEY_DEST_MDHA_SPLIT | KEY_ENC); -+ else -+ append_proto_dkp(desc, adata); -+ -+ set_jump_tgt_here(desc, skip_key_load); -+ -+ op |= OP_ALG_AAI_HMAC_PRECOMP; -+ } -+ -+ /* If needed, import context from software */ -+ if (import_ctx) -+ append_seq_load(desc, ctx_len, LDST_CLASS_2_CCB | -+ LDST_SRCDST_BYTE_CONTEXT); -+ -+ /* Class 2 operation */ -+ append_operation(desc, op | state | OP_ALG_ENCRYPT); -+ -+ /* -+ * Load from buf and/or src and write to req->result or state->context -+ * Calculate remaining bytes to read -+ */ -+ append_math_add(desc, VARSEQINLEN, SEQINLEN, REG0, CAAM_CMD_SZ); -+ /* Read remaining bytes */ -+ append_seq_fifo_load(desc, 0, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 | -+ FIFOLD_TYPE_MSG | KEY_VLF); -+ /* Store class2 context bytes */ -+ append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | -+ LDST_SRCDST_BYTE_CONTEXT); -+} -+EXPORT_SYMBOL(cnstr_shdsc_ahash); -+ -+MODULE_LICENSE("Dual BSD/GPL"); -+MODULE_DESCRIPTION("FSL CAAM ahash descriptors support"); -+MODULE_AUTHOR("NXP Semiconductors"); ---- /dev/null -+++ b/drivers/crypto/caam/caamhash_desc.h -@@ -0,0 +1,49 @@ -+/* -+ * Shared descriptors for ahash algorithms -+ * -+ * Copyright 2017 NXP -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the names of the above-listed copyright holders nor the -+ * names of any contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE -+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -+ * POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#ifndef _CAAMHASH_DESC_H_ -+#define _CAAMHASH_DESC_H_ -+ -+/* length of descriptors text */ -+#define DESC_AHASH_BASE (3 * CAAM_CMD_SZ) -+#define DESC_AHASH_UPDATE_LEN (6 * CAAM_CMD_SZ) -+#define DESC_AHASH_UPDATE_FIRST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ) -+#define DESC_AHASH_FINAL_LEN (DESC_AHASH_BASE + 5 * CAAM_CMD_SZ) -+#define DESC_AHASH_DIGEST_LEN (DESC_AHASH_BASE + 4 * CAAM_CMD_SZ) -+ -+void cnstr_shdsc_ahash(u32 * const desc, struct alginfo *adata, u32 state, -+ int digestsize, int ctx_len, bool import_ctx, int era); -+ -+#endif /* _CAAMHASH_DESC_H_ */ ---- a/drivers/crypto/caam/caampkc.c -+++ b/drivers/crypto/caam/caampkc.c -@@ -18,6 +18,10 @@ - #define DESC_RSA_PUB_LEN (2 * CAAM_CMD_SZ + sizeof(struct rsa_pub_pdb)) - #define DESC_RSA_PRIV_F1_LEN (2 * CAAM_CMD_SZ + \ - sizeof(struct rsa_priv_f1_pdb)) -+#define DESC_RSA_PRIV_F2_LEN (2 * CAAM_CMD_SZ + \ -+ sizeof(struct rsa_priv_f2_pdb)) -+#define DESC_RSA_PRIV_F3_LEN (2 * CAAM_CMD_SZ + \ -+ sizeof(struct rsa_priv_f3_pdb)) - - static void rsa_io_unmap(struct device *dev, struct rsa_edesc *edesc, - struct akcipher_request *req) -@@ -54,6 +58,42 @@ static void rsa_priv_f1_unmap(struct dev - dma_unmap_single(dev, pdb->d_dma, key->d_sz, DMA_TO_DEVICE); - } - -+static void rsa_priv_f2_unmap(struct device *dev, struct rsa_edesc *edesc, -+ struct akcipher_request *req) -+{ -+ struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); -+ struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); -+ struct caam_rsa_key *key = &ctx->key; -+ struct rsa_priv_f2_pdb *pdb = &edesc->pdb.priv_f2; -+ size_t p_sz = key->p_sz; -+ size_t q_sz = key->p_sz; -+ -+ dma_unmap_single(dev, pdb->d_dma, key->d_sz, DMA_TO_DEVICE); -+ dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE); -+ dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE); -+ dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_TO_DEVICE); -+ dma_unmap_single(dev, pdb->tmp2_dma, q_sz, DMA_TO_DEVICE); -+} -+ -+static void rsa_priv_f3_unmap(struct device *dev, struct rsa_edesc *edesc, -+ struct akcipher_request *req) -+{ -+ struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); -+ struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); -+ struct caam_rsa_key *key = &ctx->key; -+ struct rsa_priv_f3_pdb *pdb = &edesc->pdb.priv_f3; -+ size_t p_sz = key->p_sz; -+ size_t q_sz = key->p_sz; -+ -+ dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE); -+ dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE); -+ dma_unmap_single(dev, pdb->dp_dma, p_sz, DMA_TO_DEVICE); -+ dma_unmap_single(dev, pdb->dq_dma, q_sz, DMA_TO_DEVICE); -+ dma_unmap_single(dev, pdb->c_dma, p_sz, DMA_TO_DEVICE); -+ dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_TO_DEVICE); -+ dma_unmap_single(dev, pdb->tmp2_dma, q_sz, DMA_TO_DEVICE); -+} -+ - /* RSA Job Completion handler */ - static void rsa_pub_done(struct device *dev, u32 *desc, u32 err, void *context) - { -@@ -90,6 +130,42 @@ static void rsa_priv_f1_done(struct devi - akcipher_request_complete(req, err); - } - -+static void rsa_priv_f2_done(struct device *dev, u32 *desc, u32 err, -+ void *context) -+{ -+ struct akcipher_request *req = context; -+ struct rsa_edesc *edesc; -+ -+ if (err) -+ caam_jr_strstatus(dev, err); -+ -+ edesc = container_of(desc, struct rsa_edesc, hw_desc[0]); -+ -+ rsa_priv_f2_unmap(dev, edesc, req); -+ rsa_io_unmap(dev, edesc, req); -+ kfree(edesc); -+ -+ akcipher_request_complete(req, err); -+} -+ -+static void rsa_priv_f3_done(struct device *dev, u32 *desc, u32 err, -+ void *context) -+{ -+ struct akcipher_request *req = context; -+ struct rsa_edesc *edesc; -+ -+ if (err) -+ caam_jr_strstatus(dev, err); -+ -+ edesc = container_of(desc, struct rsa_edesc, hw_desc[0]); -+ -+ rsa_priv_f3_unmap(dev, edesc, req); -+ rsa_io_unmap(dev, edesc, req); -+ kfree(edesc); -+ -+ akcipher_request_complete(req, err); -+} -+ - static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, - size_t desclen) - { -@@ -97,8 +173,8 @@ static struct rsa_edesc *rsa_edesc_alloc - struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); - struct device *dev = ctx->dev; - struct rsa_edesc *edesc; -- gfp_t flags = (req->base.flags & (CRYPTO_TFM_REQ_MAY_BACKLOG | -- CRYPTO_TFM_REQ_MAY_SLEEP)) ? GFP_KERNEL : GFP_ATOMIC; -+ gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? -+ GFP_KERNEL : GFP_ATOMIC; - int sgc; - int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; - int src_nents, dst_nents; -@@ -258,6 +334,172 @@ static int set_rsa_priv_f1_pdb(struct ak - return 0; - } - -+static int set_rsa_priv_f2_pdb(struct akcipher_request *req, -+ struct rsa_edesc *edesc) -+{ -+ struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); -+ struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); -+ struct caam_rsa_key *key = &ctx->key; -+ struct device *dev = ctx->dev; -+ struct rsa_priv_f2_pdb *pdb = &edesc->pdb.priv_f2; -+ int sec4_sg_index = 0; -+ size_t p_sz = key->p_sz; -+ size_t q_sz = key->p_sz; -+ -+ pdb->d_dma = dma_map_single(dev, key->d, key->d_sz, DMA_TO_DEVICE); -+ if (dma_mapping_error(dev, pdb->d_dma)) { -+ dev_err(dev, "Unable to map RSA private exponent memory\n"); -+ return -ENOMEM; -+ } -+ -+ pdb->p_dma = dma_map_single(dev, key->p, p_sz, DMA_TO_DEVICE); -+ if (dma_mapping_error(dev, pdb->p_dma)) { -+ dev_err(dev, "Unable to map RSA prime factor p memory\n"); -+ goto unmap_d; -+ } -+ -+ pdb->q_dma = dma_map_single(dev, key->q, q_sz, DMA_TO_DEVICE); -+ if (dma_mapping_error(dev, pdb->q_dma)) { -+ dev_err(dev, "Unable to map RSA prime factor q memory\n"); -+ goto unmap_p; -+ } -+ -+ pdb->tmp1_dma = dma_map_single(dev, key->tmp1, p_sz, DMA_TO_DEVICE); -+ if (dma_mapping_error(dev, pdb->tmp1_dma)) { -+ dev_err(dev, "Unable to map RSA tmp1 memory\n"); -+ goto unmap_q; -+ } -+ -+ pdb->tmp2_dma = dma_map_single(dev, key->tmp2, q_sz, DMA_TO_DEVICE); -+ if (dma_mapping_error(dev, pdb->tmp2_dma)) { -+ dev_err(dev, "Unable to map RSA tmp2 memory\n"); -+ goto unmap_tmp1; -+ } -+ -+ if (edesc->src_nents > 1) { -+ pdb->sgf |= RSA_PRIV_PDB_SGF_G; -+ pdb->g_dma = edesc->sec4_sg_dma; -+ sec4_sg_index += edesc->src_nents; -+ } else { -+ pdb->g_dma = sg_dma_address(req->src); -+ } -+ -+ if (edesc->dst_nents > 1) { -+ pdb->sgf |= RSA_PRIV_PDB_SGF_F; -+ pdb->f_dma = edesc->sec4_sg_dma + -+ sec4_sg_index * sizeof(struct sec4_sg_entry); -+ } else { -+ pdb->f_dma = sg_dma_address(req->dst); -+ } -+ -+ pdb->sgf |= (key->d_sz << RSA_PDB_D_SHIFT) | key->n_sz; -+ pdb->p_q_len = (q_sz << RSA_PDB_Q_SHIFT) | p_sz; -+ -+ return 0; -+ -+unmap_tmp1: -+ dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_TO_DEVICE); -+unmap_q: -+ dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE); -+unmap_p: -+ dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE); -+unmap_d: -+ dma_unmap_single(dev, pdb->d_dma, key->d_sz, DMA_TO_DEVICE); -+ -+ return -ENOMEM; -+} -+ -+static int set_rsa_priv_f3_pdb(struct akcipher_request *req, -+ struct rsa_edesc *edesc) -+{ -+ struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); -+ struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); -+ struct caam_rsa_key *key = &ctx->key; -+ struct device *dev = ctx->dev; -+ struct rsa_priv_f3_pdb *pdb = &edesc->pdb.priv_f3; -+ int sec4_sg_index = 0; -+ size_t p_sz = key->p_sz; -+ size_t q_sz = key->p_sz; -+ -+ pdb->p_dma = dma_map_single(dev, key->p, p_sz, DMA_TO_DEVICE); -+ if (dma_mapping_error(dev, pdb->p_dma)) { -+ dev_err(dev, "Unable to map RSA prime factor p memory\n"); -+ return -ENOMEM; -+ } -+ -+ pdb->q_dma = dma_map_single(dev, key->q, q_sz, DMA_TO_DEVICE); -+ if (dma_mapping_error(dev, pdb->q_dma)) { -+ dev_err(dev, "Unable to map RSA prime factor q memory\n"); -+ goto unmap_p; -+ } -+ -+ pdb->dp_dma = dma_map_single(dev, key->dp, p_sz, DMA_TO_DEVICE); -+ if (dma_mapping_error(dev, pdb->dp_dma)) { -+ dev_err(dev, "Unable to map RSA exponent dp memory\n"); -+ goto unmap_q; -+ } -+ -+ pdb->dq_dma = dma_map_single(dev, key->dq, q_sz, DMA_TO_DEVICE); -+ if (dma_mapping_error(dev, pdb->dq_dma)) { -+ dev_err(dev, "Unable to map RSA exponent dq memory\n"); -+ goto unmap_dp; -+ } -+ -+ pdb->c_dma = dma_map_single(dev, key->qinv, p_sz, DMA_TO_DEVICE); -+ if (dma_mapping_error(dev, pdb->c_dma)) { -+ dev_err(dev, "Unable to map RSA CRT coefficient qinv memory\n"); -+ goto unmap_dq; -+ } -+ -+ pdb->tmp1_dma = dma_map_single(dev, key->tmp1, p_sz, DMA_TO_DEVICE); -+ if (dma_mapping_error(dev, pdb->tmp1_dma)) { -+ dev_err(dev, "Unable to map RSA tmp1 memory\n"); -+ goto unmap_qinv; -+ } -+ -+ pdb->tmp2_dma = dma_map_single(dev, key->tmp2, q_sz, DMA_TO_DEVICE); -+ if (dma_mapping_error(dev, pdb->tmp2_dma)) { -+ dev_err(dev, "Unable to map RSA tmp2 memory\n"); -+ goto unmap_tmp1; -+ } -+ -+ if (edesc->src_nents > 1) { -+ pdb->sgf |= RSA_PRIV_PDB_SGF_G; -+ pdb->g_dma = edesc->sec4_sg_dma; -+ sec4_sg_index += edesc->src_nents; -+ } else { -+ pdb->g_dma = sg_dma_address(req->src); -+ } -+ -+ if (edesc->dst_nents > 1) { -+ pdb->sgf |= RSA_PRIV_PDB_SGF_F; -+ pdb->f_dma = edesc->sec4_sg_dma + -+ sec4_sg_index * sizeof(struct sec4_sg_entry); -+ } else { -+ pdb->f_dma = sg_dma_address(req->dst); -+ } -+ -+ pdb->sgf |= key->n_sz; -+ pdb->p_q_len = (q_sz << RSA_PDB_Q_SHIFT) | p_sz; -+ -+ return 0; -+ -+unmap_tmp1: -+ dma_unmap_single(dev, pdb->tmp1_dma, p_sz, DMA_TO_DEVICE); -+unmap_qinv: -+ dma_unmap_single(dev, pdb->c_dma, p_sz, DMA_TO_DEVICE); -+unmap_dq: -+ dma_unmap_single(dev, pdb->dq_dma, q_sz, DMA_TO_DEVICE); -+unmap_dp: -+ dma_unmap_single(dev, pdb->dp_dma, p_sz, DMA_TO_DEVICE); -+unmap_q: -+ dma_unmap_single(dev, pdb->q_dma, q_sz, DMA_TO_DEVICE); -+unmap_p: -+ dma_unmap_single(dev, pdb->p_dma, p_sz, DMA_TO_DEVICE); -+ -+ return -ENOMEM; -+} -+ - static int caam_rsa_enc(struct akcipher_request *req) - { - struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); -@@ -301,24 +543,14 @@ init_fail: - return ret; - } - --static int caam_rsa_dec(struct akcipher_request *req) -+static int caam_rsa_dec_priv_f1(struct akcipher_request *req) - { - struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); - struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); -- struct caam_rsa_key *key = &ctx->key; - struct device *jrdev = ctx->dev; - struct rsa_edesc *edesc; - int ret; - -- if (unlikely(!key->n || !key->d)) -- return -EINVAL; -- -- if (req->dst_len < key->n_sz) { -- req->dst_len = key->n_sz; -- dev_err(jrdev, "Output buffer length less than parameter n\n"); -- return -EOVERFLOW; -- } -- - /* Allocate extended descriptor */ - edesc = rsa_edesc_alloc(req, DESC_RSA_PRIV_F1_LEN); - if (IS_ERR(edesc)) -@@ -344,17 +576,147 @@ init_fail: - return ret; - } - -+static int caam_rsa_dec_priv_f2(struct akcipher_request *req) -+{ -+ struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); -+ struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); -+ struct device *jrdev = ctx->dev; -+ struct rsa_edesc *edesc; -+ int ret; -+ -+ /* Allocate extended descriptor */ -+ edesc = rsa_edesc_alloc(req, DESC_RSA_PRIV_F2_LEN); -+ if (IS_ERR(edesc)) -+ return PTR_ERR(edesc); -+ -+ /* Set RSA Decrypt Protocol Data Block - Private Key Form #2 */ -+ ret = set_rsa_priv_f2_pdb(req, edesc); -+ if (ret) -+ goto init_fail; -+ -+ /* Initialize Job Descriptor */ -+ init_rsa_priv_f2_desc(edesc->hw_desc, &edesc->pdb.priv_f2); -+ -+ ret = caam_jr_enqueue(jrdev, edesc->hw_desc, rsa_priv_f2_done, req); -+ if (!ret) -+ return -EINPROGRESS; -+ -+ rsa_priv_f2_unmap(jrdev, edesc, req); -+ -+init_fail: -+ rsa_io_unmap(jrdev, edesc, req); -+ kfree(edesc); -+ return ret; -+} -+ -+static int caam_rsa_dec_priv_f3(struct akcipher_request *req) -+{ -+ struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); -+ struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); -+ struct device *jrdev = ctx->dev; -+ struct rsa_edesc *edesc; -+ int ret; -+ -+ /* Allocate extended descriptor */ -+ edesc = rsa_edesc_alloc(req, DESC_RSA_PRIV_F3_LEN); -+ if (IS_ERR(edesc)) -+ return PTR_ERR(edesc); -+ -+ /* Set RSA Decrypt Protocol Data Block - Private Key Form #3 */ -+ ret = set_rsa_priv_f3_pdb(req, edesc); -+ if (ret) -+ goto init_fail; -+ -+ /* Initialize Job Descriptor */ -+ init_rsa_priv_f3_desc(edesc->hw_desc, &edesc->pdb.priv_f3); -+ -+ ret = caam_jr_enqueue(jrdev, edesc->hw_desc, rsa_priv_f3_done, req); -+ if (!ret) -+ return -EINPROGRESS; -+ -+ rsa_priv_f3_unmap(jrdev, edesc, req); -+ -+init_fail: -+ rsa_io_unmap(jrdev, edesc, req); -+ kfree(edesc); -+ return ret; -+} -+ -+static int caam_rsa_dec(struct akcipher_request *req) -+{ -+ struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); -+ struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); -+ struct caam_rsa_key *key = &ctx->key; -+ int ret; -+ -+ if (unlikely(!key->n || !key->d)) -+ return -EINVAL; -+ -+ if (req->dst_len < key->n_sz) { -+ req->dst_len = key->n_sz; -+ dev_err(ctx->dev, "Output buffer length less than parameter n\n"); -+ return -EOVERFLOW; -+ } -+ -+ if (key->priv_form == FORM3) -+ ret = caam_rsa_dec_priv_f3(req); -+ else if (key->priv_form == FORM2) -+ ret = caam_rsa_dec_priv_f2(req); -+ else -+ ret = caam_rsa_dec_priv_f1(req); -+ -+ return ret; -+} -+ - static void caam_rsa_free_key(struct caam_rsa_key *key) - { - kzfree(key->d); -+ kzfree(key->p); -+ kzfree(key->q); -+ kzfree(key->dp); -+ kzfree(key->dq); -+ kzfree(key->qinv); -+ kzfree(key->tmp1); -+ kzfree(key->tmp2); - kfree(key->e); - kfree(key->n); -- key->d = NULL; -- key->e = NULL; -- key->n = NULL; -- key->d_sz = 0; -- key->e_sz = 0; -- key->n_sz = 0; -+ memset(key, 0, sizeof(*key)); -+} -+ -+static void caam_rsa_drop_leading_zeros(const u8 **ptr, size_t *nbytes) -+{ -+ while (!**ptr && *nbytes) { -+ (*ptr)++; -+ (*nbytes)--; -+ } -+} -+ -+/** -+ * caam_read_rsa_crt - Used for reading dP, dQ, qInv CRT members. -+ * dP, dQ and qInv could decode to less than corresponding p, q length, as the -+ * BER-encoding requires that the minimum number of bytes be used to encode the -+ * integer. dP, dQ, qInv decoded values have to be zero-padded to appropriate -+ * length. -+ * -+ * @ptr : pointer to {dP, dQ, qInv} CRT member -+ * @nbytes: length in bytes of {dP, dQ, qInv} CRT member -+ * @dstlen: length in bytes of corresponding p or q prime factor -+ */ -+static u8 *caam_read_rsa_crt(const u8 *ptr, size_t nbytes, size_t dstlen) -+{ -+ u8 *dst; -+ -+ caam_rsa_drop_leading_zeros(&ptr, &nbytes); -+ if (!nbytes) -+ return NULL; -+ -+ dst = kzalloc(dstlen, GFP_DMA | GFP_KERNEL); -+ if (!dst) -+ return NULL; -+ -+ memcpy(dst + (dstlen - nbytes), ptr, nbytes); -+ -+ return dst; - } - - /** -@@ -370,10 +732,9 @@ static inline u8 *caam_read_raw_data(con - { - u8 *val; - -- while (!*buf && *nbytes) { -- buf++; -- (*nbytes)--; -- } -+ caam_rsa_drop_leading_zeros(&buf, nbytes); -+ if (!*nbytes) -+ return NULL; - - val = kzalloc(*nbytes, GFP_DMA | GFP_KERNEL); - if (!val) -@@ -395,7 +756,7 @@ static int caam_rsa_set_pub_key(struct c - unsigned int keylen) - { - struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); -- struct rsa_key raw_key = {0}; -+ struct rsa_key raw_key = {NULL}; - struct caam_rsa_key *rsa_key = &ctx->key; - int ret; - -@@ -437,11 +798,69 @@ err: - return -ENOMEM; - } - -+static void caam_rsa_set_priv_key_form(struct caam_rsa_ctx *ctx, -+ struct rsa_key *raw_key) -+{ -+ struct caam_rsa_key *rsa_key = &ctx->key; -+ size_t p_sz = raw_key->p_sz; -+ size_t q_sz = raw_key->q_sz; -+ -+ rsa_key->p = caam_read_raw_data(raw_key->p, &p_sz); -+ if (!rsa_key->p) -+ return; -+ rsa_key->p_sz = p_sz; -+ -+ rsa_key->q = caam_read_raw_data(raw_key->q, &q_sz); -+ if (!rsa_key->q) -+ goto free_p; -+ rsa_key->q_sz = q_sz; -+ -+ rsa_key->tmp1 = kzalloc(raw_key->p_sz, GFP_DMA | GFP_KERNEL); -+ if (!rsa_key->tmp1) -+ goto free_q; -+ -+ rsa_key->tmp2 = kzalloc(raw_key->q_sz, GFP_DMA | GFP_KERNEL); -+ if (!rsa_key->tmp2) -+ goto free_tmp1; -+ -+ rsa_key->priv_form = FORM2; -+ -+ rsa_key->dp = caam_read_rsa_crt(raw_key->dp, raw_key->dp_sz, p_sz); -+ if (!rsa_key->dp) -+ goto free_tmp2; -+ -+ rsa_key->dq = caam_read_rsa_crt(raw_key->dq, raw_key->dq_sz, q_sz); -+ if (!rsa_key->dq) -+ goto free_dp; -+ -+ rsa_key->qinv = caam_read_rsa_crt(raw_key->qinv, raw_key->qinv_sz, -+ q_sz); -+ if (!rsa_key->qinv) -+ goto free_dq; -+ -+ rsa_key->priv_form = FORM3; -+ -+ return; -+ -+free_dq: -+ kzfree(rsa_key->dq); -+free_dp: -+ kzfree(rsa_key->dp); -+free_tmp2: -+ kzfree(rsa_key->tmp2); -+free_tmp1: -+ kzfree(rsa_key->tmp1); -+free_q: -+ kzfree(rsa_key->q); -+free_p: -+ kzfree(rsa_key->p); -+} -+ - static int caam_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key, - unsigned int keylen) - { - struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); -- struct rsa_key raw_key = {0}; -+ struct rsa_key raw_key = {NULL}; - struct caam_rsa_key *rsa_key = &ctx->key; - int ret; - -@@ -483,6 +902,8 @@ static int caam_rsa_set_priv_key(struct - memcpy(rsa_key->d, raw_key.d, raw_key.d_sz); - memcpy(rsa_key->e, raw_key.e, raw_key.e_sz); - -+ caam_rsa_set_priv_key_form(ctx, &raw_key); -+ - return 0; - - err: ---- a/drivers/crypto/caam/caampkc.h -+++ b/drivers/crypto/caam/caampkc.h -@@ -13,21 +13,75 @@ - #include "pdb.h" - - /** -+ * caam_priv_key_form - CAAM RSA private key representation -+ * CAAM RSA private key may have either of three forms. -+ * -+ * 1. The first representation consists of the pair (n, d), where the -+ * components have the following meanings: -+ * n the RSA modulus -+ * d the RSA private exponent -+ * -+ * 2. The second representation consists of the triplet (p, q, d), where the -+ * components have the following meanings: -+ * p the first prime factor of the RSA modulus n -+ * q the second prime factor of the RSA modulus n -+ * d the RSA private exponent -+ * -+ * 3. The third representation consists of the quintuple (p, q, dP, dQ, qInv), -+ * where the components have the following meanings: -+ * p the first prime factor of the RSA modulus n -+ * q the second prime factor of the RSA modulus n -+ * dP the first factors's CRT exponent -+ * dQ the second factors's CRT exponent -+ * qInv the (first) CRT coefficient -+ * -+ * The benefit of using the third or the second key form is lower computational -+ * cost for the decryption and signature operations. -+ */ -+enum caam_priv_key_form { -+ FORM1, -+ FORM2, -+ FORM3 -+}; -+ -+/** - * caam_rsa_key - CAAM RSA key structure. Keys are allocated in DMA zone. - * @n : RSA modulus raw byte stream - * @e : RSA public exponent raw byte stream - * @d : RSA private exponent raw byte stream -+ * @p : RSA prime factor p of RSA modulus n -+ * @q : RSA prime factor q of RSA modulus n -+ * @dp : RSA CRT exponent of p -+ * @dp : RSA CRT exponent of q -+ * @qinv : RSA CRT coefficient -+ * @tmp1 : CAAM uses this temporary buffer as internal state buffer. -+ * It is assumed to be as long as p. -+ * @tmp2 : CAAM uses this temporary buffer as internal state buffer. -+ * It is assumed to be as long as q. - * @n_sz : length in bytes of RSA modulus n - * @e_sz : length in bytes of RSA public exponent - * @d_sz : length in bytes of RSA private exponent -+ * @p_sz : length in bytes of RSA prime factor p of RSA modulus n -+ * @q_sz : length in bytes of RSA prime factor q of RSA modulus n -+ * @priv_form : CAAM RSA private key representation - */ - struct caam_rsa_key { - u8 *n; - u8 *e; - u8 *d; -+ u8 *p; -+ u8 *q; -+ u8 *dp; -+ u8 *dq; -+ u8 *qinv; -+ u8 *tmp1; -+ u8 *tmp2; - size_t n_sz; - size_t e_sz; - size_t d_sz; -+ size_t p_sz; -+ size_t q_sz; -+ enum caam_priv_key_form priv_form; - }; - - /** -@@ -59,6 +113,8 @@ struct rsa_edesc { - union { - struct rsa_pub_pdb pub; - struct rsa_priv_f1_pdb priv_f1; -+ struct rsa_priv_f2_pdb priv_f2; -+ struct rsa_priv_f3_pdb priv_f3; - } pdb; - u32 hw_desc[]; - }; -@@ -66,5 +122,7 @@ struct rsa_edesc { - /* Descriptor construction primitives. */ - void init_rsa_pub_desc(u32 *desc, struct rsa_pub_pdb *pdb); - void init_rsa_priv_f1_desc(u32 *desc, struct rsa_priv_f1_pdb *pdb); -+void init_rsa_priv_f2_desc(u32 *desc, struct rsa_priv_f2_pdb *pdb); -+void init_rsa_priv_f3_desc(u32 *desc, struct rsa_priv_f3_pdb *pdb); - - #endif ---- a/drivers/crypto/caam/caamrng.c -+++ b/drivers/crypto/caam/caamrng.c -@@ -52,7 +52,7 @@ - - /* length of descriptors */ - #define DESC_JOB_O_LEN (CAAM_CMD_SZ * 2 + CAAM_PTR_SZ * 2) --#define DESC_RNG_LEN (4 * CAAM_CMD_SZ) -+#define DESC_RNG_LEN (3 * CAAM_CMD_SZ) - - /* Buffer, its dma address and lock */ - struct buf_data { -@@ -100,8 +100,7 @@ static void rng_done(struct device *jrde - { - struct buf_data *bd; - -- bd = (struct buf_data *)((char *)desc - -- offsetof(struct buf_data, hw_desc)); -+ bd = container_of(desc, struct buf_data, hw_desc[0]); - - if (err) - caam_jr_strstatus(jrdev, err); -@@ -196,9 +195,6 @@ static inline int rng_create_sh_desc(str - - init_sh_desc(desc, HDR_SHARE_SERIAL); - -- /* Propagate errors from shared to job descriptor */ -- append_cmd(desc, SET_OK_NO_PROP_ERRORS | CMD_LOAD); -- - /* Generate random bytes */ - append_operation(desc, OP_ALG_ALGSEL_RNG | OP_TYPE_CLASS1_ALG); - -@@ -289,11 +285,7 @@ static int caam_init_rng(struct caam_rng - if (err) - return err; - -- err = caam_init_buf(ctx, 1); -- if (err) -- return err; -- -- return 0; -+ return caam_init_buf(ctx, 1); - } - - static struct hwrng caam_rng = { -@@ -351,7 +343,7 @@ static int __init caam_rng_init(void) - pr_err("Job Ring Device allocation for transform failed\n"); - return PTR_ERR(dev); - } -- rng_ctx = kmalloc(sizeof(*rng_ctx), GFP_DMA); -+ rng_ctx = kmalloc(sizeof(*rng_ctx), GFP_DMA | GFP_KERNEL); - if (!rng_ctx) { - err = -ENOMEM; - goto free_caam_alloc; ---- a/drivers/crypto/caam/compat.h -+++ b/drivers/crypto/caam/compat.h -@@ -16,6 +16,7 @@ - #include <linux/of_platform.h> - #include <linux/dma-mapping.h> - #include <linux/io.h> -+#include <linux/iommu.h> - #include <linux/spinlock.h> - #include <linux/rtnetlink.h> - #include <linux/in.h> ---- a/drivers/crypto/caam/ctrl.c -+++ b/drivers/crypto/caam/ctrl.c -@@ -2,40 +2,41 @@ - * Controller-level driver, kernel property detection, initialization - * - * Copyright 2008-2012 Freescale Semiconductor, Inc. -+ * Copyright 2017 NXP - */ - - #include <linux/device.h> - #include <linux/of_address.h> - #include <linux/of_irq.h> -+#include <linux/sys_soc.h> - - #include "compat.h" - #include "regs.h" - #include "intern.h" - #include "jr.h" - #include "desc_constr.h" --#include "error.h" - #include "ctrl.h" - - bool caam_little_end; - EXPORT_SYMBOL(caam_little_end); -+bool caam_imx; -+EXPORT_SYMBOL(caam_imx); -+bool caam_dpaa2; -+EXPORT_SYMBOL(caam_dpaa2); -+ -+#ifdef CONFIG_CAAM_QI -+#include "qi.h" -+#endif - - /* - * i.MX targets tend to have clock control subsystems that can - * enable/disable clocking to our device. - */ --#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX --static inline struct clk *caam_drv_identify_clk(struct device *dev, -- char *clk_name) --{ -- return devm_clk_get(dev, clk_name); --} --#else - static inline struct clk *caam_drv_identify_clk(struct device *dev, - char *clk_name) - { -- return NULL; -+ return caam_imx ? devm_clk_get(dev, clk_name) : NULL; - } --#endif - - /* - * Descriptor to instantiate RNG State Handle 0 in normal mode and -@@ -274,7 +275,7 @@ static int deinstantiate_rng(struct devi - /* - * If the corresponding bit is set, then it means the state - * handle was initialized by us, and thus it needs to be -- * deintialized as well -+ * deinitialized as well - */ - if ((1 << sh_idx) & state_handle_mask) { - /* -@@ -307,20 +308,24 @@ static int caam_remove(struct platform_d - struct device *ctrldev; - struct caam_drv_private *ctrlpriv; - struct caam_ctrl __iomem *ctrl; -- int ring; - - ctrldev = &pdev->dev; - ctrlpriv = dev_get_drvdata(ctrldev); - ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl; - -- /* Remove platform devices for JobRs */ -- for (ring = 0; ring < ctrlpriv->total_jobrs; ring++) { -- if (ctrlpriv->jrpdev[ring]) -- of_device_unregister(ctrlpriv->jrpdev[ring]); -- } -+ /* Remove platform devices under the crypto node */ -+ of_platform_depopulate(ctrldev); -+ -+#ifdef CONFIG_CAAM_QI -+ if (ctrlpriv->qidev) -+ caam_qi_shutdown(ctrlpriv->qidev); -+#endif - -- /* De-initialize RNG state handles initialized by this driver. */ -- if (ctrlpriv->rng4_sh_init) -+ /* -+ * De-initialize RNG state handles initialized by this driver. -+ * In case of DPAA 2.x, RNG is managed by MC firmware. -+ */ -+ if (!caam_dpaa2 && ctrlpriv->rng4_sh_init) - deinstantiate_rng(ctrldev, ctrlpriv->rng4_sh_init); - - /* Shut down debug views */ -@@ -335,8 +340,8 @@ static int caam_remove(struct platform_d - clk_disable_unprepare(ctrlpriv->caam_ipg); - clk_disable_unprepare(ctrlpriv->caam_mem); - clk_disable_unprepare(ctrlpriv->caam_aclk); -- clk_disable_unprepare(ctrlpriv->caam_emi_slow); -- -+ if (ctrlpriv->caam_emi_slow) -+ clk_disable_unprepare(ctrlpriv->caam_emi_slow); - return 0; - } - -@@ -370,11 +375,8 @@ static void kick_trng(struct platform_de - */ - val = (rd_reg32(&r4tst->rtsdctl) & RTSDCTL_ENT_DLY_MASK) - >> RTSDCTL_ENT_DLY_SHIFT; -- if (ent_delay <= val) { -- /* put RNG4 into run mode */ -- clrsetbits_32(&r4tst->rtmctl, RTMCTL_PRGM, 0); -- return; -- } -+ if (ent_delay <= val) -+ goto start_rng; - - val = rd_reg32(&r4tst->rtsdctl); - val = (val & ~RTSDCTL_ENT_DLY_MASK) | -@@ -386,15 +388,12 @@ static void kick_trng(struct platform_de - wr_reg32(&r4tst->rtfrqmax, RTFRQMAX_DISABLE); - /* read the control register */ - val = rd_reg32(&r4tst->rtmctl); -+start_rng: - /* - * select raw sampling in both entropy shifter -- * and statistical checker -+ * and statistical checker; ; put RNG4 into run mode - */ -- clrsetbits_32(&val, 0, RTMCTL_SAMP_MODE_RAW_ES_SC); -- /* put RNG4 into run mode */ -- clrsetbits_32(&val, RTMCTL_PRGM, 0); -- /* write back the control register */ -- wr_reg32(&r4tst->rtmctl, val); -+ clrsetbits_32(&r4tst->rtmctl, RTMCTL_PRGM, RTMCTL_SAMP_MODE_RAW_ES_SC); - } - - /** -@@ -415,28 +414,26 @@ int caam_get_era(void) - } - EXPORT_SYMBOL(caam_get_era); - --#ifdef CONFIG_DEBUG_FS --static int caam_debugfs_u64_get(void *data, u64 *val) --{ -- *val = caam64_to_cpu(*(u64 *)data); -- return 0; --} -- --static int caam_debugfs_u32_get(void *data, u64 *val) --{ -- *val = caam32_to_cpu(*(u32 *)data); -- return 0; --} -- --DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n"); --DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n"); --#endif -+static const struct of_device_id caam_match[] = { -+ { -+ .compatible = "fsl,sec-v4.0", -+ }, -+ { -+ .compatible = "fsl,sec4.0", -+ }, -+ {}, -+}; -+MODULE_DEVICE_TABLE(of, caam_match); - - /* Probe routine for CAAM top (controller) level */ - static int caam_probe(struct platform_device *pdev) - { -- int ret, ring, rspec, gen_sk, ent_delay = RTSDCTL_ENT_DLY_MIN; -+ int ret, ring, gen_sk, ent_delay = RTSDCTL_ENT_DLY_MIN; - u64 caam_id; -+ static const struct soc_device_attribute imx_soc[] = { -+ {.family = "Freescale i.MX"}, -+ {}, -+ }; - struct device *dev; - struct device_node *nprop, *np; - struct caam_ctrl __iomem *ctrl; -@@ -456,9 +453,10 @@ static int caam_probe(struct platform_de - - dev = &pdev->dev; - dev_set_drvdata(dev, ctrlpriv); -- ctrlpriv->pdev = pdev; - nprop = pdev->dev.of_node; - -+ caam_imx = (bool)soc_device_match(imx_soc); -+ - /* Enable clocking */ - clk = caam_drv_identify_clk(&pdev->dev, "ipg"); - if (IS_ERR(clk)) { -@@ -487,14 +485,16 @@ static int caam_probe(struct platform_de - } - ctrlpriv->caam_aclk = clk; - -- clk = caam_drv_identify_clk(&pdev->dev, "emi_slow"); -- if (IS_ERR(clk)) { -- ret = PTR_ERR(clk); -- dev_err(&pdev->dev, -- "can't identify CAAM emi_slow clk: %d\n", ret); -- return ret; -+ if (!of_machine_is_compatible("fsl,imx6ul")) { -+ clk = caam_drv_identify_clk(&pdev->dev, "emi_slow"); -+ if (IS_ERR(clk)) { -+ ret = PTR_ERR(clk); -+ dev_err(&pdev->dev, -+ "can't identify CAAM emi_slow clk: %d\n", ret); -+ return ret; -+ } -+ ctrlpriv->caam_emi_slow = clk; - } -- ctrlpriv->caam_emi_slow = clk; - - ret = clk_prepare_enable(ctrlpriv->caam_ipg); - if (ret < 0) { -@@ -515,11 +515,13 @@ static int caam_probe(struct platform_de - goto disable_caam_mem; - } - -- ret = clk_prepare_enable(ctrlpriv->caam_emi_slow); -- if (ret < 0) { -- dev_err(&pdev->dev, "can't enable CAAM emi slow clock: %d\n", -- ret); -- goto disable_caam_aclk; -+ if (ctrlpriv->caam_emi_slow) { -+ ret = clk_prepare_enable(ctrlpriv->caam_emi_slow); -+ if (ret < 0) { -+ dev_err(&pdev->dev, "can't enable CAAM emi slow clock: %d\n", -+ ret); -+ goto disable_caam_aclk; -+ } - } - - /* Get configuration properties from device tree */ -@@ -546,13 +548,13 @@ static int caam_probe(struct platform_de - else - BLOCK_OFFSET = PG_SIZE_64K; - -- ctrlpriv->ctrl = (struct caam_ctrl __force *)ctrl; -- ctrlpriv->assure = (struct caam_assurance __force *) -- ((uint8_t *)ctrl + -+ ctrlpriv->ctrl = (struct caam_ctrl __iomem __force *)ctrl; -+ ctrlpriv->assure = (struct caam_assurance __iomem __force *) -+ ((__force uint8_t *)ctrl + - BLOCK_OFFSET * ASSURE_BLOCK_NUMBER - ); -- ctrlpriv->deco = (struct caam_deco __force *) -- ((uint8_t *)ctrl + -+ ctrlpriv->deco = (struct caam_deco __iomem __force *) -+ ((__force uint8_t *)ctrl + - BLOCK_OFFSET * DECO_BLOCK_NUMBER - ); - -@@ -561,12 +563,17 @@ static int caam_probe(struct platform_de - - /* - * Enable DECO watchdogs and, if this is a PHYS_ADDR_T_64BIT kernel, -- * long pointers in master configuration register -+ * long pointers in master configuration register. -+ * In case of DPAA 2.x, Management Complex firmware performs -+ * the configuration. - */ -- clrsetbits_32(&ctrl->mcr, MCFGR_AWCACHE_MASK | MCFGR_LONG_PTR, -- MCFGR_AWCACHE_CACH | MCFGR_AWCACHE_BUFF | -- MCFGR_WDENABLE | MCFGR_LARGE_BURST | -- (sizeof(dma_addr_t) == sizeof(u64) ? MCFGR_LONG_PTR : 0)); -+ caam_dpaa2 = !!(comp_params & CTPR_MS_DPAA2); -+ if (!caam_dpaa2) -+ clrsetbits_32(&ctrl->mcr, MCFGR_AWCACHE_MASK | MCFGR_LONG_PTR, -+ MCFGR_AWCACHE_CACH | MCFGR_AWCACHE_BUFF | -+ MCFGR_WDENABLE | MCFGR_LARGE_BURST | -+ (sizeof(dma_addr_t) == sizeof(u64) ? -+ MCFGR_LONG_PTR : 0)); - - /* - * Read the Compile Time paramters and SCFGR to determine -@@ -594,64 +601,69 @@ static int caam_probe(struct platform_de - JRSTART_JR1_START | JRSTART_JR2_START | - JRSTART_JR3_START); - -- if (sizeof(dma_addr_t) == sizeof(u64)) -- if (of_device_is_compatible(nprop, "fsl,sec-v5.0")) -- dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40)); -+ if (sizeof(dma_addr_t) == sizeof(u64)) { -+ if (caam_dpaa2) -+ ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(49)); -+ else if (of_device_is_compatible(nprop, "fsl,sec-v5.0")) -+ ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40)); - else -- dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36)); -- else -- dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); -+ ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(36)); -+ } else { -+ ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); -+ } -+ if (ret) { -+ dev_err(dev, "dma_set_mask_and_coherent failed (%d)\n", ret); -+ goto iounmap_ctrl; -+ } - -- /* -- * Detect and enable JobRs -- * First, find out how many ring spec'ed, allocate references -- * for all, then go probe each one. -- */ -- rspec = 0; -- for_each_available_child_of_node(nprop, np) -- if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") || -- of_device_is_compatible(np, "fsl,sec4.0-job-ring")) -- rspec++; -+ ctrlpriv->era = caam_get_era(); - -- ctrlpriv->jrpdev = devm_kcalloc(&pdev->dev, rspec, -- sizeof(*ctrlpriv->jrpdev), GFP_KERNEL); -- if (ctrlpriv->jrpdev == NULL) { -- ret = -ENOMEM; -+ ret = of_platform_populate(nprop, caam_match, NULL, dev); -+ if (ret) { -+ dev_err(dev, "JR platform devices creation error\n"); - goto iounmap_ctrl; - } - -+#ifdef CONFIG_DEBUG_FS -+ /* -+ * FIXME: needs better naming distinction, as some amalgamation of -+ * "caam" and nprop->full_name. The OF name isn't distinctive, -+ * but does separate instances -+ */ -+ perfmon = (struct caam_perfmon __force *)&ctrl->perfmon; -+ -+ ctrlpriv->dfs_root = debugfs_create_dir(dev_name(dev), NULL); -+ ctrlpriv->ctl = debugfs_create_dir("ctl", ctrlpriv->dfs_root); -+#endif - ring = 0; -- ctrlpriv->total_jobrs = 0; - for_each_available_child_of_node(nprop, np) - if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") || - of_device_is_compatible(np, "fsl,sec4.0-job-ring")) { -- ctrlpriv->jrpdev[ring] = -- of_platform_device_create(np, NULL, dev); -- if (!ctrlpriv->jrpdev[ring]) { -- pr_warn("JR%d Platform device creation error\n", -- ring); -- continue; -- } -- ctrlpriv->jr[ring] = (struct caam_job_ring __force *) -- ((uint8_t *)ctrl + -+ ctrlpriv->jr[ring] = (struct caam_job_ring __iomem __force *) -+ ((__force uint8_t *)ctrl + - (ring + JR_BLOCK_NUMBER) * - BLOCK_OFFSET - ); - ctrlpriv->total_jobrs++; - ring++; -- } -+ } - -- /* Check to see if QI present. If so, enable */ -- ctrlpriv->qi_present = -- !!(rd_reg32(&ctrl->perfmon.comp_parms_ms) & -- CTPR_MS_QI_MASK); -- if (ctrlpriv->qi_present) { -- ctrlpriv->qi = (struct caam_queue_if __force *) -- ((uint8_t *)ctrl + -+ /* Check to see if (DPAA 1.x) QI present. If so, enable */ -+ ctrlpriv->qi_present = !!(comp_params & CTPR_MS_QI_MASK); -+ if (ctrlpriv->qi_present && !caam_dpaa2) { -+ ctrlpriv->qi = (struct caam_queue_if __iomem __force *) -+ ((__force uint8_t *)ctrl + - BLOCK_OFFSET * QI_BLOCK_NUMBER - ); - /* This is all that's required to physically enable QI */ - wr_reg32(&ctrlpriv->qi->qi_control_lo, QICTL_DQEN); -+ -+ /* If QMAN driver is present, init CAAM-QI backend */ -+#ifdef CONFIG_CAAM_QI -+ ret = caam_qi_init(pdev); -+ if (ret) -+ dev_err(dev, "caam qi i/f init failed: %d\n", ret); -+#endif - } - - /* If no QI and no rings specified, quit and go home */ -@@ -666,8 +678,10 @@ static int caam_probe(struct platform_de - /* - * If SEC has RNG version >= 4 and RNG state handle has not been - * already instantiated, do RNG instantiation -+ * In case of DPAA 2.x, RNG is managed by MC firmware. - */ -- if ((cha_vid_ls & CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT >= 4) { -+ if (!caam_dpaa2 && -+ (cha_vid_ls & CHA_ID_LS_RNG_MASK) >> CHA_ID_LS_RNG_SHIFT >= 4) { - ctrlpriv->rng4_sh_init = - rd_reg32(&ctrl->r4tst[0].rdsta); - /* -@@ -734,78 +748,47 @@ static int caam_probe(struct platform_de - - /* Report "alive" for developer to see */ - dev_info(dev, "device ID = 0x%016llx (Era %d)\n", caam_id, -- caam_get_era()); -- dev_info(dev, "job rings = %d, qi = %d\n", -- ctrlpriv->total_jobrs, ctrlpriv->qi_present); -+ ctrlpriv->era); -+ dev_info(dev, "job rings = %d, qi = %d, dpaa2 = %s\n", -+ ctrlpriv->total_jobrs, ctrlpriv->qi_present, -+ caam_dpaa2 ? "yes" : "no"); - - #ifdef CONFIG_DEBUG_FS -- /* -- * FIXME: needs better naming distinction, as some amalgamation of -- * "caam" and nprop->full_name. The OF name isn't distinctive, -- * but does separate instances -- */ -- perfmon = (struct caam_perfmon __force *)&ctrl->perfmon; -- -- ctrlpriv->dfs_root = debugfs_create_dir(dev_name(dev), NULL); -- ctrlpriv->ctl = debugfs_create_dir("ctl", ctrlpriv->dfs_root); -- -- /* Controller-level - performance monitor counters */ -- -- ctrlpriv->ctl_rq_dequeued = -- debugfs_create_file("rq_dequeued", -- S_IRUSR | S_IRGRP | S_IROTH, -- ctrlpriv->ctl, &perfmon->req_dequeued, -- &caam_fops_u64_ro); -- ctrlpriv->ctl_ob_enc_req = -- debugfs_create_file("ob_rq_encrypted", -- S_IRUSR | S_IRGRP | S_IROTH, -- ctrlpriv->ctl, &perfmon->ob_enc_req, -- &caam_fops_u64_ro); -- ctrlpriv->ctl_ib_dec_req = -- debugfs_create_file("ib_rq_decrypted", -- S_IRUSR | S_IRGRP | S_IROTH, -- ctrlpriv->ctl, &perfmon->ib_dec_req, -- &caam_fops_u64_ro); -- ctrlpriv->ctl_ob_enc_bytes = -- debugfs_create_file("ob_bytes_encrypted", -- S_IRUSR | S_IRGRP | S_IROTH, -- ctrlpriv->ctl, &perfmon->ob_enc_bytes, -- &caam_fops_u64_ro); -- ctrlpriv->ctl_ob_prot_bytes = -- debugfs_create_file("ob_bytes_protected", -- S_IRUSR | S_IRGRP | S_IROTH, -- ctrlpriv->ctl, &perfmon->ob_prot_bytes, -- &caam_fops_u64_ro); -- ctrlpriv->ctl_ib_dec_bytes = -- debugfs_create_file("ib_bytes_decrypted", -- S_IRUSR | S_IRGRP | S_IROTH, -- ctrlpriv->ctl, &perfmon->ib_dec_bytes, -- &caam_fops_u64_ro); -- ctrlpriv->ctl_ib_valid_bytes = -- debugfs_create_file("ib_bytes_validated", -- S_IRUSR | S_IRGRP | S_IROTH, -- ctrlpriv->ctl, &perfmon->ib_valid_bytes, -- &caam_fops_u64_ro); -+ debugfs_create_file("rq_dequeued", S_IRUSR | S_IRGRP | S_IROTH, -+ ctrlpriv->ctl, &perfmon->req_dequeued, -+ &caam_fops_u64_ro); -+ debugfs_create_file("ob_rq_encrypted", S_IRUSR | S_IRGRP | S_IROTH, -+ ctrlpriv->ctl, &perfmon->ob_enc_req, -+ &caam_fops_u64_ro); -+ debugfs_create_file("ib_rq_decrypted", S_IRUSR | S_IRGRP | S_IROTH, -+ ctrlpriv->ctl, &perfmon->ib_dec_req, -+ &caam_fops_u64_ro); -+ debugfs_create_file("ob_bytes_encrypted", S_IRUSR | S_IRGRP | S_IROTH, -+ ctrlpriv->ctl, &perfmon->ob_enc_bytes, -+ &caam_fops_u64_ro); -+ debugfs_create_file("ob_bytes_protected", S_IRUSR | S_IRGRP | S_IROTH, -+ ctrlpriv->ctl, &perfmon->ob_prot_bytes, -+ &caam_fops_u64_ro); -+ debugfs_create_file("ib_bytes_decrypted", S_IRUSR | S_IRGRP | S_IROTH, -+ ctrlpriv->ctl, &perfmon->ib_dec_bytes, -+ &caam_fops_u64_ro); -+ debugfs_create_file("ib_bytes_validated", S_IRUSR | S_IRGRP | S_IROTH, -+ ctrlpriv->ctl, &perfmon->ib_valid_bytes, -+ &caam_fops_u64_ro); - - /* Controller level - global status values */ -- ctrlpriv->ctl_faultaddr = -- debugfs_create_file("fault_addr", -- S_IRUSR | S_IRGRP | S_IROTH, -- ctrlpriv->ctl, &perfmon->faultaddr, -- &caam_fops_u32_ro); -- ctrlpriv->ctl_faultdetail = -- debugfs_create_file("fault_detail", -- S_IRUSR | S_IRGRP | S_IROTH, -- ctrlpriv->ctl, &perfmon->faultdetail, -- &caam_fops_u32_ro); -- ctrlpriv->ctl_faultstatus = -- debugfs_create_file("fault_status", -- S_IRUSR | S_IRGRP | S_IROTH, -- ctrlpriv->ctl, &perfmon->status, -- &caam_fops_u32_ro); -+ debugfs_create_file("fault_addr", S_IRUSR | S_IRGRP | S_IROTH, -+ ctrlpriv->ctl, &perfmon->faultaddr, -+ &caam_fops_u32_ro); -+ debugfs_create_file("fault_detail", S_IRUSR | S_IRGRP | S_IROTH, -+ ctrlpriv->ctl, &perfmon->faultdetail, -+ &caam_fops_u32_ro); -+ debugfs_create_file("fault_status", S_IRUSR | S_IRGRP | S_IROTH, -+ ctrlpriv->ctl, &perfmon->status, -+ &caam_fops_u32_ro); - - /* Internal covering keys (useful in non-secure mode only) */ -- ctrlpriv->ctl_kek_wrap.data = &ctrlpriv->ctrl->kek[0]; -+ ctrlpriv->ctl_kek_wrap.data = (__force void *)&ctrlpriv->ctrl->kek[0]; - ctrlpriv->ctl_kek_wrap.size = KEK_KEY_SIZE * sizeof(u32); - ctrlpriv->ctl_kek = debugfs_create_blob("kek", - S_IRUSR | -@@ -813,7 +796,7 @@ static int caam_probe(struct platform_de - ctrlpriv->ctl, - &ctrlpriv->ctl_kek_wrap); - -- ctrlpriv->ctl_tkek_wrap.data = &ctrlpriv->ctrl->tkek[0]; -+ ctrlpriv->ctl_tkek_wrap.data = (__force void *)&ctrlpriv->ctrl->tkek[0]; - ctrlpriv->ctl_tkek_wrap.size = KEK_KEY_SIZE * sizeof(u32); - ctrlpriv->ctl_tkek = debugfs_create_blob("tkek", - S_IRUSR | -@@ -821,7 +804,7 @@ static int caam_probe(struct platform_de - ctrlpriv->ctl, - &ctrlpriv->ctl_tkek_wrap); - -- ctrlpriv->ctl_tdsk_wrap.data = &ctrlpriv->ctrl->tdsk[0]; -+ ctrlpriv->ctl_tdsk_wrap.data = (__force void *)&ctrlpriv->ctrl->tdsk[0]; - ctrlpriv->ctl_tdsk_wrap.size = KEK_KEY_SIZE * sizeof(u32); - ctrlpriv->ctl_tdsk = debugfs_create_blob("tdsk", - S_IRUSR | -@@ -832,13 +815,17 @@ static int caam_probe(struct platform_de - return 0; - - caam_remove: -+#ifdef CONFIG_DEBUG_FS -+ debugfs_remove_recursive(ctrlpriv->dfs_root); -+#endif - caam_remove(pdev); - return ret; - - iounmap_ctrl: - iounmap(ctrl); - disable_caam_emi_slow: -- clk_disable_unprepare(ctrlpriv->caam_emi_slow); -+ if (ctrlpriv->caam_emi_slow) -+ clk_disable_unprepare(ctrlpriv->caam_emi_slow); - disable_caam_aclk: - clk_disable_unprepare(ctrlpriv->caam_aclk); - disable_caam_mem: -@@ -848,17 +835,6 @@ disable_caam_ipg: - return ret; - } - --static struct of_device_id caam_match[] = { -- { -- .compatible = "fsl,sec-v4.0", -- }, -- { -- .compatible = "fsl,sec4.0", -- }, -- {}, --}; --MODULE_DEVICE_TABLE(of, caam_match); -- - static struct platform_driver caam_driver = { - .driver = { - .name = "caam", ---- a/drivers/crypto/caam/ctrl.h -+++ b/drivers/crypto/caam/ctrl.h -@@ -10,4 +10,6 @@ - /* Prototypes for backend-level services exposed to APIs */ - int caam_get_era(void); - -+extern bool caam_dpaa2; -+ - #endif /* CTRL_H */ ---- a/drivers/crypto/caam/desc.h -+++ b/drivers/crypto/caam/desc.h -@@ -22,12 +22,6 @@ - #define SEC4_SG_LEN_MASK 0x3fffffff /* Excludes EXT and FINAL */ - #define SEC4_SG_OFFSET_MASK 0x00001fff - --struct sec4_sg_entry { -- u64 ptr; -- u32 len; -- u32 bpid_offset; --}; -- - /* Max size of any CAAM descriptor in 32-bit words, inclusive of header */ - #define MAX_CAAM_DESCSIZE 64 - -@@ -47,6 +41,7 @@ struct sec4_sg_entry { - #define CMD_SEQ_LOAD (0x03 << CMD_SHIFT) - #define CMD_FIFO_LOAD (0x04 << CMD_SHIFT) - #define CMD_SEQ_FIFO_LOAD (0x05 << CMD_SHIFT) -+#define CMD_MOVEB (0x07 << CMD_SHIFT) - #define CMD_STORE (0x0a << CMD_SHIFT) - #define CMD_SEQ_STORE (0x0b << CMD_SHIFT) - #define CMD_FIFO_STORE (0x0c << CMD_SHIFT) -@@ -90,8 +85,8 @@ struct sec4_sg_entry { - #define HDR_ZRO 0x00008000 - - /* Start Index or SharedDesc Length */ --#define HDR_START_IDX_MASK 0x3f - #define HDR_START_IDX_SHIFT 16 -+#define HDR_START_IDX_MASK (0x3f << HDR_START_IDX_SHIFT) - - /* If shared descriptor header, 6-bit length */ - #define HDR_DESCLEN_SHR_MASK 0x3f -@@ -121,10 +116,10 @@ struct sec4_sg_entry { - #define HDR_PROP_DNR 0x00000800 - - /* JobDesc/SharedDesc share property */ --#define HDR_SD_SHARE_MASK 0x03 - #define HDR_SD_SHARE_SHIFT 8 --#define HDR_JD_SHARE_MASK 0x07 -+#define HDR_SD_SHARE_MASK (0x03 << HDR_SD_SHARE_SHIFT) - #define HDR_JD_SHARE_SHIFT 8 -+#define HDR_JD_SHARE_MASK (0x07 << HDR_JD_SHARE_SHIFT) - - #define HDR_SHARE_NEVER (0x00 << HDR_SD_SHARE_SHIFT) - #define HDR_SHARE_WAIT (0x01 << HDR_SD_SHARE_SHIFT) -@@ -235,7 +230,7 @@ struct sec4_sg_entry { - #define LDST_SRCDST_WORD_DECO_MATH2 (0x0a << LDST_SRCDST_SHIFT) - #define LDST_SRCDST_WORD_DECO_AAD_SZ (0x0b << LDST_SRCDST_SHIFT) - #define LDST_SRCDST_WORD_DECO_MATH3 (0x0b << LDST_SRCDST_SHIFT) --#define LDST_SRCDST_WORD_CLASS1_ICV_SZ (0x0c << LDST_SRCDST_SHIFT) -+#define LDST_SRCDST_WORD_CLASS1_IV_SZ (0x0c << LDST_SRCDST_SHIFT) - #define LDST_SRCDST_WORD_ALTDS_CLASS1 (0x0f << LDST_SRCDST_SHIFT) - #define LDST_SRCDST_WORD_PKHA_A_SZ (0x10 << LDST_SRCDST_SHIFT) - #define LDST_SRCDST_WORD_PKHA_B_SZ (0x11 << LDST_SRCDST_SHIFT) -@@ -360,6 +355,7 @@ struct sec4_sg_entry { - #define FIFOLD_TYPE_PK_N (0x08 << FIFOLD_TYPE_SHIFT) - #define FIFOLD_TYPE_PK_A (0x0c << FIFOLD_TYPE_SHIFT) - #define FIFOLD_TYPE_PK_B (0x0d << FIFOLD_TYPE_SHIFT) -+#define FIFOLD_TYPE_IFIFO (0x0f << FIFOLD_TYPE_SHIFT) - - /* Other types. Need to OR in last/flush bits as desired */ - #define FIFOLD_TYPE_MSG_MASK (0x38 << FIFOLD_TYPE_SHIFT) -@@ -400,7 +396,7 @@ struct sec4_sg_entry { - #define FIFOST_TYPE_PKHA_N (0x08 << FIFOST_TYPE_SHIFT) - #define FIFOST_TYPE_PKHA_A (0x0c << FIFOST_TYPE_SHIFT) - #define FIFOST_TYPE_PKHA_B (0x0d << FIFOST_TYPE_SHIFT) --#define FIFOST_TYPE_AF_SBOX_JKEK (0x10 << FIFOST_TYPE_SHIFT) -+#define FIFOST_TYPE_AF_SBOX_JKEK (0x20 << FIFOST_TYPE_SHIFT) - #define FIFOST_TYPE_AF_SBOX_TKEK (0x21 << FIFOST_TYPE_SHIFT) - #define FIFOST_TYPE_PKHA_E_JKEK (0x22 << FIFOST_TYPE_SHIFT) - #define FIFOST_TYPE_PKHA_E_TKEK (0x23 << FIFOST_TYPE_SHIFT) -@@ -413,6 +409,7 @@ struct sec4_sg_entry { - #define FIFOST_TYPE_MESSAGE_DATA (0x30 << FIFOST_TYPE_SHIFT) - #define FIFOST_TYPE_RNGSTORE (0x34 << FIFOST_TYPE_SHIFT) - #define FIFOST_TYPE_RNGFIFO (0x35 << FIFOST_TYPE_SHIFT) -+#define FIFOST_TYPE_METADATA (0x3e << FIFOST_TYPE_SHIFT) - #define FIFOST_TYPE_SKIP (0x3f << FIFOST_TYPE_SHIFT) - - /* -@@ -449,6 +446,18 @@ struct sec4_sg_entry { - #define OP_PCLID_DSAVERIFY (0x16 << OP_PCLID_SHIFT) - #define OP_PCLID_RSAENC_PUBKEY (0x18 << OP_PCLID_SHIFT) - #define OP_PCLID_RSADEC_PRVKEY (0x19 << OP_PCLID_SHIFT) -+#define OP_PCLID_DKP_MD5 (0x20 << OP_PCLID_SHIFT) -+#define OP_PCLID_DKP_SHA1 (0x21 << OP_PCLID_SHIFT) -+#define OP_PCLID_DKP_SHA224 (0x22 << OP_PCLID_SHIFT) -+#define OP_PCLID_DKP_SHA256 (0x23 << OP_PCLID_SHIFT) -+#define OP_PCLID_DKP_SHA384 (0x24 << OP_PCLID_SHIFT) -+#define OP_PCLID_DKP_SHA512 (0x25 << OP_PCLID_SHIFT) -+#define OP_PCLID_DKP_RIF_MD5 (0x60 << OP_PCLID_SHIFT) -+#define OP_PCLID_DKP_RIF_SHA1 (0x61 << OP_PCLID_SHIFT) -+#define OP_PCLID_DKP_RIF_SHA224 (0x62 << OP_PCLID_SHIFT) -+#define OP_PCLID_DKP_RIF_SHA256 (0x63 << OP_PCLID_SHIFT) -+#define OP_PCLID_DKP_RIF_SHA384 (0x64 << OP_PCLID_SHIFT) -+#define OP_PCLID_DKP_RIF_SHA512 (0x65 << OP_PCLID_SHIFT) - - /* Assuming OP_TYPE = OP_TYPE_DECAP_PROTOCOL/ENCAP_PROTOCOL */ - #define OP_PCLID_IPSEC (0x01 << OP_PCLID_SHIFT) -@@ -1098,6 +1107,22 @@ struct sec4_sg_entry { - /* MacSec protinfos */ - #define OP_PCL_MACSEC 0x0001 - -+/* Derived Key Protocol (DKP) Protinfo */ -+#define OP_PCL_DKP_SRC_SHIFT 14 -+#define OP_PCL_DKP_SRC_MASK (3 << OP_PCL_DKP_SRC_SHIFT) -+#define OP_PCL_DKP_SRC_IMM (0 << OP_PCL_DKP_SRC_SHIFT) -+#define OP_PCL_DKP_SRC_SEQ (1 << OP_PCL_DKP_SRC_SHIFT) -+#define OP_PCL_DKP_SRC_PTR (2 << OP_PCL_DKP_SRC_SHIFT) -+#define OP_PCL_DKP_SRC_SGF (3 << OP_PCL_DKP_SRC_SHIFT) -+#define OP_PCL_DKP_DST_SHIFT 12 -+#define OP_PCL_DKP_DST_MASK (3 << OP_PCL_DKP_DST_SHIFT) -+#define OP_PCL_DKP_DST_IMM (0 << OP_PCL_DKP_DST_SHIFT) -+#define OP_PCL_DKP_DST_SEQ (1 << OP_PCL_DKP_DST_SHIFT) -+#define OP_PCL_DKP_DST_PTR (2 << OP_PCL_DKP_DST_SHIFT) -+#define OP_PCL_DKP_DST_SGF (3 << OP_PCL_DKP_DST_SHIFT) -+#define OP_PCL_DKP_KEY_SHIFT 0 -+#define OP_PCL_DKP_KEY_MASK (0xfff << OP_PCL_DKP_KEY_SHIFT) -+ - /* PKI unidirectional protocol protinfo bits */ - #define OP_PCL_PKPROT_TEST 0x0008 - #define OP_PCL_PKPROT_DECRYPT 0x0004 -@@ -1107,8 +1132,8 @@ struct sec4_sg_entry { - /* For non-protocol/alg-only op commands */ - #define OP_ALG_TYPE_SHIFT 24 - #define OP_ALG_TYPE_MASK (0x7 << OP_ALG_TYPE_SHIFT) --#define OP_ALG_TYPE_CLASS1 2 --#define OP_ALG_TYPE_CLASS2 4 -+#define OP_ALG_TYPE_CLASS1 (2 << OP_ALG_TYPE_SHIFT) -+#define OP_ALG_TYPE_CLASS2 (4 << OP_ALG_TYPE_SHIFT) - - #define OP_ALG_ALGSEL_SHIFT 16 - #define OP_ALG_ALGSEL_MASK (0xff << OP_ALG_ALGSEL_SHIFT) -@@ -1249,7 +1274,7 @@ struct sec4_sg_entry { - #define OP_ALG_PKMODE_MOD_PRIMALITY 0x00f - - /* PKHA mode copy-memory functions */ --#define OP_ALG_PKMODE_SRC_REG_SHIFT 13 -+#define OP_ALG_PKMODE_SRC_REG_SHIFT 17 - #define OP_ALG_PKMODE_SRC_REG_MASK (7 << OP_ALG_PKMODE_SRC_REG_SHIFT) - #define OP_ALG_PKMODE_DST_REG_SHIFT 10 - #define OP_ALG_PKMODE_DST_REG_MASK (7 << OP_ALG_PKMODE_DST_REG_SHIFT) -@@ -1445,10 +1470,11 @@ struct sec4_sg_entry { - #define MATH_SRC1_REG2 (0x02 << MATH_SRC1_SHIFT) - #define MATH_SRC1_REG3 (0x03 << MATH_SRC1_SHIFT) - #define MATH_SRC1_IMM (0x04 << MATH_SRC1_SHIFT) --#define MATH_SRC1_DPOVRD (0x07 << MATH_SRC0_SHIFT) -+#define MATH_SRC1_DPOVRD (0x07 << MATH_SRC1_SHIFT) - #define MATH_SRC1_INFIFO (0x0a << MATH_SRC1_SHIFT) - #define MATH_SRC1_OUTFIFO (0x0b << MATH_SRC1_SHIFT) - #define MATH_SRC1_ONE (0x0c << MATH_SRC1_SHIFT) -+#define MATH_SRC1_ZERO (0x0f << MATH_SRC1_SHIFT) - - /* Destination selectors */ - #define MATH_DEST_SHIFT 8 -@@ -1457,6 +1483,7 @@ struct sec4_sg_entry { - #define MATH_DEST_REG1 (0x01 << MATH_DEST_SHIFT) - #define MATH_DEST_REG2 (0x02 << MATH_DEST_SHIFT) - #define MATH_DEST_REG3 (0x03 << MATH_DEST_SHIFT) -+#define MATH_DEST_DPOVRD (0x07 << MATH_DEST_SHIFT) - #define MATH_DEST_SEQINLEN (0x08 << MATH_DEST_SHIFT) - #define MATH_DEST_SEQOUTLEN (0x09 << MATH_DEST_SHIFT) - #define MATH_DEST_VARSEQINLEN (0x0a << MATH_DEST_SHIFT) -@@ -1629,4 +1656,31 @@ struct sec4_sg_entry { - /* Frame Descriptor Command for Replacement Job Descriptor */ - #define FD_CMD_REPLACE_JOB_DESC 0x20000000 - -+/* CHA Control Register bits */ -+#define CCTRL_RESET_CHA_ALL 0x1 -+#define CCTRL_RESET_CHA_AESA 0x2 -+#define CCTRL_RESET_CHA_DESA 0x4 -+#define CCTRL_RESET_CHA_AFHA 0x8 -+#define CCTRL_RESET_CHA_KFHA 0x10 -+#define CCTRL_RESET_CHA_SF8A 0x20 -+#define CCTRL_RESET_CHA_PKHA 0x40 -+#define CCTRL_RESET_CHA_MDHA 0x80 -+#define CCTRL_RESET_CHA_CRCA 0x100 -+#define CCTRL_RESET_CHA_RNG 0x200 -+#define CCTRL_RESET_CHA_SF9A 0x400 -+#define CCTRL_RESET_CHA_ZUCE 0x800 -+#define CCTRL_RESET_CHA_ZUCA 0x1000 -+#define CCTRL_UNLOAD_PK_A0 0x10000 -+#define CCTRL_UNLOAD_PK_A1 0x20000 -+#define CCTRL_UNLOAD_PK_A2 0x40000 -+#define CCTRL_UNLOAD_PK_A3 0x80000 -+#define CCTRL_UNLOAD_PK_B0 0x100000 -+#define CCTRL_UNLOAD_PK_B1 0x200000 -+#define CCTRL_UNLOAD_PK_B2 0x400000 -+#define CCTRL_UNLOAD_PK_B3 0x800000 -+#define CCTRL_UNLOAD_PK_N 0x1000000 -+#define CCTRL_UNLOAD_PK_A 0x4000000 -+#define CCTRL_UNLOAD_PK_B 0x8000000 -+#define CCTRL_UNLOAD_SBOX 0x10000000 -+ - #endif /* DESC_H */ ---- a/drivers/crypto/caam/desc_constr.h -+++ b/drivers/crypto/caam/desc_constr.h -@@ -4,6 +4,9 @@ - * Copyright 2008-2012 Freescale Semiconductor, Inc. - */ - -+#ifndef DESC_CONSTR_H -+#define DESC_CONSTR_H -+ - #include "desc.h" - #include "regs.h" - -@@ -33,38 +36,39 @@ - - extern bool caam_little_end; - --static inline int desc_len(u32 *desc) -+static inline int desc_len(u32 * const desc) - { - return caam32_to_cpu(*desc) & HDR_DESCLEN_MASK; - } - --static inline int desc_bytes(void *desc) -+static inline int desc_bytes(void * const desc) - { - return desc_len(desc) * CAAM_CMD_SZ; - } - --static inline u32 *desc_end(u32 *desc) -+static inline u32 *desc_end(u32 * const desc) - { - return desc + desc_len(desc); - } - --static inline void *sh_desc_pdb(u32 *desc) -+static inline void *sh_desc_pdb(u32 * const desc) - { - return desc + 1; - } - --static inline void init_desc(u32 *desc, u32 options) -+static inline void init_desc(u32 * const desc, u32 options) - { - *desc = cpu_to_caam32((options | HDR_ONE) + 1); - } - --static inline void init_sh_desc(u32 *desc, u32 options) -+static inline void init_sh_desc(u32 * const desc, u32 options) - { - PRINT_POS; - init_desc(desc, CMD_SHARED_DESC_HDR | options); - } - --static inline void init_sh_desc_pdb(u32 *desc, u32 options, size_t pdb_bytes) -+static inline void init_sh_desc_pdb(u32 * const desc, u32 options, -+ size_t pdb_bytes) - { - u32 pdb_len = (pdb_bytes + CAAM_CMD_SZ - 1) / CAAM_CMD_SZ; - -@@ -72,19 +76,20 @@ static inline void init_sh_desc_pdb(u32 - options); - } - --static inline void init_job_desc(u32 *desc, u32 options) -+static inline void init_job_desc(u32 * const desc, u32 options) - { - init_desc(desc, CMD_DESC_HDR | options); - } - --static inline void init_job_desc_pdb(u32 *desc, u32 options, size_t pdb_bytes) -+static inline void init_job_desc_pdb(u32 * const desc, u32 options, -+ size_t pdb_bytes) - { - u32 pdb_len = (pdb_bytes + CAAM_CMD_SZ - 1) / CAAM_CMD_SZ; - - init_job_desc(desc, (((pdb_len + 1) << HDR_START_IDX_SHIFT)) | options); - } - --static inline void append_ptr(u32 *desc, dma_addr_t ptr) -+static inline void append_ptr(u32 * const desc, dma_addr_t ptr) - { - dma_addr_t *offset = (dma_addr_t *)desc_end(desc); - -@@ -94,8 +99,8 @@ static inline void append_ptr(u32 *desc, - CAAM_PTR_SZ / CAAM_CMD_SZ); - } - --static inline void init_job_desc_shared(u32 *desc, dma_addr_t ptr, int len, -- u32 options) -+static inline void init_job_desc_shared(u32 * const desc, dma_addr_t ptr, -+ int len, u32 options) - { - PRINT_POS; - init_job_desc(desc, HDR_SHARED | options | -@@ -103,7 +108,7 @@ static inline void init_job_desc_shared( - append_ptr(desc, ptr); - } - --static inline void append_data(u32 *desc, void *data, int len) -+static inline void append_data(u32 * const desc, const void *data, int len) - { - u32 *offset = desc_end(desc); - -@@ -114,7 +119,7 @@ static inline void append_data(u32 *desc - (len + CAAM_CMD_SZ - 1) / CAAM_CMD_SZ); - } - --static inline void append_cmd(u32 *desc, u32 command) -+static inline void append_cmd(u32 * const desc, u32 command) - { - u32 *cmd = desc_end(desc); - -@@ -125,7 +130,7 @@ static inline void append_cmd(u32 *desc, - - #define append_u32 append_cmd - --static inline void append_u64(u32 *desc, u64 data) -+static inline void append_u64(u32 * const desc, u64 data) - { - u32 *offset = desc_end(desc); - -@@ -142,14 +147,14 @@ static inline void append_u64(u32 *desc, - } - - /* Write command without affecting header, and return pointer to next word */ --static inline u32 *write_cmd(u32 *desc, u32 command) -+static inline u32 *write_cmd(u32 * const desc, u32 command) - { - *desc = cpu_to_caam32(command); - - return desc + 1; - } - --static inline void append_cmd_ptr(u32 *desc, dma_addr_t ptr, int len, -+static inline void append_cmd_ptr(u32 * const desc, dma_addr_t ptr, int len, - u32 command) - { - append_cmd(desc, command | len); -@@ -157,7 +162,7 @@ static inline void append_cmd_ptr(u32 *d - } - - /* Write length after pointer, rather than inside command */ --static inline void append_cmd_ptr_extlen(u32 *desc, dma_addr_t ptr, -+static inline void append_cmd_ptr_extlen(u32 * const desc, dma_addr_t ptr, - unsigned int len, u32 command) - { - append_cmd(desc, command); -@@ -166,7 +171,7 @@ static inline void append_cmd_ptr_extlen - append_cmd(desc, len); - } - --static inline void append_cmd_data(u32 *desc, void *data, int len, -+static inline void append_cmd_data(u32 * const desc, const void *data, int len, - u32 command) - { - append_cmd(desc, command | IMMEDIATE | len); -@@ -174,7 +179,7 @@ static inline void append_cmd_data(u32 * - } - - #define APPEND_CMD_RET(cmd, op) \ --static inline u32 *append_##cmd(u32 *desc, u32 options) \ -+static inline u32 *append_##cmd(u32 * const desc, u32 options) \ - { \ - u32 *cmd = desc_end(desc); \ - PRINT_POS; \ -@@ -183,14 +188,15 @@ static inline u32 *append_##cmd(u32 *des - } - APPEND_CMD_RET(jump, JUMP) - APPEND_CMD_RET(move, MOVE) -+APPEND_CMD_RET(moveb, MOVEB) - --static inline void set_jump_tgt_here(u32 *desc, u32 *jump_cmd) -+static inline void set_jump_tgt_here(u32 * const desc, u32 *jump_cmd) - { - *jump_cmd = cpu_to_caam32(caam32_to_cpu(*jump_cmd) | - (desc_len(desc) - (jump_cmd - desc))); - } - --static inline void set_move_tgt_here(u32 *desc, u32 *move_cmd) -+static inline void set_move_tgt_here(u32 * const desc, u32 *move_cmd) - { - u32 val = caam32_to_cpu(*move_cmd); - -@@ -200,7 +206,7 @@ static inline void set_move_tgt_here(u32 - } - - #define APPEND_CMD(cmd, op) \ --static inline void append_##cmd(u32 *desc, u32 options) \ -+static inline void append_##cmd(u32 * const desc, u32 options) \ - { \ - PRINT_POS; \ - append_cmd(desc, CMD_##op | options); \ -@@ -208,7 +214,8 @@ static inline void append_##cmd(u32 *des - APPEND_CMD(operation, OPERATION) - - #define APPEND_CMD_LEN(cmd, op) \ --static inline void append_##cmd(u32 *desc, unsigned int len, u32 options) \ -+static inline void append_##cmd(u32 * const desc, unsigned int len, \ -+ u32 options) \ - { \ - PRINT_POS; \ - append_cmd(desc, CMD_##op | len | options); \ -@@ -220,8 +227,8 @@ APPEND_CMD_LEN(seq_fifo_load, SEQ_FIFO_L - APPEND_CMD_LEN(seq_fifo_store, SEQ_FIFO_STORE) - - #define APPEND_CMD_PTR(cmd, op) \ --static inline void append_##cmd(u32 *desc, dma_addr_t ptr, unsigned int len, \ -- u32 options) \ -+static inline void append_##cmd(u32 * const desc, dma_addr_t ptr, \ -+ unsigned int len, u32 options) \ - { \ - PRINT_POS; \ - append_cmd_ptr(desc, ptr, len, CMD_##op | options); \ -@@ -231,8 +238,8 @@ APPEND_CMD_PTR(load, LOAD) - APPEND_CMD_PTR(fifo_load, FIFO_LOAD) - APPEND_CMD_PTR(fifo_store, FIFO_STORE) - --static inline void append_store(u32 *desc, dma_addr_t ptr, unsigned int len, -- u32 options) -+static inline void append_store(u32 * const desc, dma_addr_t ptr, -+ unsigned int len, u32 options) - { - u32 cmd_src; - -@@ -249,7 +256,8 @@ static inline void append_store(u32 *des - } - - #define APPEND_SEQ_PTR_INTLEN(cmd, op) \ --static inline void append_seq_##cmd##_ptr_intlen(u32 *desc, dma_addr_t ptr, \ -+static inline void append_seq_##cmd##_ptr_intlen(u32 * const desc, \ -+ dma_addr_t ptr, \ - unsigned int len, \ - u32 options) \ - { \ -@@ -263,7 +271,7 @@ APPEND_SEQ_PTR_INTLEN(in, IN) - APPEND_SEQ_PTR_INTLEN(out, OUT) - - #define APPEND_CMD_PTR_TO_IMM(cmd, op) \ --static inline void append_##cmd##_as_imm(u32 *desc, void *data, \ -+static inline void append_##cmd##_as_imm(u32 * const desc, const void *data, \ - unsigned int len, u32 options) \ - { \ - PRINT_POS; \ -@@ -273,7 +281,7 @@ APPEND_CMD_PTR_TO_IMM(load, LOAD); - APPEND_CMD_PTR_TO_IMM(fifo_load, FIFO_LOAD); - - #define APPEND_CMD_PTR_EXTLEN(cmd, op) \ --static inline void append_##cmd##_extlen(u32 *desc, dma_addr_t ptr, \ -+static inline void append_##cmd##_extlen(u32 * const desc, dma_addr_t ptr, \ - unsigned int len, u32 options) \ - { \ - PRINT_POS; \ -@@ -287,7 +295,7 @@ APPEND_CMD_PTR_EXTLEN(seq_out_ptr, SEQ_O - * the size of its type - */ - #define APPEND_CMD_PTR_LEN(cmd, op, type) \ --static inline void append_##cmd(u32 *desc, dma_addr_t ptr, \ -+static inline void append_##cmd(u32 * const desc, dma_addr_t ptr, \ - type len, u32 options) \ - { \ - PRINT_POS; \ -@@ -304,7 +312,7 @@ APPEND_CMD_PTR_LEN(seq_out_ptr, SEQ_OUT_ - * from length of immediate data provided, e.g., split keys - */ - #define APPEND_CMD_PTR_TO_IMM2(cmd, op) \ --static inline void append_##cmd##_as_imm(u32 *desc, void *data, \ -+static inline void append_##cmd##_as_imm(u32 * const desc, const void *data, \ - unsigned int data_len, \ - unsigned int len, u32 options) \ - { \ -@@ -315,7 +323,7 @@ static inline void append_##cmd##_as_imm - APPEND_CMD_PTR_TO_IMM2(key, KEY); - - #define APPEND_CMD_RAW_IMM(cmd, op, type) \ --static inline void append_##cmd##_imm_##type(u32 *desc, type immediate, \ -+static inline void append_##cmd##_imm_##type(u32 * const desc, type immediate, \ - u32 options) \ - { \ - PRINT_POS; \ -@@ -426,3 +434,107 @@ do { \ - APPEND_MATH_IMM_u64(LSHIFT, desc, dest, src0, src1, data) - #define append_math_rshift_imm_u64(desc, dest, src0, src1, data) \ - APPEND_MATH_IMM_u64(RSHIFT, desc, dest, src0, src1, data) -+ -+/** -+ * struct alginfo - Container for algorithm details -+ * @algtype: algorithm selector; for valid values, see documentation of the -+ * functions where it is used. -+ * @keylen: length of the provided algorithm key, in bytes -+ * @keylen_pad: padded length of the provided algorithm key, in bytes -+ * @key: address where algorithm key resides; virtual address if key_inline -+ * is true, dma (bus) address if key_inline is false. -+ * @key_inline: true - key can be inlined in the descriptor; false - key is -+ * referenced by the descriptor -+ */ -+struct alginfo { -+ u32 algtype; -+ unsigned int keylen; -+ unsigned int keylen_pad; -+ union { -+ dma_addr_t key_dma; -+ const void *key_virt; -+ }; -+ bool key_inline; -+}; -+ -+/** -+ * desc_inline_query() - Provide indications on which data items can be inlined -+ * and which shall be referenced in a shared descriptor. -+ * @sd_base_len: Shared descriptor base length - bytes consumed by the commands, -+ * excluding the data items to be inlined (or corresponding -+ * pointer if an item is not inlined). Each cnstr_* function that -+ * generates descriptors should have a define mentioning -+ * corresponding length. -+ * @jd_len: Maximum length of the job descriptor(s) that will be used -+ * together with the shared descriptor. -+ * @data_len: Array of lengths of the data items trying to be inlined -+ * @inl_mask: 32bit mask with bit x = 1 if data item x can be inlined, 0 -+ * otherwise. -+ * @count: Number of data items (size of @data_len array); must be <= 32 -+ * -+ * Return: 0 if data can be inlined / referenced, negative value if not. If 0, -+ * check @inl_mask for details. -+ */ -+static inline int desc_inline_query(unsigned int sd_base_len, -+ unsigned int jd_len, unsigned int *data_len, -+ u32 *inl_mask, unsigned int count) -+{ -+ int rem_bytes = (int)(CAAM_DESC_BYTES_MAX - sd_base_len - jd_len); -+ unsigned int i; -+ -+ *inl_mask = 0; -+ for (i = 0; (i < count) && (rem_bytes > 0); i++) { -+ if (rem_bytes - (int)(data_len[i] + -+ (count - i - 1) * CAAM_PTR_SZ) >= 0) { -+ rem_bytes -= data_len[i]; -+ *inl_mask |= (1 << i); -+ } else { -+ rem_bytes -= CAAM_PTR_SZ; -+ } -+ } -+ -+ return (rem_bytes >= 0) ? 0 : -1; -+} -+ -+/** -+ * append_proto_dkp - Derived Key Protocol (DKP): key -> split key -+ * @desc: pointer to buffer used for descriptor construction -+ * @adata: pointer to authentication transform definitions. -+ * keylen should be the length of initial key, while keylen_pad -+ * the length of the derived (split) key. -+ * Valid algorithm values - one of OP_ALG_ALGSEL_{MD5, SHA1, SHA224, -+ * SHA256, SHA384, SHA512}. -+ */ -+static inline void append_proto_dkp(u32 * const desc, struct alginfo *adata) -+{ -+ u32 protid; -+ -+ /* -+ * Quick & dirty translation from OP_ALG_ALGSEL_{MD5, SHA*} -+ * to OP_PCLID_DKP_{MD5, SHA*} -+ */ -+ protid = (adata->algtype & OP_ALG_ALGSEL_SUBMASK) | -+ (0x20 << OP_ALG_ALGSEL_SHIFT); -+ -+ if (adata->key_inline) { -+ int words; -+ -+ append_operation(desc, OP_TYPE_UNI_PROTOCOL | protid | -+ OP_PCL_DKP_SRC_IMM | OP_PCL_DKP_DST_IMM | -+ adata->keylen); -+ append_data(desc, adata->key_virt, adata->keylen); -+ -+ /* Reserve space in descriptor buffer for the derived key */ -+ words = (ALIGN(adata->keylen_pad, CAAM_CMD_SZ) - -+ ALIGN(adata->keylen, CAAM_CMD_SZ)) / CAAM_CMD_SZ; -+ if (words) -+ (*desc) = cpu_to_caam32(caam32_to_cpu(*desc) + words); -+ } else { -+ append_operation(desc, OP_TYPE_UNI_PROTOCOL | protid | -+ OP_PCL_DKP_SRC_PTR | OP_PCL_DKP_DST_PTR | -+ adata->keylen); -+ append_ptr(desc, adata->key_dma); -+ } -+} -+ -+#endif /* DESC_CONSTR_H */ ---- /dev/null -+++ b/drivers/crypto/caam/dpseci.c -@@ -0,0 +1,858 @@ -+/* -+ * Copyright 2013-2016 Freescale Semiconductor Inc. -+ * Copyright 2017 NXP -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the names of the above-listed copyright holders nor the -+ * names of any contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE -+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -+ * POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#include <linux/fsl/mc.h> -+#include "../../../drivers/staging/fsl-mc/include/dpopr.h" -+#include "dpseci.h" -+#include "dpseci_cmd.h" -+ -+/** -+ * dpseci_open() - Open a control session for the specified object -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @dpseci_id: DPSECI unique ID -+ * @token: Returned token; use in subsequent API calls -+ * -+ * This function can be used to open a control session for an already created -+ * object; an object may have been declared in the DPL or by calling the -+ * dpseci_create() function. -+ * This function returns a unique authentication token, associated with the -+ * specific object ID and the specific MC portal; this token must be used in all -+ * subsequent commands for this specific object. -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_open(struct fsl_mc_io *mc_io, u32 cmd_flags, int dpseci_id, -+ u16 *token) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpseci_cmd_open *cmd_params; -+ int err; -+ -+ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_OPEN, -+ cmd_flags, -+ 0); -+ cmd_params = (struct dpseci_cmd_open *)cmd.params; -+ cmd_params->dpseci_id = cpu_to_le32(dpseci_id); -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ *token = mc_cmd_hdr_read_token(&cmd); -+ -+ return 0; -+} -+ -+/** -+ * dpseci_close() - Close the control session of the object -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPSECI object -+ * -+ * After this function is called, no further operations are allowed on the -+ * object without opening a new control session. -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_close(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ -+ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_CLOSE, -+ cmd_flags, -+ token); -+ return mc_send_command(mc_io, &cmd); -+} -+ -+/** -+ * dpseci_create() - Create the DPSECI object -+ * @mc_io: Pointer to MC portal's I/O object -+ * @dprc_token: Parent container token; '0' for default container -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @cfg: Configuration structure -+ * @obj_id: returned object id -+ * -+ * Create the DPSECI object, allocate required resources and perform required -+ * initialization. -+ * -+ * The object can be created either by declaring it in the DPL file, or by -+ * calling this function. -+ * -+ * The function accepts an authentication token of a parent container that this -+ * object should be assigned to. The token can be '0' so the object will be -+ * assigned to the default container. -+ * The newly created object can be opened with the returned object id and using -+ * the container's associated tokens and MC portals. -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_create(struct fsl_mc_io *mc_io, u16 dprc_token, u32 cmd_flags, -+ const struct dpseci_cfg *cfg, u32 *obj_id) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpseci_cmd_create *cmd_params; -+ int i, err; -+ -+ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_CREATE, -+ cmd_flags, -+ dprc_token); -+ cmd_params = (struct dpseci_cmd_create *)cmd.params; -+ for (i = 0; i < 8; i++) -+ cmd_params->priorities[i] = cfg->priorities[i]; -+ cmd_params->num_tx_queues = cfg->num_tx_queues; -+ cmd_params->num_rx_queues = cfg->num_rx_queues; -+ cmd_params->options = cpu_to_le32(cfg->options); -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ *obj_id = mc_cmd_read_object_id(&cmd); -+ -+ return 0; -+} -+ -+/** -+ * dpseci_destroy() - Destroy the DPSECI object and release all its resources -+ * @mc_io: Pointer to MC portal's I/O object -+ * @dprc_token: Parent container token; '0' for default container -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @object_id: The object id; it must be a valid id within the container that -+ * created this object -+ * -+ * The function accepts the authentication token of the parent container that -+ * created the object (not the one that currently owns the object). The object -+ * is searched within parent using the provided 'object_id'. -+ * All tokens to the object must be closed before calling destroy. -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_destroy(struct fsl_mc_io *mc_io, u16 dprc_token, u32 cmd_flags, -+ u32 object_id) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpseci_cmd_destroy *cmd_params; -+ -+ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_DESTROY, -+ cmd_flags, -+ dprc_token); -+ cmd_params = (struct dpseci_cmd_destroy *)cmd.params; -+ cmd_params->object_id = cpu_to_le32(object_id); -+ -+ return mc_send_command(mc_io, &cmd); -+} -+ -+/** -+ * dpseci_enable() - Enable the DPSECI, allow sending and receiving frames -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPSECI object -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ -+ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_ENABLE, -+ cmd_flags, -+ token); -+ return mc_send_command(mc_io, &cmd); -+} -+ -+/** -+ * dpseci_disable() - Disable the DPSECI, stop sending and receiving frames -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPSECI object -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_disable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ -+ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_DISABLE, -+ cmd_flags, -+ token); -+ -+ return mc_send_command(mc_io, &cmd); -+} -+ -+/** -+ * dpseci_is_enabled() - Check if the DPSECI is enabled. -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPSECI object -+ * @en: Returns '1' if object is enabled; '0' otherwise -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_is_enabled(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ int *en) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpseci_rsp_is_enabled *rsp_params; -+ int err; -+ -+ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_IS_ENABLED, -+ cmd_flags, -+ token); -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ rsp_params = (struct dpseci_rsp_is_enabled *)cmd.params; -+ *en = le32_to_cpu(rsp_params->is_enabled); -+ -+ return 0; -+} -+ -+/** -+ * dpseci_reset() - Reset the DPSECI, returns the object to initial state. -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPSECI object -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_reset(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ -+ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_RESET, -+ cmd_flags, -+ token); -+ -+ return mc_send_command(mc_io, &cmd); -+} -+ -+/** -+ * dpseci_get_irq_enable() - Get overall interrupt state -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPSECI object -+ * @irq_index: The interrupt index to configure -+ * @en: Returned Interrupt state - enable = 1, disable = 0 -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_get_irq_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ u8 irq_index, u8 *en) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpseci_cmd_irq_enable *cmd_params; -+ struct dpseci_rsp_get_irq_enable *rsp_params; -+ int err; -+ -+ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_IRQ_ENABLE, -+ cmd_flags, -+ token); -+ cmd_params = (struct dpseci_cmd_irq_enable *)cmd.params; -+ cmd_params->irq_index = irq_index; -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ rsp_params = (struct dpseci_rsp_get_irq_enable *)cmd.params; -+ *en = rsp_params->enable_state; -+ -+ return 0; -+} -+ -+/** -+ * dpseci_set_irq_enable() - Set overall interrupt state. -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPSECI object -+ * @irq_index: The interrupt index to configure -+ * @en: Interrupt state - enable = 1, disable = 0 -+ * -+ * Allows GPP software to control when interrupts are generated. -+ * Each interrupt can have up to 32 causes. The enable/disable control's the -+ * overall interrupt state. If the interrupt is disabled no causes will cause -+ * an interrupt. -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_set_irq_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ u8 irq_index, u8 en) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpseci_cmd_irq_enable *cmd_params; -+ -+ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_SET_IRQ_ENABLE, -+ cmd_flags, -+ token); -+ cmd_params = (struct dpseci_cmd_irq_enable *)cmd.params; -+ cmd_params->irq_index = irq_index; -+ cmd_params->enable_state = en; -+ -+ return mc_send_command(mc_io, &cmd); -+} -+ -+/** -+ * dpseci_get_irq_mask() - Get interrupt mask. -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPSECI object -+ * @irq_index: The interrupt index to configure -+ * @mask: Returned event mask to trigger interrupt -+ * -+ * Every interrupt can have up to 32 causes and the interrupt model supports -+ * masking/unmasking each cause independently. -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_get_irq_mask(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ u8 irq_index, u32 *mask) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpseci_cmd_irq_mask *cmd_params; -+ int err; -+ -+ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_IRQ_MASK, -+ cmd_flags, -+ token); -+ cmd_params = (struct dpseci_cmd_irq_mask *)cmd.params; -+ cmd_params->irq_index = irq_index; -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ *mask = le32_to_cpu(cmd_params->mask); -+ -+ return 0; -+} -+ -+/** -+ * dpseci_set_irq_mask() - Set interrupt mask. -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPSECI object -+ * @irq_index: The interrupt index to configure -+ * @mask: event mask to trigger interrupt; -+ * each bit: -+ * 0 = ignore event -+ * 1 = consider event for asserting IRQ -+ * -+ * Every interrupt can have up to 32 causes and the interrupt model supports -+ * masking/unmasking each cause independently -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_set_irq_mask(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ u8 irq_index, u32 mask) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpseci_cmd_irq_mask *cmd_params; -+ -+ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_SET_IRQ_MASK, -+ cmd_flags, -+ token); -+ cmd_params = (struct dpseci_cmd_irq_mask *)cmd.params; -+ cmd_params->mask = cpu_to_le32(mask); -+ cmd_params->irq_index = irq_index; -+ -+ return mc_send_command(mc_io, &cmd); -+} -+ -+/** -+ * dpseci_get_irq_status() - Get the current status of any pending interrupts -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPSECI object -+ * @irq_index: The interrupt index to configure -+ * @status: Returned interrupts status - one bit per cause: -+ * 0 = no interrupt pending -+ * 1 = interrupt pending -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_get_irq_status(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ u8 irq_index, u32 *status) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpseci_cmd_irq_status *cmd_params; -+ int err; -+ -+ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_IRQ_STATUS, -+ cmd_flags, -+ token); -+ cmd_params = (struct dpseci_cmd_irq_status *)cmd.params; -+ cmd_params->status = cpu_to_le32(*status); -+ cmd_params->irq_index = irq_index; -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ *status = le32_to_cpu(cmd_params->status); -+ -+ return 0; -+} -+ -+/** -+ * dpseci_clear_irq_status() - Clear a pending interrupt's status -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPSECI object -+ * @irq_index: The interrupt index to configure -+ * @status: bits to clear (W1C) - one bit per cause: -+ * 0 = don't change -+ * 1 = clear status bit -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_clear_irq_status(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ u8 irq_index, u32 status) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpseci_cmd_irq_status *cmd_params; -+ -+ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_CLEAR_IRQ_STATUS, -+ cmd_flags, -+ token); -+ cmd_params = (struct dpseci_cmd_irq_status *)cmd.params; -+ cmd_params->status = cpu_to_le32(status); -+ cmd_params->irq_index = irq_index; -+ -+ return mc_send_command(mc_io, &cmd); -+} -+ -+/** -+ * dpseci_get_attributes() - Retrieve DPSECI attributes -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPSECI object -+ * @attr: Returned object's attributes -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_get_attributes(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ struct dpseci_attr *attr) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpseci_rsp_get_attributes *rsp_params; -+ int err; -+ -+ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_ATTR, -+ cmd_flags, -+ token); -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ rsp_params = (struct dpseci_rsp_get_attributes *)cmd.params; -+ attr->id = le32_to_cpu(rsp_params->id); -+ attr->num_tx_queues = rsp_params->num_tx_queues; -+ attr->num_rx_queues = rsp_params->num_rx_queues; -+ attr->options = le32_to_cpu(rsp_params->options); -+ -+ return 0; -+} -+ -+/** -+ * dpseci_set_rx_queue() - Set Rx queue configuration -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPSECI object -+ * @queue: Select the queue relative to number of priorities configured at -+ * DPSECI creation; use DPSECI_ALL_QUEUES to configure all -+ * Rx queues identically. -+ * @cfg: Rx queue configuration -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ u8 queue, const struct dpseci_rx_queue_cfg *cfg) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpseci_cmd_queue *cmd_params; -+ -+ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_SET_RX_QUEUE, -+ cmd_flags, -+ token); -+ cmd_params = (struct dpseci_cmd_queue *)cmd.params; -+ cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id); -+ cmd_params->priority = cfg->dest_cfg.priority; -+ cmd_params->queue = queue; -+ cmd_params->dest_type = cfg->dest_cfg.dest_type; -+ cmd_params->user_ctx = cpu_to_le64(cfg->user_ctx); -+ cmd_params->options = cpu_to_le32(cfg->options); -+ cmd_params->order_preservation_en = -+ cpu_to_le32(cfg->order_preservation_en); -+ -+ return mc_send_command(mc_io, &cmd); -+} -+ -+/** -+ * dpseci_get_rx_queue() - Retrieve Rx queue attributes -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPSECI object -+ * @queue: Select the queue relative to number of priorities configured at -+ * DPSECI creation -+ * @attr: Returned Rx queue attributes -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ u8 queue, struct dpseci_rx_queue_attr *attr) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpseci_cmd_queue *cmd_params; -+ int err; -+ -+ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_RX_QUEUE, -+ cmd_flags, -+ token); -+ cmd_params = (struct dpseci_cmd_queue *)cmd.params; -+ cmd_params->queue = queue; -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ attr->dest_cfg.dest_id = le32_to_cpu(cmd_params->dest_id); -+ attr->dest_cfg.priority = cmd_params->priority; -+ attr->dest_cfg.dest_type = cmd_params->dest_type; -+ attr->user_ctx = le64_to_cpu(cmd_params->user_ctx); -+ attr->fqid = le32_to_cpu(cmd_params->fqid); -+ attr->order_preservation_en = -+ le32_to_cpu(cmd_params->order_preservation_en); -+ -+ return 0; -+} -+ -+/** -+ * dpseci_get_tx_queue() - Retrieve Tx queue attributes -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPSECI object -+ * @queue: Select the queue relative to number of priorities configured at -+ * DPSECI creation -+ * @attr: Returned Tx queue attributes -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ u8 queue, struct dpseci_tx_queue_attr *attr) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpseci_cmd_queue *cmd_params; -+ struct dpseci_rsp_get_tx_queue *rsp_params; -+ int err; -+ -+ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_TX_QUEUE, -+ cmd_flags, -+ token); -+ cmd_params = (struct dpseci_cmd_queue *)cmd.params; -+ cmd_params->queue = queue; -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ rsp_params = (struct dpseci_rsp_get_tx_queue *)cmd.params; -+ attr->fqid = le32_to_cpu(rsp_params->fqid); -+ attr->priority = rsp_params->priority; -+ -+ return 0; -+} -+ -+/** -+ * dpseci_get_sec_attr() - Retrieve SEC accelerator attributes -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPSECI object -+ * @attr: Returned SEC attributes -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_get_sec_attr(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ struct dpseci_sec_attr *attr) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpseci_rsp_get_sec_attr *rsp_params; -+ int err; -+ -+ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_SEC_ATTR, -+ cmd_flags, -+ token); -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ rsp_params = (struct dpseci_rsp_get_sec_attr *)cmd.params; -+ attr->ip_id = le16_to_cpu(rsp_params->ip_id); -+ attr->major_rev = rsp_params->major_rev; -+ attr->minor_rev = rsp_params->minor_rev; -+ attr->era = rsp_params->era; -+ attr->deco_num = rsp_params->deco_num; -+ attr->zuc_auth_acc_num = rsp_params->zuc_auth_acc_num; -+ attr->zuc_enc_acc_num = rsp_params->zuc_enc_acc_num; -+ attr->snow_f8_acc_num = rsp_params->snow_f8_acc_num; -+ attr->snow_f9_acc_num = rsp_params->snow_f9_acc_num; -+ attr->crc_acc_num = rsp_params->crc_acc_num; -+ attr->pk_acc_num = rsp_params->pk_acc_num; -+ attr->kasumi_acc_num = rsp_params->kasumi_acc_num; -+ attr->rng_acc_num = rsp_params->rng_acc_num; -+ attr->md_acc_num = rsp_params->md_acc_num; -+ attr->arc4_acc_num = rsp_params->arc4_acc_num; -+ attr->des_acc_num = rsp_params->des_acc_num; -+ attr->aes_acc_num = rsp_params->aes_acc_num; -+ -+ return 0; -+} -+ -+/** -+ * dpseci_get_sec_counters() - Retrieve SEC accelerator counters -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPSECI object -+ * @counters: Returned SEC counters -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_get_sec_counters(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ struct dpseci_sec_counters *counters) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpseci_rsp_get_sec_counters *rsp_params; -+ int err; -+ -+ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_SEC_COUNTERS, -+ cmd_flags, -+ token); -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ rsp_params = (struct dpseci_rsp_get_sec_counters *)cmd.params; -+ counters->dequeued_requests = -+ le64_to_cpu(rsp_params->dequeued_requests); -+ counters->ob_enc_requests = le64_to_cpu(rsp_params->ob_enc_requests); -+ counters->ib_dec_requests = le64_to_cpu(rsp_params->ib_dec_requests); -+ counters->ob_enc_bytes = le64_to_cpu(rsp_params->ob_enc_bytes); -+ counters->ob_prot_bytes = le64_to_cpu(rsp_params->ob_prot_bytes); -+ counters->ib_dec_bytes = le64_to_cpu(rsp_params->ib_dec_bytes); -+ counters->ib_valid_bytes = le64_to_cpu(rsp_params->ib_valid_bytes); -+ -+ return 0; -+} -+ -+/** -+ * dpseci_get_api_version() - Get Data Path SEC Interface API version -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @major_ver: Major version of data path sec API -+ * @minor_ver: Minor version of data path sec API -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_get_api_version(struct fsl_mc_io *mc_io, u32 cmd_flags, -+ u16 *major_ver, u16 *minor_ver) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpseci_rsp_get_api_version *rsp_params; -+ int err; -+ -+ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_API_VERSION, -+ cmd_flags, 0); -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ rsp_params = (struct dpseci_rsp_get_api_version *)cmd.params; -+ *major_ver = le16_to_cpu(rsp_params->major); -+ *minor_ver = le16_to_cpu(rsp_params->minor); -+ -+ return 0; -+} -+ -+/** -+ * dpseci_set_opr() - Set Order Restoration configuration -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPSECI object -+ * @index: The queue index -+ * @options: Configuration mode options; can be OPR_OPT_CREATE or -+ * OPR_OPT_RETIRE -+ * @cfg: Configuration options for the OPR -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_set_opr(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, u8 index, -+ u8 options, struct opr_cfg *cfg) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpseci_cmd_opr *cmd_params; -+ -+ cmd.header = mc_encode_cmd_header( -+ DPSECI_CMDID_SET_OPR, -+ cmd_flags, -+ token); -+ cmd_params = (struct dpseci_cmd_opr *)cmd.params; -+ cmd_params->index = index; -+ cmd_params->options = options; -+ cmd_params->oloe = cfg->oloe; -+ cmd_params->oeane = cfg->oeane; -+ cmd_params->olws = cfg->olws; -+ cmd_params->oa = cfg->oa; -+ cmd_params->oprrws = cfg->oprrws; -+ -+ return mc_send_command(mc_io, &cmd); -+} -+ -+/** -+ * dpseci_get_opr() - Retrieve Order Restoration config and query -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPSECI object -+ * @index: The queue index -+ * @cfg: Returned OPR configuration -+ * @qry: Returned OPR query -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_get_opr(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, u8 index, -+ struct opr_cfg *cfg, struct opr_qry *qry) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpseci_cmd_opr *cmd_params; -+ struct dpseci_rsp_get_opr *rsp_params; -+ int err; -+ -+ cmd.header = mc_encode_cmd_header(DPSECI_CMDID_GET_OPR, -+ cmd_flags, -+ token); -+ cmd_params = (struct dpseci_cmd_opr *)cmd.params; -+ cmd_params->index = index; -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ rsp_params = (struct dpseci_rsp_get_opr *)cmd.params; -+ qry->rip = dpseci_get_field(rsp_params->rip_enable, OPR_RIP); -+ qry->enable = dpseci_get_field(rsp_params->rip_enable, OPR_ENABLE); -+ cfg->oloe = rsp_params->oloe; -+ cfg->oeane = rsp_params->oeane; -+ cfg->olws = rsp_params->olws; -+ cfg->oa = rsp_params->oa; -+ cfg->oprrws = rsp_params->oprrws; -+ qry->nesn = le16_to_cpu(rsp_params->nesn); -+ qry->ndsn = le16_to_cpu(rsp_params->ndsn); -+ qry->ea_tseq = le16_to_cpu(rsp_params->ea_tseq); -+ qry->tseq_nlis = dpseci_get_field(rsp_params->tseq_nlis, OPR_TSEQ_NLIS); -+ qry->ea_hseq = le16_to_cpu(rsp_params->ea_hseq); -+ qry->hseq_nlis = dpseci_get_field(rsp_params->tseq_nlis, OPR_HSEQ_NLIS); -+ qry->ea_hptr = le16_to_cpu(rsp_params->ea_hptr); -+ qry->ea_tptr = le16_to_cpu(rsp_params->ea_tptr); -+ qry->opr_vid = le16_to_cpu(rsp_params->opr_vid); -+ qry->opr_id = le16_to_cpu(rsp_params->opr_id); -+ -+ return 0; -+} -+ -+/** -+ * dpseci_set_congestion_notification() - Set congestion group -+ * notification configuration -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPSECI object -+ * @cfg: congestion notification configuration -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_set_congestion_notification(struct fsl_mc_io *mc_io, u32 cmd_flags, -+ u16 token, const struct dpseci_congestion_notification_cfg *cfg) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpseci_cmd_congestion_notification *cmd_params; -+ -+ cmd.header = mc_encode_cmd_header( -+ DPSECI_CMDID_SET_CONGESTION_NOTIFICATION, -+ cmd_flags, -+ token); -+ cmd_params = (struct dpseci_cmd_congestion_notification *)cmd.params; -+ cmd_params->dest_id = cpu_to_le32(cfg->dest_cfg.dest_id); -+ cmd_params->notification_mode = cpu_to_le16(cfg->notification_mode); -+ cmd_params->priority = cfg->dest_cfg.priority; -+ dpseci_set_field(cmd_params->options, CGN_DEST_TYPE, -+ cfg->dest_cfg.dest_type); -+ dpseci_set_field(cmd_params->options, CGN_UNITS, cfg->units); -+ cmd_params->message_iova = cpu_to_le64(cfg->message_iova); -+ cmd_params->message_ctx = cpu_to_le64(cfg->message_ctx); -+ cmd_params->threshold_entry = cpu_to_le32(cfg->threshold_entry); -+ cmd_params->threshold_exit = cpu_to_le32(cfg->threshold_exit); -+ -+ return mc_send_command(mc_io, &cmd); -+} -+ -+/** -+ * dpseci_get_congestion_notification() - Get congestion group notification -+ * configuration -+ * @mc_io: Pointer to MC portal's I/O object -+ * @cmd_flags: Command flags; one or more of 'MC_CMD_FLAG_' -+ * @token: Token of DPSECI object -+ * @cfg: congestion notification configuration -+ * -+ * Return: '0' on success, error code otherwise -+ */ -+int dpseci_get_congestion_notification(struct fsl_mc_io *mc_io, u32 cmd_flags, -+ u16 token, struct dpseci_congestion_notification_cfg *cfg) -+{ -+ struct fsl_mc_command cmd = { 0 }; -+ struct dpseci_cmd_congestion_notification *rsp_params; -+ int err; -+ -+ cmd.header = mc_encode_cmd_header( -+ DPSECI_CMDID_GET_CONGESTION_NOTIFICATION, -+ cmd_flags, -+ token); -+ err = mc_send_command(mc_io, &cmd); -+ if (err) -+ return err; -+ -+ rsp_params = (struct dpseci_cmd_congestion_notification *)cmd.params; -+ cfg->dest_cfg.dest_id = le32_to_cpu(rsp_params->dest_id); -+ cfg->notification_mode = le16_to_cpu(rsp_params->notification_mode); -+ cfg->dest_cfg.priority = rsp_params->priority; -+ cfg->dest_cfg.dest_type = dpseci_get_field(rsp_params->options, -+ CGN_DEST_TYPE); -+ cfg->units = dpseci_get_field(rsp_params->options, CGN_UNITS); -+ cfg->message_iova = le64_to_cpu(rsp_params->message_iova); -+ cfg->message_ctx = le64_to_cpu(rsp_params->message_ctx); -+ cfg->threshold_entry = le32_to_cpu(rsp_params->threshold_entry); -+ cfg->threshold_exit = le32_to_cpu(rsp_params->threshold_exit); -+ -+ return 0; -+} ---- /dev/null -+++ b/drivers/crypto/caam/dpseci.h -@@ -0,0 +1,395 @@ -+/* -+ * Copyright 2013-2016 Freescale Semiconductor Inc. -+ * Copyright 2017 NXP -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the names of the above-listed copyright holders nor the -+ * names of any contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE -+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -+ * POSSIBILITY OF SUCH DAMAGE. -+ */ -+#ifndef _DPSECI_H_ -+#define _DPSECI_H_ -+ -+/* -+ * Data Path SEC Interface API -+ * Contains initialization APIs and runtime control APIs for DPSECI -+ */ -+ -+struct fsl_mc_io; -+struct opr_cfg; -+struct opr_qry; -+ -+/** -+ * General DPSECI macros -+ */ -+ -+/** -+ * Maximum number of Tx/Rx priorities per DPSECI object -+ */ -+#define DPSECI_PRIO_NUM 8 -+ -+/** -+ * All queues considered; see dpseci_set_rx_queue() -+ */ -+#define DPSECI_ALL_QUEUES (u8)(-1) -+ -+int dpseci_open(struct fsl_mc_io *mc_io, u32 cmd_flags, int dpseci_id, -+ u16 *token); -+ -+int dpseci_close(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); -+ -+/** -+ * Enable the Congestion Group support -+ */ -+#define DPSECI_OPT_HAS_CG 0x000020 -+ -+/** -+ * Enable the Order Restoration support -+ */ -+#define DPSECI_OPT_HAS_OPR 0x000040 -+ -+/** -+ * Order Point Records are shared for the entire DPSECI -+ */ -+#define DPSECI_OPT_OPR_SHARED 0x000080 -+ -+/** -+ * struct dpseci_cfg - Structure representing DPSECI configuration -+ * @options: Any combination of the following options: -+ * DPSECI_OPT_HAS_CG -+ * DPSECI_OPT_HAS_OPR -+ * DPSECI_OPT_OPR_SHARED -+ * @num_tx_queues: num of queues towards the SEC -+ * @num_rx_queues: num of queues back from the SEC -+ * @priorities: Priorities for the SEC hardware processing; -+ * each place in the array is the priority of the tx queue -+ * towards the SEC; -+ * valid priorities are configured with values 1-8; -+ */ -+struct dpseci_cfg { -+ u32 options; -+ u8 num_tx_queues; -+ u8 num_rx_queues; -+ u8 priorities[DPSECI_PRIO_NUM]; -+}; -+ -+int dpseci_create(struct fsl_mc_io *mc_io, u16 dprc_token, u32 cmd_flags, -+ const struct dpseci_cfg *cfg, u32 *obj_id); -+ -+int dpseci_destroy(struct fsl_mc_io *mc_io, u16 dprc_token, u32 cmd_flags, -+ u32 object_id); -+ -+int dpseci_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); -+ -+int dpseci_disable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); -+ -+int dpseci_is_enabled(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ int *en); -+ -+int dpseci_reset(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token); -+ -+int dpseci_get_irq_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ u8 irq_index, u8 *en); -+ -+int dpseci_set_irq_enable(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ u8 irq_index, u8 en); -+ -+int dpseci_get_irq_mask(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ u8 irq_index, u32 *mask); -+ -+int dpseci_set_irq_mask(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ u8 irq_index, u32 mask); -+ -+int dpseci_get_irq_status(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ u8 irq_index, u32 *status); -+ -+int dpseci_clear_irq_status(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ u8 irq_index, u32 status); -+ -+/** -+ * struct dpseci_attr - Structure representing DPSECI attributes -+ * @id: DPSECI object ID -+ * @num_tx_queues: number of queues towards the SEC -+ * @num_rx_queues: number of queues back from the SEC -+ * @options: any combination of the following options: -+ * DPSECI_OPT_HAS_CG -+ * DPSECI_OPT_HAS_OPR -+ * DPSECI_OPT_OPR_SHARED -+ */ -+struct dpseci_attr { -+ int id; -+ u8 num_tx_queues; -+ u8 num_rx_queues; -+ u32 options; -+}; -+ -+int dpseci_get_attributes(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ struct dpseci_attr *attr); -+ -+/** -+ * enum dpseci_dest - DPSECI destination types -+ * @DPSECI_DEST_NONE: Unassigned destination; The queue is set in parked mode -+ * and does not generate FQDAN notifications; user is expected to dequeue -+ * from the queue based on polling or other user-defined method -+ * @DPSECI_DEST_DPIO: The queue is set in schedule mode and generates FQDAN -+ * notifications to the specified DPIO; user is expected to dequeue from -+ * the queue only after notification is received -+ * @DPSECI_DEST_DPCON: The queue is set in schedule mode and does not generate -+ * FQDAN notifications, but is connected to the specified DPCON object; -+ * user is expected to dequeue from the DPCON channel -+ */ -+enum dpseci_dest { -+ DPSECI_DEST_NONE = 0, -+ DPSECI_DEST_DPIO, -+ DPSECI_DEST_DPCON -+}; -+ -+/** -+ * struct dpseci_dest_cfg - Structure representing DPSECI destination parameters -+ * @dest_type: Destination type -+ * @dest_id: Either DPIO ID or DPCON ID, depending on the destination type -+ * @priority: Priority selection within the DPIO or DPCON channel; valid values -+ * are 0-1 or 0-7, depending on the number of priorities in that channel; -+ * not relevant for 'DPSECI_DEST_NONE' option -+ */ -+struct dpseci_dest_cfg { -+ enum dpseci_dest dest_type; -+ int dest_id; -+ u8 priority; -+}; -+ -+/** -+ * DPSECI queue modification options -+ */ -+ -+/** -+ * Select to modify the user's context associated with the queue -+ */ -+#define DPSECI_QUEUE_OPT_USER_CTX 0x00000001 -+ -+/** -+ * Select to modify the queue's destination -+ */ -+#define DPSECI_QUEUE_OPT_DEST 0x00000002 -+ -+/** -+ * Select to modify the queue's order preservation -+ */ -+#define DPSECI_QUEUE_OPT_ORDER_PRESERVATION 0x00000004 -+ -+/** -+ * struct dpseci_rx_queue_cfg - DPSECI RX queue configuration -+ * @options: Flags representing the suggested modifications to the queue; -+ * Use any combination of 'DPSECI_QUEUE_OPT_<X>' flags -+ * @order_preservation_en: order preservation configuration for the rx queue -+ * valid only if 'DPSECI_QUEUE_OPT_ORDER_PRESERVATION' is contained in 'options' -+ * @user_ctx: User context value provided in the frame descriptor of each -+ * dequeued frame; valid only if 'DPSECI_QUEUE_OPT_USER_CTX' is contained -+ * in 'options' -+ * @dest_cfg: Queue destination parameters; valid only if -+ * 'DPSECI_QUEUE_OPT_DEST' is contained in 'options' -+ */ -+struct dpseci_rx_queue_cfg { -+ u32 options; -+ int order_preservation_en; -+ u64 user_ctx; -+ struct dpseci_dest_cfg dest_cfg; -+}; -+ -+int dpseci_set_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ u8 queue, const struct dpseci_rx_queue_cfg *cfg); -+ -+/** -+ * struct dpseci_rx_queue_attr - Structure representing attributes of Rx queues -+ * @user_ctx: User context value provided in the frame descriptor of each -+ * dequeued frame -+ * @order_preservation_en: Status of the order preservation configuration on the -+ * queue -+ * @dest_cfg: Queue destination configuration -+ * @fqid: Virtual FQID value to be used for dequeue operations -+ */ -+struct dpseci_rx_queue_attr { -+ u64 user_ctx; -+ int order_preservation_en; -+ struct dpseci_dest_cfg dest_cfg; -+ u32 fqid; -+}; -+ -+int dpseci_get_rx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ u8 queue, struct dpseci_rx_queue_attr *attr); -+ -+/** -+ * struct dpseci_tx_queue_attr - Structure representing attributes of Tx queues -+ * @fqid: Virtual FQID to be used for sending frames to SEC hardware -+ * @priority: SEC hardware processing priority for the queue -+ */ -+struct dpseci_tx_queue_attr { -+ u32 fqid; -+ u8 priority; -+}; -+ -+int dpseci_get_tx_queue(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ u8 queue, struct dpseci_tx_queue_attr *attr); -+ -+/** -+ * struct dpseci_sec_attr - Structure representing attributes of the SEC -+ * hardware accelerator -+ * @ip_id: ID for SEC -+ * @major_rev: Major revision number for SEC -+ * @minor_rev: Minor revision number for SEC -+ * @era: SEC Era -+ * @deco_num: The number of copies of the DECO that are implemented in this -+ * version of SEC -+ * @zuc_auth_acc_num: The number of copies of ZUCA that are implemented in this -+ * version of SEC -+ * @zuc_enc_acc_num: The number of copies of ZUCE that are implemented in this -+ * version of SEC -+ * @snow_f8_acc_num: The number of copies of the SNOW-f8 module that are -+ * implemented in this version of SEC -+ * @snow_f9_acc_num: The number of copies of the SNOW-f9 module that are -+ * implemented in this version of SEC -+ * @crc_acc_num: The number of copies of the CRC module that are implemented in -+ * this version of SEC -+ * @pk_acc_num: The number of copies of the Public Key module that are -+ * implemented in this version of SEC -+ * @kasumi_acc_num: The number of copies of the Kasumi module that are -+ * implemented in this version of SEC -+ * @rng_acc_num: The number of copies of the Random Number Generator that are -+ * implemented in this version of SEC -+ * @md_acc_num: The number of copies of the MDHA (Hashing module) that are -+ * implemented in this version of SEC -+ * @arc4_acc_num: The number of copies of the ARC4 module that are implemented -+ * in this version of SEC -+ * @des_acc_num: The number of copies of the DES module that are implemented in -+ * this version of SEC -+ * @aes_acc_num: The number of copies of the AES module that are implemented in -+ * this version of SEC -+ **/ -+struct dpseci_sec_attr { -+ u16 ip_id; -+ u8 major_rev; -+ u8 minor_rev; -+ u8 era; -+ u8 deco_num; -+ u8 zuc_auth_acc_num; -+ u8 zuc_enc_acc_num; -+ u8 snow_f8_acc_num; -+ u8 snow_f9_acc_num; -+ u8 crc_acc_num; -+ u8 pk_acc_num; -+ u8 kasumi_acc_num; -+ u8 rng_acc_num; -+ u8 md_acc_num; -+ u8 arc4_acc_num; -+ u8 des_acc_num; -+ u8 aes_acc_num; -+}; -+ -+int dpseci_get_sec_attr(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ struct dpseci_sec_attr *attr); -+ -+/** -+ * struct dpseci_sec_counters - Structure representing global SEC counters and -+ * not per dpseci counters -+ * @dequeued_requests: Number of Requests Dequeued -+ * @ob_enc_requests: Number of Outbound Encrypt Requests -+ * @ib_dec_requests: Number of Inbound Decrypt Requests -+ * @ob_enc_bytes: Number of Outbound Bytes Encrypted -+ * @ob_prot_bytes: Number of Outbound Bytes Protected -+ * @ib_dec_bytes: Number of Inbound Bytes Decrypted -+ * @ib_valid_bytes: Number of Inbound Bytes Validated -+ */ -+struct dpseci_sec_counters { -+ u64 dequeued_requests; -+ u64 ob_enc_requests; -+ u64 ib_dec_requests; -+ u64 ob_enc_bytes; -+ u64 ob_prot_bytes; -+ u64 ib_dec_bytes; -+ u64 ib_valid_bytes; -+}; -+ -+int dpseci_get_sec_counters(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, -+ struct dpseci_sec_counters *counters); -+ -+int dpseci_get_api_version(struct fsl_mc_io *mc_io, u32 cmd_flags, -+ u16 *major_ver, u16 *minor_ver); -+ -+int dpseci_set_opr(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, u8 index, -+ u8 options, struct opr_cfg *cfg); -+ -+int dpseci_get_opr(struct fsl_mc_io *mc_io, u32 cmd_flags, u16 token, u8 index, -+ struct opr_cfg *cfg, struct opr_qry *qry); -+ -+/** -+ * enum dpseci_congestion_unit - DPSECI congestion units -+ * @DPSECI_CONGESTION_UNIT_BYTES: bytes units -+ * @DPSECI_CONGESTION_UNIT_FRAMES: frames units -+ */ -+enum dpseci_congestion_unit { -+ DPSECI_CONGESTION_UNIT_BYTES = 0, -+ DPSECI_CONGESTION_UNIT_FRAMES -+}; -+ -+#define DPSECI_CGN_MODE_WRITE_MEM_ON_ENTER 0x00000001 -+#define DPSECI_CGN_MODE_WRITE_MEM_ON_EXIT 0x00000002 -+#define DPSECI_CGN_MODE_COHERENT_WRITE 0x00000004 -+#define DPSECI_CGN_MODE_NOTIFY_DEST_ON_ENTER 0x00000008 -+#define DPSECI_CGN_MODE_NOTIFY_DEST_ON_EXIT 0x00000010 -+#define DPSECI_CGN_MODE_INTR_COALESCING_DISABLED 0x00000020 -+ -+/** -+ * struct dpseci_congestion_notification_cfg - congestion notification -+ * configuration -+ * @units: units type -+ * @threshold_entry: above this threshold we enter a congestion state. -+ * set it to '0' to disable it -+ * @threshold_exit: below this threshold we exit the congestion state. -+ * @message_ctx: The context that will be part of the CSCN message -+ * @message_iova: I/O virtual address (must be in DMA-able memory), -+ * must be 16B aligned; -+ * @dest_cfg: CSCN can be send to either DPIO or DPCON WQ channel -+ * @notification_mode: Mask of available options; use 'DPSECI_CGN_MODE_<X>' -+ * values -+ */ -+struct dpseci_congestion_notification_cfg { -+ enum dpseci_congestion_unit units; -+ u32 threshold_entry; -+ u32 threshold_exit; -+ u64 message_ctx; -+ u64 message_iova; -+ struct dpseci_dest_cfg dest_cfg; -+ u16 notification_mode; -+}; -+ -+int dpseci_set_congestion_notification(struct fsl_mc_io *mc_io, u32 cmd_flags, -+ u16 token, const struct dpseci_congestion_notification_cfg *cfg); -+ -+int dpseci_get_congestion_notification(struct fsl_mc_io *mc_io, u32 cmd_flags, -+ u16 token, struct dpseci_congestion_notification_cfg *cfg); -+ -+#endif /* _DPSECI_H_ */ ---- /dev/null -+++ b/drivers/crypto/caam/dpseci_cmd.h -@@ -0,0 +1,261 @@ -+/* -+ * Copyright 2013-2016 Freescale Semiconductor Inc. -+ * Copyright 2017 NXP -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the names of the above-listed copyright holders nor the -+ * names of any contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE -+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -+ * POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#ifndef _DPSECI_CMD_H_ -+#define _DPSECI_CMD_H_ -+ -+/* DPSECI Version */ -+#define DPSECI_VER_MAJOR 5 -+#define DPSECI_VER_MINOR 1 -+ -+#define DPSECI_VER(maj, min) (((maj) << 16) | (min)) -+#define DPSECI_VERSION DPSECI_VER(DPSECI_VER_MAJOR, DPSECI_VER_MINOR) -+ -+/* Command IDs */ -+ -+#define DPSECI_CMDID_CLOSE 0x8001 -+#define DPSECI_CMDID_OPEN 0x8091 -+#define DPSECI_CMDID_CREATE 0x9092 -+#define DPSECI_CMDID_DESTROY 0x9891 -+#define DPSECI_CMDID_GET_API_VERSION 0xa091 -+ -+#define DPSECI_CMDID_ENABLE 0x0021 -+#define DPSECI_CMDID_DISABLE 0x0031 -+#define DPSECI_CMDID_GET_ATTR 0x0041 -+#define DPSECI_CMDID_RESET 0x0051 -+#define DPSECI_CMDID_IS_ENABLED 0x0061 -+ -+#define DPSECI_CMDID_SET_IRQ_ENABLE 0x0121 -+#define DPSECI_CMDID_GET_IRQ_ENABLE 0x0131 -+#define DPSECI_CMDID_SET_IRQ_MASK 0x0141 -+#define DPSECI_CMDID_GET_IRQ_MASK 0x0151 -+#define DPSECI_CMDID_GET_IRQ_STATUS 0x0161 -+#define DPSECI_CMDID_CLEAR_IRQ_STATUS 0x0171 -+ -+#define DPSECI_CMDID_SET_RX_QUEUE 0x1941 -+#define DPSECI_CMDID_GET_RX_QUEUE 0x1961 -+#define DPSECI_CMDID_GET_TX_QUEUE 0x1971 -+#define DPSECI_CMDID_GET_SEC_ATTR 0x1981 -+#define DPSECI_CMDID_GET_SEC_COUNTERS 0x1991 -+#define DPSECI_CMDID_SET_OPR 0x19A1 -+#define DPSECI_CMDID_GET_OPR 0x19B1 -+ -+#define DPSECI_CMDID_SET_CONGESTION_NOTIFICATION 0x1701 -+#define DPSECI_CMDID_GET_CONGESTION_NOTIFICATION 0x1711 -+ -+/* Macros for accessing command fields smaller than 1 byte */ -+#define DPSECI_MASK(field) \ -+ GENMASK(DPSECI_##field##_SHIFT + DPSECI_##field##_SIZE - 1, \ -+ DPSECI_##field##_SHIFT) -+ -+#define dpseci_set_field(var, field, val) \ -+ ((var) |= (((val) << DPSECI_##field##_SHIFT) & DPSECI_MASK(field))) -+ -+#define dpseci_get_field(var, field) \ -+ (((var) & DPSECI_MASK(field)) >> DPSECI_##field##_SHIFT) -+ -+struct dpseci_cmd_open { -+ __le32 dpseci_id; -+}; -+ -+struct dpseci_cmd_create { -+ u8 priorities[8]; -+ u8 num_tx_queues; -+ u8 num_rx_queues; -+ __le16 pad; -+ __le32 options; -+}; -+ -+struct dpseci_cmd_destroy { -+ __le32 object_id; -+}; -+ -+struct dpseci_rsp_is_enabled { -+ __le32 is_enabled; -+}; -+ -+struct dpseci_cmd_irq_enable { -+ u8 enable_state; -+ u8 pad[3]; -+ u8 irq_index; -+}; -+ -+struct dpseci_rsp_get_irq_enable { -+ u8 enable_state; -+}; -+ -+struct dpseci_cmd_irq_mask { -+ __le32 mask; -+ u8 irq_index; -+}; -+ -+struct dpseci_cmd_irq_status { -+ __le32 status; -+ u8 irq_index; -+}; -+ -+struct dpseci_rsp_get_attributes { -+ __le32 id; -+ __le32 pad0; -+ u8 num_tx_queues; -+ u8 num_rx_queues; -+ u8 pad1[6]; -+ __le32 options; -+}; -+ -+struct dpseci_cmd_queue { -+ __le32 dest_id; -+ u8 priority; -+ u8 queue; -+ u8 dest_type; -+ u8 pad; -+ __le64 user_ctx; -+ union { -+ __le32 options; -+ __le32 fqid; -+ }; -+ __le32 order_preservation_en; -+}; -+ -+struct dpseci_rsp_get_tx_queue { -+ __le32 pad; -+ __le32 fqid; -+ u8 priority; -+}; -+ -+struct dpseci_rsp_get_sec_attr { -+ __le16 ip_id; -+ u8 major_rev; -+ u8 minor_rev; -+ u8 era; -+ u8 pad0[3]; -+ u8 deco_num; -+ u8 zuc_auth_acc_num; -+ u8 zuc_enc_acc_num; -+ u8 pad1; -+ u8 snow_f8_acc_num; -+ u8 snow_f9_acc_num; -+ u8 crc_acc_num; -+ u8 pad2; -+ u8 pk_acc_num; -+ u8 kasumi_acc_num; -+ u8 rng_acc_num; -+ u8 pad3; -+ u8 md_acc_num; -+ u8 arc4_acc_num; -+ u8 des_acc_num; -+ u8 aes_acc_num; -+}; -+ -+struct dpseci_rsp_get_sec_counters { -+ __le64 dequeued_requests; -+ __le64 ob_enc_requests; -+ __le64 ib_dec_requests; -+ __le64 ob_enc_bytes; -+ __le64 ob_prot_bytes; -+ __le64 ib_dec_bytes; -+ __le64 ib_valid_bytes; -+}; -+ -+struct dpseci_rsp_get_api_version { -+ __le16 major; -+ __le16 minor; -+}; -+ -+struct dpseci_cmd_opr { -+ __le16 pad; -+ u8 index; -+ u8 options; -+ u8 pad1[7]; -+ u8 oloe; -+ u8 oeane; -+ u8 olws; -+ u8 oa; -+ u8 oprrws; -+}; -+ -+#define DPSECI_OPR_RIP_SHIFT 0 -+#define DPSECI_OPR_RIP_SIZE 1 -+#define DPSECI_OPR_ENABLE_SHIFT 1 -+#define DPSECI_OPR_ENABLE_SIZE 1 -+#define DPSECI_OPR_TSEQ_NLIS_SHIFT 1 -+#define DPSECI_OPR_TSEQ_NLIS_SIZE 1 -+#define DPSECI_OPR_HSEQ_NLIS_SHIFT 1 -+#define DPSECI_OPR_HSEQ_NLIS_SIZE 1 -+ -+struct dpseci_rsp_get_opr { -+ __le64 pad; -+ u8 rip_enable; -+ u8 pad0[2]; -+ u8 oloe; -+ u8 oeane; -+ u8 olws; -+ u8 oa; -+ u8 oprrws; -+ __le16 nesn; -+ __le16 pad1; -+ __le16 ndsn; -+ __le16 pad2; -+ __le16 ea_tseq; -+ u8 tseq_nlis; -+ u8 pad3; -+ __le16 ea_hseq; -+ u8 hseq_nlis; -+ u8 pad4; -+ __le16 ea_hptr; -+ __le16 pad5; -+ __le16 ea_tptr; -+ __le16 pad6; -+ __le16 opr_vid; -+ __le16 pad7; -+ __le16 opr_id; -+}; -+ -+#define DPSECI_CGN_DEST_TYPE_SHIFT 0 -+#define DPSECI_CGN_DEST_TYPE_SIZE 4 -+#define DPSECI_CGN_UNITS_SHIFT 4 -+#define DPSECI_CGN_UNITS_SIZE 2 -+ -+struct dpseci_cmd_congestion_notification { -+ __le32 dest_id; -+ __le16 notification_mode; -+ u8 priority; -+ u8 options; -+ __le64 message_iova; -+ __le64 message_ctx; -+ __le32 threshold_entry; -+ __le32 threshold_exit; -+}; -+ -+#endif /* _DPSECI_CMD_H_ */ ---- a/drivers/crypto/caam/error.c -+++ b/drivers/crypto/caam/error.c -@@ -6,11 +6,54 @@ - - #include "compat.h" - #include "regs.h" --#include "intern.h" - #include "desc.h" --#include "jr.h" - #include "error.h" - -+#ifdef DEBUG -+ -+#include <linux/highmem.h> -+ -+void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type, -+ int rowsize, int groupsize, struct scatterlist *sg, -+ size_t tlen, bool ascii) -+{ -+ struct scatterlist *it; -+ void *it_page; -+ size_t len; -+ void *buf; -+ -+ for (it = sg; it && tlen > 0 ; it = sg_next(sg)) { -+ /* -+ * make sure the scatterlist's page -+ * has a valid virtual memory mapping -+ */ -+ it_page = kmap_atomic(sg_page(it)); -+ if (unlikely(!it_page)) { -+ pr_err("caam_dump_sg: kmap failed\n"); -+ return; -+ } -+ -+ buf = it_page + it->offset; -+ len = min_t(size_t, tlen, it->length); -+ print_hex_dump(level, prefix_str, prefix_type, rowsize, -+ groupsize, buf, len, ascii); -+ tlen -= len; -+ -+ kunmap_atomic(it_page); -+ } -+} -+ -+#else -+ -+void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type, -+ int rowsize, int groupsize, struct scatterlist *sg, -+ size_t tlen, bool ascii) -+{} -+ -+#endif -+ -+EXPORT_SYMBOL(caam_dump_sg); -+ - static const struct { - u8 value; - const char *error_text; -@@ -69,6 +112,54 @@ static const struct { - { 0xF1, "3GPP HFN matches or exceeds the Threshold" }, - }; - -+static const struct { -+ u8 value; -+ const char *error_text; -+} qi_error_list[] = { -+ { 0x1F, "Job terminated by FQ or ICID flush" }, -+ { 0x20, "FD format error"}, -+ { 0x21, "FD command format error"}, -+ { 0x23, "FL format error"}, -+ { 0x25, "CRJD specified in FD, but not enabled in FLC"}, -+ { 0x30, "Max. buffer size too small"}, -+ { 0x31, "DHR exceeds max. buffer size (allocate mode, S/G format)"}, -+ { 0x32, "SGT exceeds max. buffer size (allocate mode, S/G format"}, -+ { 0x33, "Size over/underflow (allocate mode)"}, -+ { 0x34, "Size over/underflow (reuse mode)"}, -+ { 0x35, "Length exceeds max. short length (allocate mode, S/G/ format)"}, -+ { 0x36, "Memory footprint exceeds max. value (allocate mode, S/G/ format)"}, -+ { 0x41, "SBC frame format not supported (allocate mode)"}, -+ { 0x42, "Pool 0 invalid / pool 1 size < pool 0 size (allocate mode)"}, -+ { 0x43, "Annotation output enabled but ASAR = 0 (allocate mode)"}, -+ { 0x44, "Unsupported or reserved frame format or SGHR = 1 (reuse mode)"}, -+ { 0x45, "DHR correction underflow (reuse mode, single buffer format)"}, -+ { 0x46, "Annotation length exceeds offset (reuse mode)"}, -+ { 0x48, "Annotation output enabled but ASA limited by ASAR (reuse mode)"}, -+ { 0x49, "Data offset correction exceeds input frame data length (reuse mode)"}, -+ { 0x4B, "Annotation output enabled but ASA cannote be expanded (frame list)"}, -+ { 0x51, "Unsupported IF reuse mode"}, -+ { 0x52, "Unsupported FL use mode"}, -+ { 0x53, "Unsupported RJD use mode"}, -+ { 0x54, "Unsupported inline descriptor use mode"}, -+ { 0xC0, "Table buffer pool 0 depletion"}, -+ { 0xC1, "Table buffer pool 1 depletion"}, -+ { 0xC2, "Data buffer pool 0 depletion, no OF allocated"}, -+ { 0xC3, "Data buffer pool 1 depletion, no OF allocated"}, -+ { 0xC4, "Data buffer pool 0 depletion, partial OF allocated"}, -+ { 0xC5, "Data buffer pool 1 depletion, partial OF allocated"}, -+ { 0xD0, "FLC read error"}, -+ { 0xD1, "FL read error"}, -+ { 0xD2, "FL write error"}, -+ { 0xD3, "OF SGT write error"}, -+ { 0xD4, "PTA read error"}, -+ { 0xD5, "PTA write error"}, -+ { 0xD6, "OF SGT F-bit write error"}, -+ { 0xD7, "ASA write error"}, -+ { 0xE1, "FLC[ICR]=0 ICID error"}, -+ { 0xE2, "FLC[ICR]=1 ICID error"}, -+ { 0xE4, "source of ICID flush not trusted (BDI = 0)"}, -+}; -+ - static const char * const cha_id_list[] = { - "", - "AES", -@@ -146,10 +237,9 @@ static void report_ccb_status(struct dev - strlen(rng_err_id_list[err_id])) { - /* RNG-only error */ - err_str = rng_err_id_list[err_id]; -- } else if (err_id < ARRAY_SIZE(err_id_list)) -+ } else { - err_str = err_id_list[err_id]; -- else -- snprintf(err_err_code, sizeof(err_err_code), "%02x", err_id); -+ } - - /* - * CCB ICV check failures are part of normal operation life; -@@ -198,6 +288,27 @@ static void report_deco_status(struct de - status, error, idx_str, idx, err_str, err_err_code); - } - -+static void report_qi_status(struct device *qidev, const u32 status, -+ const char *error) -+{ -+ u8 err_id = status & JRSTA_QIERR_ERROR_MASK; -+ const char *err_str = "unidentified error value 0x"; -+ char err_err_code[3] = { 0 }; -+ int i; -+ -+ for (i = 0; i < ARRAY_SIZE(qi_error_list); i++) -+ if (qi_error_list[i].value == err_id) -+ break; -+ -+ if (i != ARRAY_SIZE(qi_error_list) && qi_error_list[i].error_text) -+ err_str = qi_error_list[i].error_text; -+ else -+ snprintf(err_err_code, sizeof(err_err_code), "%02x", err_id); -+ -+ dev_err(qidev, "%08x: %s: %s%s\n", -+ status, error, err_str, err_err_code); -+} -+ - static void report_jr_status(struct device *jrdev, const u32 status, - const char *error) - { -@@ -212,7 +323,7 @@ static void report_cond_code_status(stru - status, error, __func__); - } - --void caam_jr_strstatus(struct device *jrdev, u32 status) -+void caam_strstatus(struct device *jrdev, u32 status, bool qi_v2) - { - static const struct stat_src { - void (*report_ssed)(struct device *jrdev, const u32 status, -@@ -224,7 +335,7 @@ void caam_jr_strstatus(struct device *jr - { report_ccb_status, "CCB" }, - { report_jump_status, "Jump" }, - { report_deco_status, "DECO" }, -- { NULL, "Queue Manager Interface" }, -+ { report_qi_status, "Queue Manager Interface" }, - { report_jr_status, "Job Ring" }, - { report_cond_code_status, "Condition Code" }, - { NULL, NULL }, -@@ -250,4 +361,4 @@ void caam_jr_strstatus(struct device *jr - else - dev_err(jrdev, "%d: unknown error source\n", ssrc); - } --EXPORT_SYMBOL(caam_jr_strstatus); -+EXPORT_SYMBOL(caam_strstatus); ---- a/drivers/crypto/caam/error.h -+++ b/drivers/crypto/caam/error.h -@@ -7,5 +7,13 @@ - #ifndef CAAM_ERROR_H - #define CAAM_ERROR_H - #define CAAM_ERROR_STR_MAX 302 --void caam_jr_strstatus(struct device *jrdev, u32 status); -+ -+void caam_strstatus(struct device *dev, u32 status, bool qi_v2); -+ -+#define caam_jr_strstatus(jrdev, status) caam_strstatus(jrdev, status, false) -+#define caam_qi2_strstatus(qidev, status) caam_strstatus(qidev, status, true) -+ -+void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type, -+ int rowsize, int groupsize, struct scatterlist *sg, -+ size_t tlen, bool ascii); - #endif /* CAAM_ERROR_H */ ---- a/drivers/crypto/caam/intern.h -+++ b/drivers/crypto/caam/intern.h -@@ -64,10 +64,9 @@ struct caam_drv_private_jr { - * Driver-private storage for a single CAAM block instance - */ - struct caam_drv_private { -- -- struct device *dev; -- struct platform_device **jrpdev; /* Alloc'ed array per sub-device */ -- struct platform_device *pdev; -+#ifdef CONFIG_CAAM_QI -+ struct device *qidev; -+#endif - - /* Physical-presence section */ - struct caam_ctrl __iomem *ctrl; /* controller region */ -@@ -84,6 +83,7 @@ struct caam_drv_private { - u8 qi_present; /* Nonzero if QI present in device */ - int secvio_irq; /* Security violation interrupt number */ - int virt_en; /* Virtualization enabled in CAAM */ -+ int era; /* CAAM Era (internal HW revision) */ - - #define RNG4_MAX_HANDLES 2 - /* RNG4 block */ -@@ -103,11 +103,6 @@ struct caam_drv_private { - #ifdef CONFIG_DEBUG_FS - struct dentry *dfs_root; - struct dentry *ctl; /* controller dir */ -- struct dentry *ctl_rq_dequeued, *ctl_ob_enc_req, *ctl_ib_dec_req; -- struct dentry *ctl_ob_enc_bytes, *ctl_ob_prot_bytes; -- struct dentry *ctl_ib_dec_bytes, *ctl_ib_valid_bytes; -- struct dentry *ctl_faultaddr, *ctl_faultdetail, *ctl_faultstatus; -- - struct debugfs_blob_wrapper ctl_kek_wrap, ctl_tkek_wrap, ctl_tdsk_wrap; - struct dentry *ctl_kek, *ctl_tkek, *ctl_tdsk; - #endif -@@ -115,4 +110,22 @@ struct caam_drv_private { - - void caam_jr_algapi_init(struct device *dev); - void caam_jr_algapi_remove(struct device *dev); -+ -+#ifdef CONFIG_DEBUG_FS -+static int caam_debugfs_u64_get(void *data, u64 *val) -+{ -+ *val = caam64_to_cpu(*(u64 *)data); -+ return 0; -+} -+ -+static int caam_debugfs_u32_get(void *data, u64 *val) -+{ -+ *val = caam32_to_cpu(*(u32 *)data); -+ return 0; -+} -+ -+DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u32_ro, caam_debugfs_u32_get, NULL, "%llu\n"); -+DEFINE_SIMPLE_ATTRIBUTE(caam_fops_u64_ro, caam_debugfs_u64_get, NULL, "%llu\n"); -+#endif -+ - #endif /* INTERN_H */ ---- a/drivers/crypto/caam/jr.c -+++ b/drivers/crypto/caam/jr.c -@@ -9,6 +9,7 @@ - #include <linux/of_address.h> - - #include "compat.h" -+#include "ctrl.h" - #include "regs.h" - #include "jr.h" - #include "desc.h" -@@ -22,6 +23,14 @@ struct jr_driver_data { - - static struct jr_driver_data driver_data; - -+static int jr_driver_probed; -+ -+int caam_jr_driver_probed(void) -+{ -+ return jr_driver_probed; -+} -+EXPORT_SYMBOL(caam_jr_driver_probed); -+ - static int caam_reset_hw_jr(struct device *dev) - { - struct caam_drv_private_jr *jrp = dev_get_drvdata(dev); -@@ -118,6 +127,8 @@ static int caam_jr_remove(struct platfor - dev_err(jrdev, "Failed to shut down job ring\n"); - irq_dispose_mapping(jrpriv->irq); - -+ jr_driver_probed--; -+ - return ret; - } - -@@ -281,6 +292,36 @@ struct device *caam_jr_alloc(void) - EXPORT_SYMBOL(caam_jr_alloc); - - /** -+ * caam_jridx_alloc() - Alloc a specific job ring based on its index. -+ * -+ * returns : pointer to the newly allocated physical -+ * JobR dev can be written to if successful. -+ **/ -+struct device *caam_jridx_alloc(int idx) -+{ -+ struct caam_drv_private_jr *jrpriv; -+ struct device *dev = ERR_PTR(-ENODEV); -+ -+ spin_lock(&driver_data.jr_alloc_lock); -+ -+ if (list_empty(&driver_data.jr_list)) -+ goto end; -+ -+ list_for_each_entry(jrpriv, &driver_data.jr_list, list_node) { -+ if (jrpriv->ridx == idx) { -+ atomic_inc(&jrpriv->tfm_count); -+ dev = jrpriv->dev; -+ break; -+ } -+ } -+ -+end: -+ spin_unlock(&driver_data.jr_alloc_lock); -+ return dev; -+} -+EXPORT_SYMBOL(caam_jridx_alloc); -+ -+/** - * caam_jr_free() - Free the Job Ring - * @rdev - points to the dev that identifies the Job ring to - * be released. -@@ -497,15 +538,28 @@ static int caam_jr_probe(struct platform - return -ENOMEM; - } - -- jrpriv->rregs = (struct caam_job_ring __force *)ctrl; -+ jrpriv->rregs = (struct caam_job_ring __iomem __force *)ctrl; - -- if (sizeof(dma_addr_t) == sizeof(u64)) -- if (of_device_is_compatible(nprop, "fsl,sec-v5.0-job-ring")) -- dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(40)); -+ if (sizeof(dma_addr_t) == sizeof(u64)) { -+ if (caam_dpaa2) -+ error = dma_set_mask_and_coherent(jrdev, -+ DMA_BIT_MASK(49)); -+ else if (of_device_is_compatible(nprop, -+ "fsl,sec-v5.0-job-ring")) -+ error = dma_set_mask_and_coherent(jrdev, -+ DMA_BIT_MASK(40)); - else -- dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(36)); -- else -- dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(32)); -+ error = dma_set_mask_and_coherent(jrdev, -+ DMA_BIT_MASK(36)); -+ } else { -+ error = dma_set_mask_and_coherent(jrdev, DMA_BIT_MASK(32)); -+ } -+ if (error) { -+ dev_err(jrdev, "dma_set_mask_and_coherent failed (%d)\n", -+ error); -+ iounmap(ctrl); -+ return error; -+ } - - /* Identify the interrupt */ - jrpriv->irq = irq_of_parse_and_map(nprop, 0); -@@ -525,10 +579,12 @@ static int caam_jr_probe(struct platform - - atomic_set(&jrpriv->tfm_count, 0); - -+ jr_driver_probed++; -+ - return 0; - } - --static struct of_device_id caam_jr_match[] = { -+static const struct of_device_id caam_jr_match[] = { - { - .compatible = "fsl,sec-v4.0-job-ring", - }, ---- a/drivers/crypto/caam/jr.h -+++ b/drivers/crypto/caam/jr.h -@@ -8,7 +8,9 @@ - #define JR_H - - /* Prototypes for backend-level services exposed to APIs */ -+int caam_jr_driver_probed(void); - struct device *caam_jr_alloc(void); -+struct device *caam_jridx_alloc(int idx); - void caam_jr_free(struct device *rdev); - int caam_jr_enqueue(struct device *dev, u32 *desc, - void (*cbk)(struct device *dev, u32 *desc, u32 status, ---- a/drivers/crypto/caam/key_gen.c -+++ b/drivers/crypto/caam/key_gen.c -@@ -41,15 +41,29 @@ Split key generation-------------------- - [06] 0x64260028 fifostr: class2 mdsplit-jdk len=40 - @0xffe04000 - */ --int gen_split_key(struct device *jrdev, u8 *key_out, int split_key_len, -- int split_key_pad_len, const u8 *key_in, u32 keylen, -- u32 alg_op) -+int gen_split_key(struct device *jrdev, u8 *key_out, -+ struct alginfo * const adata, const u8 *key_in, u32 keylen, -+ int max_keylen) - { - u32 *desc; - struct split_key_result result; - dma_addr_t dma_addr_in, dma_addr_out; - int ret = -ENOMEM; - -+ adata->keylen = split_key_len(adata->algtype & OP_ALG_ALGSEL_MASK); -+ adata->keylen_pad = split_key_pad_len(adata->algtype & -+ OP_ALG_ALGSEL_MASK); -+ -+#ifdef DEBUG -+ dev_err(jrdev, "split keylen %d split keylen padded %d\n", -+ adata->keylen, adata->keylen_pad); -+ print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ", -+ DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1); -+#endif -+ -+ if (adata->keylen_pad > max_keylen) -+ return -EINVAL; -+ - desc = kmalloc(CAAM_CMD_SZ * 6 + CAAM_PTR_SZ * 2, GFP_KERNEL | GFP_DMA); - if (!desc) { - dev_err(jrdev, "unable to allocate key input memory\n"); -@@ -63,7 +77,7 @@ int gen_split_key(struct device *jrdev, - goto out_free; - } - -- dma_addr_out = dma_map_single(jrdev, key_out, split_key_pad_len, -+ dma_addr_out = dma_map_single(jrdev, key_out, adata->keylen_pad, - DMA_FROM_DEVICE); - if (dma_mapping_error(jrdev, dma_addr_out)) { - dev_err(jrdev, "unable to map key output memory\n"); -@@ -74,7 +88,9 @@ int gen_split_key(struct device *jrdev, - append_key(desc, dma_addr_in, keylen, CLASS_2 | KEY_DEST_CLASS_REG); - - /* Sets MDHA up into an HMAC-INIT */ -- append_operation(desc, alg_op | OP_ALG_DECRYPT | OP_ALG_AS_INIT); -+ append_operation(desc, (adata->algtype & OP_ALG_ALGSEL_MASK) | -+ OP_ALG_AAI_HMAC | OP_TYPE_CLASS2_ALG | OP_ALG_DECRYPT | -+ OP_ALG_AS_INIT); - - /* - * do a FIFO_LOAD of zero, this will trigger the internal key expansion -@@ -87,7 +103,7 @@ int gen_split_key(struct device *jrdev, - * FIFO_STORE with the explicit split-key content store - * (0x26 output type) - */ -- append_fifo_store(desc, dma_addr_out, split_key_len, -+ append_fifo_store(desc, dma_addr_out, adata->keylen, - LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK); - - #ifdef DEBUG -@@ -108,11 +124,11 @@ int gen_split_key(struct device *jrdev, - #ifdef DEBUG - print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key_out, -- split_key_pad_len, 1); -+ adata->keylen_pad, 1); - #endif - } - -- dma_unmap_single(jrdev, dma_addr_out, split_key_pad_len, -+ dma_unmap_single(jrdev, dma_addr_out, adata->keylen_pad, - DMA_FROM_DEVICE); - out_unmap_in: - dma_unmap_single(jrdev, dma_addr_in, keylen, DMA_TO_DEVICE); ---- a/drivers/crypto/caam/key_gen.h -+++ b/drivers/crypto/caam/key_gen.h -@@ -5,6 +5,36 @@ - * - */ - -+/** -+ * split_key_len - Compute MDHA split key length for a given algorithm -+ * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1, -+ * SHA224, SHA384, SHA512. -+ * -+ * Return: MDHA split key length -+ */ -+static inline u32 split_key_len(u32 hash) -+{ -+ /* Sizes for MDHA pads (*not* keys): MD5, SHA1, 224, 256, 384, 512 */ -+ static const u8 mdpadlen[] = { 16, 20, 32, 32, 64, 64 }; -+ u32 idx; -+ -+ idx = (hash & OP_ALG_ALGSEL_SUBMASK) >> OP_ALG_ALGSEL_SHIFT; -+ -+ return (u32)(mdpadlen[idx] * 2); -+} -+ -+/** -+ * split_key_pad_len - Compute MDHA split key pad length for a given algorithm -+ * @hash: Hashing algorithm selection, one of OP_ALG_ALGSEL_* - MD5, SHA1, -+ * SHA224, SHA384, SHA512. -+ * -+ * Return: MDHA split key pad length -+ */ -+static inline u32 split_key_pad_len(u32 hash) -+{ -+ return ALIGN(split_key_len(hash), 16); -+} -+ - struct split_key_result { - struct completion completion; - int err; -@@ -12,6 +42,6 @@ struct split_key_result { - - void split_key_done(struct device *dev, u32 *desc, u32 err, void *context); - --int gen_split_key(struct device *jrdev, u8 *key_out, int split_key_len, -- int split_key_pad_len, const u8 *key_in, u32 keylen, -- u32 alg_op); -+int gen_split_key(struct device *jrdev, u8 *key_out, -+ struct alginfo * const adata, const u8 *key_in, u32 keylen, -+ int max_keylen); ---- a/drivers/crypto/caam/pdb.h -+++ b/drivers/crypto/caam/pdb.h -@@ -483,6 +483,8 @@ struct dsa_verify_pdb { - #define RSA_PDB_E_MASK (0xFFF << RSA_PDB_E_SHIFT) - #define RSA_PDB_D_SHIFT 12 - #define RSA_PDB_D_MASK (0xFFF << RSA_PDB_D_SHIFT) -+#define RSA_PDB_Q_SHIFT 12 -+#define RSA_PDB_Q_MASK (0xFFF << RSA_PDB_Q_SHIFT) - - #define RSA_PDB_SGF_F (0x8 << RSA_PDB_SGF_SHIFT) - #define RSA_PDB_SGF_G (0x4 << RSA_PDB_SGF_SHIFT) -@@ -490,6 +492,8 @@ struct dsa_verify_pdb { - #define RSA_PRIV_PDB_SGF_G (0x8 << RSA_PDB_SGF_SHIFT) - - #define RSA_PRIV_KEY_FRM_1 0 -+#define RSA_PRIV_KEY_FRM_2 1 -+#define RSA_PRIV_KEY_FRM_3 2 - - /** - * RSA Encrypt Protocol Data Block -@@ -525,4 +529,62 @@ struct rsa_priv_f1_pdb { - dma_addr_t d_dma; - } __packed; - -+/** -+ * RSA Decrypt PDB - Private Key Form #2 -+ * @sgf : scatter-gather field -+ * @g_dma : dma address of encrypted input data -+ * @f_dma : dma address of output data -+ * @d_dma : dma address of RSA private exponent -+ * @p_dma : dma address of RSA prime factor p of RSA modulus n -+ * @q_dma : dma address of RSA prime factor q of RSA modulus n -+ * @tmp1_dma: dma address of temporary buffer. CAAM uses this temporary buffer -+ * as internal state buffer. It is assumed to be as long as p. -+ * @tmp2_dma: dma address of temporary buffer. CAAM uses this temporary buffer -+ * as internal state buffer. It is assumed to be as long as q. -+ * @p_q_len : length in bytes of first two prime factors of the RSA modulus n -+ */ -+struct rsa_priv_f2_pdb { -+ u32 sgf; -+ dma_addr_t g_dma; -+ dma_addr_t f_dma; -+ dma_addr_t d_dma; -+ dma_addr_t p_dma; -+ dma_addr_t q_dma; -+ dma_addr_t tmp1_dma; -+ dma_addr_t tmp2_dma; -+ u32 p_q_len; -+} __packed; -+ -+/** -+ * RSA Decrypt PDB - Private Key Form #3 -+ * This is the RSA Chinese Reminder Theorem (CRT) form for two prime factors of -+ * the RSA modulus. -+ * @sgf : scatter-gather field -+ * @g_dma : dma address of encrypted input data -+ * @f_dma : dma address of output data -+ * @c_dma : dma address of RSA CRT coefficient -+ * @p_dma : dma address of RSA prime factor p of RSA modulus n -+ * @q_dma : dma address of RSA prime factor q of RSA modulus n -+ * @dp_dma : dma address of RSA CRT exponent of RSA prime factor p -+ * @dp_dma : dma address of RSA CRT exponent of RSA prime factor q -+ * @tmp1_dma: dma address of temporary buffer. CAAM uses this temporary buffer -+ * as internal state buffer. It is assumed to be as long as p. -+ * @tmp2_dma: dma address of temporary buffer. CAAM uses this temporary buffer -+ * as internal state buffer. It is assumed to be as long as q. -+ * @p_q_len : length in bytes of first two prime factors of the RSA modulus n -+ */ -+struct rsa_priv_f3_pdb { -+ u32 sgf; -+ dma_addr_t g_dma; -+ dma_addr_t f_dma; -+ dma_addr_t c_dma; -+ dma_addr_t p_dma; -+ dma_addr_t q_dma; -+ dma_addr_t dp_dma; -+ dma_addr_t dq_dma; -+ dma_addr_t tmp1_dma; -+ dma_addr_t tmp2_dma; -+ u32 p_q_len; -+} __packed; -+ - #endif ---- a/drivers/crypto/caam/pkc_desc.c -+++ b/drivers/crypto/caam/pkc_desc.c -@@ -34,3 +34,39 @@ void init_rsa_priv_f1_desc(u32 *desc, st - append_operation(desc, OP_TYPE_UNI_PROTOCOL | OP_PCLID_RSADEC_PRVKEY | - RSA_PRIV_KEY_FRM_1); - } -+ -+/* Descriptor for RSA Private operation - Private Key Form #2 */ -+void init_rsa_priv_f2_desc(u32 *desc, struct rsa_priv_f2_pdb *pdb) -+{ -+ init_job_desc_pdb(desc, 0, sizeof(*pdb)); -+ append_cmd(desc, pdb->sgf); -+ append_ptr(desc, pdb->g_dma); -+ append_ptr(desc, pdb->f_dma); -+ append_ptr(desc, pdb->d_dma); -+ append_ptr(desc, pdb->p_dma); -+ append_ptr(desc, pdb->q_dma); -+ append_ptr(desc, pdb->tmp1_dma); -+ append_ptr(desc, pdb->tmp2_dma); -+ append_cmd(desc, pdb->p_q_len); -+ append_operation(desc, OP_TYPE_UNI_PROTOCOL | OP_PCLID_RSADEC_PRVKEY | -+ RSA_PRIV_KEY_FRM_2); -+} -+ -+/* Descriptor for RSA Private operation - Private Key Form #3 */ -+void init_rsa_priv_f3_desc(u32 *desc, struct rsa_priv_f3_pdb *pdb) -+{ -+ init_job_desc_pdb(desc, 0, sizeof(*pdb)); -+ append_cmd(desc, pdb->sgf); -+ append_ptr(desc, pdb->g_dma); -+ append_ptr(desc, pdb->f_dma); -+ append_ptr(desc, pdb->c_dma); -+ append_ptr(desc, pdb->p_dma); -+ append_ptr(desc, pdb->q_dma); -+ append_ptr(desc, pdb->dp_dma); -+ append_ptr(desc, pdb->dq_dma); -+ append_ptr(desc, pdb->tmp1_dma); -+ append_ptr(desc, pdb->tmp2_dma); -+ append_cmd(desc, pdb->p_q_len); -+ append_operation(desc, OP_TYPE_UNI_PROTOCOL | OP_PCLID_RSADEC_PRVKEY | -+ RSA_PRIV_KEY_FRM_3); -+} ---- /dev/null -+++ b/drivers/crypto/caam/qi.c -@@ -0,0 +1,804 @@ -+/* -+ * CAAM/SEC 4.x QI transport/backend driver -+ * Queue Interface backend functionality -+ * -+ * Copyright 2013-2016 Freescale Semiconductor, Inc. -+ * Copyright 2016-2017 NXP -+ */ -+ -+#include <linux/cpumask.h> -+#include <linux/kthread.h> -+#include <linux/fsl_qman.h> -+ -+#include "regs.h" -+#include "qi.h" -+#include "desc.h" -+#include "intern.h" -+#include "desc_constr.h" -+ -+#define PREHDR_RSLS_SHIFT 31 -+ -+/* -+ * Use a reasonable backlog of frames (per CPU) as congestion threshold, -+ * so that resources used by the in-flight buffers do not become a memory hog. -+ */ -+#define MAX_RSP_FQ_BACKLOG_PER_CPU 256 -+ -+#define CAAM_QI_ENQUEUE_RETRIES 10000 -+ -+#define CAAM_NAPI_WEIGHT 63 -+ -+/* -+ * caam_napi - struct holding CAAM NAPI-related params -+ * @irqtask: IRQ task for QI backend -+ * @p: QMan portal -+ */ -+struct caam_napi { -+ struct napi_struct irqtask; -+ struct qman_portal *p; -+}; -+ -+/* -+ * caam_qi_pcpu_priv - percpu private data structure to main list of pending -+ * responses expected on each cpu. -+ * @caam_napi: CAAM NAPI params -+ * @net_dev: netdev used by NAPI -+ * @rsp_fq: response FQ from CAAM -+ */ -+struct caam_qi_pcpu_priv { -+ struct caam_napi caam_napi; -+ struct net_device net_dev; -+ struct qman_fq *rsp_fq; -+} ____cacheline_aligned; -+ -+static DEFINE_PER_CPU(struct caam_qi_pcpu_priv, pcpu_qipriv); -+static DEFINE_PER_CPU(int, last_cpu); -+ -+/* -+ * caam_qi_priv - CAAM QI backend private params -+ * @cgr: QMan congestion group -+ * @qi_pdev: platform device for QI backend -+ */ -+struct caam_qi_priv { -+ struct qman_cgr cgr; -+ struct platform_device *qi_pdev; -+}; -+ -+static struct caam_qi_priv qipriv ____cacheline_aligned; -+ -+/* -+ * This is written by only one core - the one that initialized the CGR - and -+ * read by multiple cores (all the others). -+ */ -+bool caam_congested __read_mostly; -+EXPORT_SYMBOL(caam_congested); -+ -+#ifdef CONFIG_DEBUG_FS -+/* -+ * This is a counter for the number of times the congestion group (where all -+ * the request and response queueus are) reached congestion. Incremented -+ * each time the congestion callback is called with congested == true. -+ */ -+static u64 times_congested; -+#endif -+ -+/* -+ * CPU from where the module initialised. This is required because QMan driver -+ * requires CGRs to be removed from same CPU from where they were originally -+ * allocated. -+ */ -+static int mod_init_cpu; -+ -+/* -+ * This is a a cache of buffers, from which the users of CAAM QI driver -+ * can allocate short (CAAM_QI_MEMCACHE_SIZE) buffers. It's faster than -+ * doing malloc on the hotpath. -+ * NOTE: A more elegant solution would be to have some headroom in the frames -+ * being processed. This could be added by the dpaa-ethernet driver. -+ * This would pose a problem for userspace application processing which -+ * cannot know of this limitation. So for now, this will work. -+ * NOTE: The memcache is SMP-safe. No need to handle spinlocks in-here -+ */ -+static struct kmem_cache *qi_cache; -+ -+int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req) -+{ -+ struct qm_fd fd; -+ int ret; -+ int num_retries = 0; -+ -+ fd.cmd = 0; -+ fd.format = qm_fd_compound; -+ fd.cong_weight = caam32_to_cpu(req->fd_sgt[1].length); -+ fd.addr = dma_map_single(qidev, req->fd_sgt, sizeof(req->fd_sgt), -+ DMA_BIDIRECTIONAL); -+ if (dma_mapping_error(qidev, fd.addr)) { -+ dev_err(qidev, "DMA mapping error for QI enqueue request\n"); -+ return -EIO; -+ } -+ -+ do { -+ ret = qman_enqueue(req->drv_ctx->req_fq, &fd, 0); -+ if (likely(!ret)) -+ return 0; -+ -+ if (ret != -EBUSY) -+ break; -+ num_retries++; -+ } while (num_retries < CAAM_QI_ENQUEUE_RETRIES); -+ -+ dev_err(qidev, "qman_enqueue failed: %d\n", ret); -+ -+ return ret; -+} -+EXPORT_SYMBOL(caam_qi_enqueue); -+ -+static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq, -+ const struct qm_mr_entry *msg) -+{ -+ const struct qm_fd *fd; -+ struct caam_drv_req *drv_req; -+ struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev); -+ -+ fd = &msg->ern.fd; -+ -+ if (fd->format != qm_fd_compound) { -+ dev_err(qidev, "Non-compound FD from CAAM\n"); -+ return; -+ } -+ -+ drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd)); -+ if (!drv_req) { -+ dev_err(qidev, -+ "Can't find original request for CAAM response\n"); -+ return; -+ } -+ -+ dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd), -+ sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL); -+ -+ drv_req->cbk(drv_req, -EIO); -+} -+ -+static struct qman_fq *create_caam_req_fq(struct device *qidev, -+ struct qman_fq *rsp_fq, -+ dma_addr_t hwdesc, -+ int fq_sched_flag) -+{ -+ int ret; -+ struct qman_fq *req_fq; -+ struct qm_mcc_initfq opts; -+ -+ req_fq = kzalloc(sizeof(*req_fq), GFP_ATOMIC); -+ if (!req_fq) -+ return ERR_PTR(-ENOMEM); -+ -+ req_fq->cb.ern = caam_fq_ern_cb; -+ req_fq->cb.fqs = NULL; -+ -+ ret = qman_create_fq(0, QMAN_FQ_FLAG_DYNAMIC_FQID | -+ QMAN_FQ_FLAG_TO_DCPORTAL | QMAN_FQ_FLAG_LOCKED, -+ req_fq); -+ if (ret) { -+ dev_err(qidev, "Failed to create session req FQ\n"); -+ goto create_req_fq_fail; -+ } -+ -+ opts.we_mask = QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ | -+ QM_INITFQ_WE_CONTEXTB | QM_INITFQ_WE_CONTEXTA | -+ QM_INITFQ_WE_CGID; -+ opts.fqd.fq_ctrl = QM_FQCTRL_CPCSTASH | QM_FQCTRL_CGE; -+ opts.fqd.dest.channel = qm_channel_caam; -+ opts.fqd.dest.wq = 2; -+ opts.fqd.context_b = qman_fq_fqid(rsp_fq); -+ opts.fqd.context_a.hi = upper_32_bits(hwdesc); -+ opts.fqd.context_a.lo = lower_32_bits(hwdesc); -+ opts.fqd.cgid = qipriv.cgr.cgrid; -+ -+ ret = qman_init_fq(req_fq, fq_sched_flag, &opts); -+ if (ret) { -+ dev_err(qidev, "Failed to init session req FQ\n"); -+ goto init_req_fq_fail; -+ } -+ -+ dev_dbg(qidev, "Allocated request FQ %u for CPU %u\n", req_fq->fqid, -+ smp_processor_id()); -+ return req_fq; -+ -+init_req_fq_fail: -+ qman_destroy_fq(req_fq, 0); -+create_req_fq_fail: -+ kfree(req_fq); -+ return ERR_PTR(ret); -+} -+ -+static int empty_retired_fq(struct device *qidev, struct qman_fq *fq) -+{ -+ int ret; -+ -+ ret = qman_volatile_dequeue(fq, QMAN_VOLATILE_FLAG_WAIT_INT | -+ QMAN_VOLATILE_FLAG_FINISH, -+ QM_VDQCR_PRECEDENCE_VDQCR | -+ QM_VDQCR_NUMFRAMES_TILLEMPTY); -+ if (ret) { -+ dev_err(qidev, "Volatile dequeue fail for FQ: %u\n", fq->fqid); -+ return ret; -+ } -+ -+ do { -+ struct qman_portal *p; -+ -+ p = qman_get_affine_portal(smp_processor_id()); -+ qman_p_poll_dqrr(p, 16); -+ } while (fq->flags & QMAN_FQ_STATE_NE); -+ -+ return 0; -+} -+ -+static int kill_fq(struct device *qidev, struct qman_fq *fq) -+{ -+ u32 flags; -+ int ret; -+ -+ ret = qman_retire_fq(fq, &flags); -+ if (ret < 0) { -+ dev_err(qidev, "qman_retire_fq failed: %d\n", ret); -+ return ret; -+ } -+ -+ if (!ret) -+ goto empty_fq; -+ -+ /* Async FQ retirement condition */ -+ if (ret == 1) { -+ /* Retry till FQ gets in retired state */ -+ do { -+ msleep(20); -+ } while (fq->state != qman_fq_state_retired); -+ -+ WARN_ON(fq->flags & QMAN_FQ_STATE_BLOCKOOS); -+ WARN_ON(fq->flags & QMAN_FQ_STATE_ORL); -+ } -+ -+empty_fq: -+ if (fq->flags & QMAN_FQ_STATE_NE) { -+ ret = empty_retired_fq(qidev, fq); -+ if (ret) { -+ dev_err(qidev, "empty_retired_fq fail for FQ: %u\n", -+ fq->fqid); -+ return ret; -+ } -+ } -+ -+ ret = qman_oos_fq(fq); -+ if (ret) -+ dev_err(qidev, "OOS of FQID: %u failed\n", fq->fqid); -+ -+ qman_destroy_fq(fq, 0); -+ kfree(fq); -+ -+ return ret; -+} -+ -+static int empty_caam_fq(struct qman_fq *fq) -+{ -+ int ret; -+ struct qm_mcr_queryfq_np np; -+ -+ /* Wait till the older CAAM FQ get empty */ -+ do { -+ ret = qman_query_fq_np(fq, &np); -+ if (ret) -+ return ret; -+ -+ if (!np.frm_cnt) -+ break; -+ -+ msleep(20); -+ } while (1); -+ -+ /* -+ * Give extra time for pending jobs from this FQ in holding tanks -+ * to get processed -+ */ -+ msleep(20); -+ return 0; -+} -+ -+int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc) -+{ -+ int ret; -+ u32 num_words; -+ struct qman_fq *new_fq, *old_fq; -+ struct device *qidev = drv_ctx->qidev; -+ -+ num_words = desc_len(sh_desc); -+ if (num_words > MAX_SDLEN) { -+ dev_err(qidev, "Invalid descriptor len: %d words\n", num_words); -+ return -EINVAL; -+ } -+ -+ /* Note down older req FQ */ -+ old_fq = drv_ctx->req_fq; -+ -+ /* Create a new req FQ in parked state */ -+ new_fq = create_caam_req_fq(drv_ctx->qidev, drv_ctx->rsp_fq, -+ drv_ctx->context_a, 0); -+ if (unlikely(IS_ERR_OR_NULL(new_fq))) { -+ dev_err(qidev, "FQ allocation for shdesc update failed\n"); -+ return PTR_ERR(new_fq); -+ } -+ -+ /* Hook up new FQ to context so that new requests keep queuing */ -+ drv_ctx->req_fq = new_fq; -+ -+ /* Empty and remove the older FQ */ -+ ret = empty_caam_fq(old_fq); -+ if (ret) { -+ dev_err(qidev, "Old CAAM FQ empty failed: %d\n", ret); -+ -+ /* We can revert to older FQ */ -+ drv_ctx->req_fq = old_fq; -+ -+ if (kill_fq(qidev, new_fq)) -+ dev_warn(qidev, "New CAAM FQ kill failed\n"); -+ -+ return ret; -+ } -+ -+ /* -+ * Re-initialise pre-header. Set RSLS and SDLEN. -+ * Update the shared descriptor for driver context. -+ */ -+ drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) | -+ num_words); -+ memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc)); -+ dma_sync_single_for_device(qidev, drv_ctx->context_a, -+ sizeof(drv_ctx->sh_desc) + -+ sizeof(drv_ctx->prehdr), -+ DMA_BIDIRECTIONAL); -+ -+ /* Put the new FQ in scheduled state */ -+ ret = qman_schedule_fq(new_fq); -+ if (ret) { -+ dev_err(qidev, "Fail to sched new CAAM FQ, ecode = %d\n", ret); -+ -+ /* -+ * We can kill new FQ and revert to old FQ. -+ * Since the desc is already modified, it is success case -+ */ -+ -+ drv_ctx->req_fq = old_fq; -+ -+ if (kill_fq(qidev, new_fq)) -+ dev_warn(qidev, "New CAAM FQ kill failed\n"); -+ } else if (kill_fq(qidev, old_fq)) { -+ dev_warn(qidev, "Old CAAM FQ kill failed\n"); -+ } -+ -+ return 0; -+} -+EXPORT_SYMBOL(caam_drv_ctx_update); -+ -+struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev, -+ int *cpu, -+ u32 *sh_desc) -+{ -+ size_t size; -+ u32 num_words; -+ dma_addr_t hwdesc; -+ struct caam_drv_ctx *drv_ctx; -+ const cpumask_t *cpus = qman_affine_cpus(); -+ -+ num_words = desc_len(sh_desc); -+ if (num_words > MAX_SDLEN) { -+ dev_err(qidev, "Invalid descriptor len: %d words\n", -+ num_words); -+ return ERR_PTR(-EINVAL); -+ } -+ -+ drv_ctx = kzalloc(sizeof(*drv_ctx), GFP_ATOMIC); -+ if (!drv_ctx) -+ return ERR_PTR(-ENOMEM); -+ -+ /* -+ * Initialise pre-header - set RSLS and SDLEN - and shared descriptor -+ * and dma-map them. -+ */ -+ drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) | -+ num_words); -+ memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc)); -+ size = sizeof(drv_ctx->prehdr) + sizeof(drv_ctx->sh_desc); -+ hwdesc = dma_map_single(qidev, drv_ctx->prehdr, size, -+ DMA_BIDIRECTIONAL); -+ if (dma_mapping_error(qidev, hwdesc)) { -+ dev_err(qidev, "DMA map error for preheader + shdesc\n"); -+ kfree(drv_ctx); -+ return ERR_PTR(-ENOMEM); -+ } -+ drv_ctx->context_a = hwdesc; -+ -+ /* If given CPU does not own the portal, choose another one that does */ -+ if (!cpumask_test_cpu(*cpu, cpus)) { -+ int *pcpu = &get_cpu_var(last_cpu); -+ -+ *pcpu = cpumask_next(*pcpu, cpus); -+ if (*pcpu >= nr_cpu_ids) -+ *pcpu = cpumask_first(cpus); -+ *cpu = *pcpu; -+ -+ put_cpu_var(last_cpu); -+ } -+ drv_ctx->cpu = *cpu; -+ -+ /* Find response FQ hooked with this CPU */ -+ drv_ctx->rsp_fq = per_cpu(pcpu_qipriv.rsp_fq, drv_ctx->cpu); -+ -+ /* Attach request FQ */ -+ drv_ctx->req_fq = create_caam_req_fq(qidev, drv_ctx->rsp_fq, hwdesc, -+ QMAN_INITFQ_FLAG_SCHED); -+ if (unlikely(IS_ERR_OR_NULL(drv_ctx->req_fq))) { -+ dev_err(qidev, "create_caam_req_fq failed\n"); -+ dma_unmap_single(qidev, hwdesc, size, DMA_BIDIRECTIONAL); -+ kfree(drv_ctx); -+ return ERR_PTR(-ENOMEM); -+ } -+ -+ drv_ctx->qidev = qidev; -+ return drv_ctx; -+} -+EXPORT_SYMBOL(caam_drv_ctx_init); -+ -+void *qi_cache_alloc(gfp_t flags) -+{ -+ return kmem_cache_alloc(qi_cache, flags); -+} -+EXPORT_SYMBOL(qi_cache_alloc); -+ -+void qi_cache_free(void *obj) -+{ -+ kmem_cache_free(qi_cache, obj); -+} -+EXPORT_SYMBOL(qi_cache_free); -+ -+static int caam_qi_poll(struct napi_struct *napi, int budget) -+{ -+ struct caam_napi *np = container_of(napi, struct caam_napi, irqtask); -+ -+ int cleaned = qman_p_poll_dqrr(np->p, budget); -+ -+ if (cleaned < budget) { -+ napi_complete(napi); -+ qman_p_irqsource_add(np->p, QM_PIRQ_DQRI); -+ } -+ -+ return cleaned; -+} -+ -+void caam_drv_ctx_rel(struct caam_drv_ctx *drv_ctx) -+{ -+ if (IS_ERR_OR_NULL(drv_ctx)) -+ return; -+ -+ /* Remove request FQ */ -+ if (kill_fq(drv_ctx->qidev, drv_ctx->req_fq)) -+ dev_err(drv_ctx->qidev, "Crypto session req FQ kill failed\n"); -+ -+ dma_unmap_single(drv_ctx->qidev, drv_ctx->context_a, -+ sizeof(drv_ctx->sh_desc) + sizeof(drv_ctx->prehdr), -+ DMA_BIDIRECTIONAL); -+ kfree(drv_ctx); -+} -+EXPORT_SYMBOL(caam_drv_ctx_rel); -+ -+int caam_qi_shutdown(struct device *qidev) -+{ -+ int i, ret; -+ struct caam_qi_priv *priv = dev_get_drvdata(qidev); -+ const cpumask_t *cpus = qman_affine_cpus(); -+ struct cpumask old_cpumask = current->cpus_allowed; -+ -+ for_each_cpu(i, cpus) { -+ struct napi_struct *irqtask; -+ -+ irqtask = &per_cpu_ptr(&pcpu_qipriv.caam_napi, i)->irqtask; -+ napi_disable(irqtask); -+ netif_napi_del(irqtask); -+ -+ if (kill_fq(qidev, per_cpu(pcpu_qipriv.rsp_fq, i))) -+ dev_err(qidev, "Rsp FQ kill failed, cpu: %d\n", i); -+ } -+ -+ /* -+ * QMan driver requires CGRs to be deleted from same CPU from where they -+ * were instantiated. Hence we get the module removal execute from the -+ * same CPU from where it was originally inserted. -+ */ -+ set_cpus_allowed_ptr(current, get_cpu_mask(mod_init_cpu)); -+ -+ ret = qman_delete_cgr(&priv->cgr); -+ if (ret) -+ dev_err(qidev, "Deletion of CGR failed: %d\n", ret); -+ else -+ qman_release_cgrid(priv->cgr.cgrid); -+ -+ kmem_cache_destroy(qi_cache); -+ -+ /* Now that we're done with the CGRs, restore the cpus allowed mask */ -+ set_cpus_allowed_ptr(current, &old_cpumask); -+ -+ platform_device_unregister(priv->qi_pdev); -+ return ret; -+} -+ -+static void cgr_cb(struct qman_portal *qm, struct qman_cgr *cgr, int congested) -+{ -+ caam_congested = congested; -+ -+ if (congested) { -+#ifdef CONFIG_DEBUG_FS -+ times_congested++; -+#endif -+ pr_debug_ratelimited("CAAM entered congestion\n"); -+ -+ } else { -+ pr_debug_ratelimited("CAAM exited congestion\n"); -+ } -+} -+ -+static int caam_qi_napi_schedule(struct qman_portal *p, struct caam_napi *np) -+{ -+ /* -+ * In case of threaded ISR, for RT kernels in_irq() does not return -+ * appropriate value, so use in_serving_softirq to distinguish between -+ * softirq and irq contexts. -+ */ -+ if (unlikely(in_irq() || !in_serving_softirq())) { -+ /* Disable QMan IRQ source and invoke NAPI */ -+ qman_p_irqsource_remove(p, QM_PIRQ_DQRI); -+ np->p = p; -+ napi_schedule(&np->irqtask); -+ return 1; -+ } -+ return 0; -+} -+ -+static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p, -+ struct qman_fq *rsp_fq, -+ const struct qm_dqrr_entry *dqrr) -+{ -+ struct caam_napi *caam_napi = raw_cpu_ptr(&pcpu_qipriv.caam_napi); -+ struct caam_drv_req *drv_req; -+ const struct qm_fd *fd; -+ struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev); -+ -+ if (caam_qi_napi_schedule(p, caam_napi)) -+ return qman_cb_dqrr_stop; -+ -+ fd = &dqrr->fd; -+ if (unlikely(fd->status)) { -+ u32 ssrc = fd->status & JRSTA_SSRC_MASK; -+ u8 err_id = fd->status & JRSTA_CCBERR_ERRID_MASK; -+ -+ if (ssrc != JRSTA_SSRC_CCB_ERROR || -+ err_id != JRSTA_CCBERR_ERRID_ICVCHK) -+ dev_err(qidev, "Error: %#x in CAAM response FD\n", -+ fd->status); -+ } -+ -+ if (unlikely(fd->format != fd->format)) { -+ dev_err(qidev, "Non-compound FD from CAAM\n"); -+ return qman_cb_dqrr_consume; -+ } -+ -+ drv_req = (struct caam_drv_req *)phys_to_virt(fd->addr); -+ if (unlikely(!drv_req)) { -+ dev_err(qidev, -+ "Can't find original request for caam response\n"); -+ return qman_cb_dqrr_consume; -+ } -+ -+ dma_unmap_single(drv_req->drv_ctx->qidev, qm_fd_addr(fd), -+ sizeof(drv_req->fd_sgt), DMA_BIDIRECTIONAL); -+ -+ drv_req->cbk(drv_req, fd->status); -+ return qman_cb_dqrr_consume; -+} -+ -+static int alloc_rsp_fq_cpu(struct device *qidev, unsigned int cpu) -+{ -+ struct qm_mcc_initfq opts; -+ struct qman_fq *fq; -+ int ret; -+ -+ fq = kzalloc(sizeof(*fq), GFP_KERNEL | GFP_DMA); -+ if (!fq) -+ return -ENOMEM; -+ -+ fq->cb.dqrr = caam_rsp_fq_dqrr_cb; -+ -+ ret = qman_create_fq(0, QMAN_FQ_FLAG_NO_ENQUEUE | -+ QMAN_FQ_FLAG_DYNAMIC_FQID, fq); -+ if (ret) { -+ dev_err(qidev, "Rsp FQ create failed\n"); -+ kfree(fq); -+ return -ENODEV; -+ } -+ -+ opts.we_mask = QM_INITFQ_WE_FQCTRL | QM_INITFQ_WE_DESTWQ | -+ QM_INITFQ_WE_CONTEXTB | QM_INITFQ_WE_CONTEXTA | -+ QM_INITFQ_WE_CGID; -+ opts.fqd.fq_ctrl = QM_FQCTRL_CTXASTASHING | QM_FQCTRL_CPCSTASH | -+ QM_FQCTRL_CGE; -+ opts.fqd.dest.channel = qman_affine_channel(cpu); -+ opts.fqd.dest.wq = 3; -+ opts.fqd.cgid = qipriv.cgr.cgrid; -+ opts.fqd.context_a.stashing.exclusive = QM_STASHING_EXCL_CTX | -+ QM_STASHING_EXCL_DATA; -+ opts.fqd.context_a.stashing.data_cl = 1; -+ opts.fqd.context_a.stashing.context_cl = 1; -+ -+ ret = qman_init_fq(fq, QMAN_INITFQ_FLAG_SCHED, &opts); -+ if (ret) { -+ dev_err(qidev, "Rsp FQ init failed\n"); -+ kfree(fq); -+ return -ENODEV; -+ } -+ -+ per_cpu(pcpu_qipriv.rsp_fq, cpu) = fq; -+ -+ dev_dbg(qidev, "Allocated response FQ %u for CPU %u", fq->fqid, cpu); -+ return 0; -+} -+ -+static int init_cgr(struct device *qidev) -+{ -+ int ret; -+ struct qm_mcc_initcgr opts; -+ const u64 cpus = *(u64 *)qman_affine_cpus(); -+ const int num_cpus = hweight64(cpus); -+ const u64 val = num_cpus * MAX_RSP_FQ_BACKLOG_PER_CPU; -+ -+ ret = qman_alloc_cgrid(&qipriv.cgr.cgrid); -+ if (ret) { -+ dev_err(qidev, "CGR alloc failed for rsp FQs: %d\n", ret); -+ return ret; -+ } -+ -+ qipriv.cgr.cb = cgr_cb; -+ memset(&opts, 0, sizeof(opts)); -+ opts.we_mask = QM_CGR_WE_CSCN_EN | QM_CGR_WE_CS_THRES | QM_CGR_WE_MODE; -+ opts.cgr.cscn_en = QM_CGR_EN; -+ opts.cgr.mode = QMAN_CGR_MODE_FRAME; -+ qm_cgr_cs_thres_set64(&opts.cgr.cs_thres, val, 1); -+ -+ ret = qman_create_cgr(&qipriv.cgr, QMAN_CGR_FLAG_USE_INIT, &opts); -+ if (ret) { -+ dev_err(qidev, "Error %d creating CAAM CGRID: %u\n", ret, -+ qipriv.cgr.cgrid); -+ return ret; -+ } -+ -+ dev_dbg(qidev, "Congestion threshold set to %llu\n", val); -+ return 0; -+} -+ -+static int alloc_rsp_fqs(struct device *qidev) -+{ -+ int ret, i; -+ const cpumask_t *cpus = qman_affine_cpus(); -+ -+ /*Now create response FQs*/ -+ for_each_cpu(i, cpus) { -+ ret = alloc_rsp_fq_cpu(qidev, i); -+ if (ret) { -+ dev_err(qidev, "CAAM rsp FQ alloc failed, cpu: %u", i); -+ return ret; -+ } -+ } -+ -+ return 0; -+} -+ -+static void free_rsp_fqs(void) -+{ -+ int i; -+ const cpumask_t *cpus = qman_affine_cpus(); -+ -+ for_each_cpu(i, cpus) -+ kfree(per_cpu(pcpu_qipriv.rsp_fq, i)); -+} -+ -+int caam_qi_init(struct platform_device *caam_pdev) -+{ -+ int err, i; -+ struct platform_device *qi_pdev; -+ struct device *ctrldev = &caam_pdev->dev, *qidev; -+ struct caam_drv_private *ctrlpriv; -+ const cpumask_t *cpus = qman_affine_cpus(); -+ struct cpumask old_cpumask = current->cpus_allowed; -+ static struct platform_device_info qi_pdev_info = { -+ .name = "caam_qi", -+ .id = PLATFORM_DEVID_NONE -+ }; -+ -+ /* -+ * QMAN requires CGRs to be removed from same CPU+portal from where it -+ * was originally allocated. Hence we need to note down the -+ * initialisation CPU and use the same CPU for module exit. -+ * We select the first CPU to from the list of portal owning CPUs. -+ * Then we pin module init to this CPU. -+ */ -+ mod_init_cpu = cpumask_first(cpus); -+ set_cpus_allowed_ptr(current, get_cpu_mask(mod_init_cpu)); -+ -+ qi_pdev_info.parent = ctrldev; -+ qi_pdev_info.dma_mask = dma_get_mask(ctrldev); -+ qi_pdev = platform_device_register_full(&qi_pdev_info); -+ if (IS_ERR(qi_pdev)) -+ return PTR_ERR(qi_pdev); -+ arch_setup_dma_ops(&qi_pdev->dev, 0, 0, NULL, true); -+ -+ ctrlpriv = dev_get_drvdata(ctrldev); -+ qidev = &qi_pdev->dev; -+ -+ qipriv.qi_pdev = qi_pdev; -+ dev_set_drvdata(qidev, &qipriv); -+ -+ /* Initialize the congestion detection */ -+ err = init_cgr(qidev); -+ if (err) { -+ dev_err(qidev, "CGR initialization failed: %d\n", err); -+ platform_device_unregister(qi_pdev); -+ return err; -+ } -+ -+ /* Initialise response FQs */ -+ err = alloc_rsp_fqs(qidev); -+ if (err) { -+ dev_err(qidev, "Can't allocate CAAM response FQs: %d\n", err); -+ free_rsp_fqs(); -+ platform_device_unregister(qi_pdev); -+ return err; -+ } -+ -+ /* -+ * Enable the NAPI contexts on each of the core which has an affine -+ * portal. -+ */ -+ for_each_cpu(i, cpus) { -+ struct caam_qi_pcpu_priv *priv = per_cpu_ptr(&pcpu_qipriv, i); -+ struct caam_napi *caam_napi = &priv->caam_napi; -+ struct napi_struct *irqtask = &caam_napi->irqtask; -+ struct net_device *net_dev = &priv->net_dev; -+ -+ net_dev->dev = *qidev; -+ INIT_LIST_HEAD(&net_dev->napi_list); -+ -+ netif_napi_add(net_dev, irqtask, caam_qi_poll, -+ CAAM_NAPI_WEIGHT); -+ -+ napi_enable(irqtask); -+ } -+ -+ /* Hook up QI device to parent controlling caam device */ -+ ctrlpriv->qidev = qidev; -+ -+ qi_cache = kmem_cache_create("caamqicache", CAAM_QI_MEMCACHE_SIZE, 0, -+ SLAB_CACHE_DMA, NULL); -+ if (!qi_cache) { -+ dev_err(qidev, "Can't allocate CAAM cache\n"); -+ free_rsp_fqs(); -+ platform_device_unregister(qi_pdev); -+ return -ENOMEM; -+ } -+ -+ /* Done with the CGRs; restore the cpus allowed mask */ -+ set_cpus_allowed_ptr(current, &old_cpumask); -+#ifdef CONFIG_DEBUG_FS -+ debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl, -+ ×_congested, &caam_fops_u64_ro); -+#endif -+ dev_info(qidev, "Linux CAAM Queue I/F driver initialised\n"); -+ return 0; -+} ---- /dev/null -+++ b/drivers/crypto/caam/qi.h -@@ -0,0 +1,204 @@ -+/* -+ * Public definitions for the CAAM/QI (Queue Interface) backend. -+ * -+ * Copyright 2013-2016 Freescale Semiconductor, Inc. -+ * Copyright 2016-2017 NXP -+ */ -+ -+#ifndef __QI_H__ -+#define __QI_H__ -+ -+#include <linux/fsl_qman.h> -+#include "compat.h" -+#include "desc.h" -+#include "desc_constr.h" -+ -+/* -+ * CAAM hardware constructs a job descriptor which points to a shared descriptor -+ * (as pointed by context_a of to-CAAM FQ). -+ * When the job descriptor is executed by DECO, the whole job descriptor -+ * together with shared descriptor gets loaded in DECO buffer, which is -+ * 64 words (each 32-bit) long. -+ * -+ * The job descriptor constructed by CAAM hardware has the following layout: -+ * -+ * HEADER (1 word) -+ * Shdesc ptr (1 or 2 words) -+ * SEQ_OUT_PTR (1 word) -+ * Out ptr (1 or 2 words) -+ * Out length (1 word) -+ * SEQ_IN_PTR (1 word) -+ * In ptr (1 or 2 words) -+ * In length (1 word) -+ * -+ * The shdesc ptr is used to fetch shared descriptor contents into DECO buffer. -+ * -+ * Apart from shdesc contents, the total number of words that get loaded in DECO -+ * buffer are '8' or '11'. The remaining words in DECO buffer can be used for -+ * storing shared descriptor. -+ */ -+#define MAX_SDLEN ((CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) / CAAM_CMD_SZ) -+ -+/* Length of a single buffer in the QI driver memory cache */ -+#define CAAM_QI_MEMCACHE_SIZE 768 -+ -+extern bool caam_congested __read_mostly; -+ -+/* -+ * This is the request structure the driver application should fill while -+ * submitting a job to driver. -+ */ -+struct caam_drv_req; -+ -+/* -+ * caam_qi_cbk - application's callback function invoked by the driver when the -+ * request has been successfully processed. -+ * @drv_req: original request that was submitted -+ * @status: completion status of request (0 - success, non-zero - error code) -+ */ -+typedef void (*caam_qi_cbk)(struct caam_drv_req *drv_req, u32 status); -+ -+enum optype { -+ ENCRYPT, -+ DECRYPT, -+ GIVENCRYPT, -+ NUM_OP -+}; -+ -+/** -+ * caam_drv_ctx - CAAM/QI backend driver context -+ * -+ * The jobs are processed by the driver against a driver context. -+ * With every cryptographic context, a driver context is attached. -+ * The driver context contains data for private use by driver. -+ * For the applications, this is an opaque structure. -+ * -+ * @prehdr: preheader placed before shrd desc -+ * @sh_desc: shared descriptor -+ * @context_a: shared descriptor dma address -+ * @req_fq: to-CAAM request frame queue -+ * @rsp_fq: from-CAAM response frame queue -+ * @cpu: cpu on which to receive CAAM response -+ * @op_type: operation type -+ * @qidev: device pointer for CAAM/QI backend -+ */ -+struct caam_drv_ctx { -+ u32 prehdr[2]; -+ u32 sh_desc[MAX_SDLEN]; -+ dma_addr_t context_a; -+ struct qman_fq *req_fq; -+ struct qman_fq *rsp_fq; -+ int cpu; -+ enum optype op_type; -+ struct device *qidev; -+} ____cacheline_aligned; -+ -+/** -+ * caam_drv_req - The request structure the driver application should fill while -+ * submitting a job to driver. -+ * @fd_sgt: QMan S/G pointing to output (fd_sgt[0]) and input (fd_sgt[1]) -+ * buffers. -+ * @cbk: callback function to invoke when job is completed -+ * @app_ctx: arbitrary context attached with request by the application -+ * -+ * The fields mentioned below should not be used by application. -+ * These are for private use by driver. -+ * -+ * @hdr__: linked list header to maintain list of outstanding requests to CAAM -+ * @hwaddr: DMA address for the S/G table. -+ */ -+struct caam_drv_req { -+ struct qm_sg_entry fd_sgt[2]; -+ struct caam_drv_ctx *drv_ctx; -+ caam_qi_cbk cbk; -+ void *app_ctx; -+} ____cacheline_aligned; -+ -+/** -+ * caam_drv_ctx_init - Initialise a CAAM/QI driver context -+ * -+ * A CAAM/QI driver context must be attached with each cryptographic context. -+ * This function allocates memory for CAAM/QI context and returns a handle to -+ * the application. This handle must be submitted along with each enqueue -+ * request to the driver by the application. -+ * -+ * @cpu: CPU where the application prefers to the driver to receive CAAM -+ * responses. The request completion callback would be issued from this -+ * CPU. -+ * @sh_desc: shared descriptor pointer to be attached with CAAM/QI driver -+ * context. -+ * -+ * Returns a driver context on success or negative error code on failure. -+ */ -+struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev, int *cpu, -+ u32 *sh_desc); -+ -+/** -+ * caam_qi_enqueue - Submit a request to QI backend driver. -+ * -+ * The request structure must be properly filled as described above. -+ * -+ * @qidev: device pointer for QI backend -+ * @req: CAAM QI request structure -+ * -+ * Returns 0 on success or negative error code on failure. -+ */ -+int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req); -+ -+/** -+ * caam_drv_ctx_busy - Check if there are too many jobs pending with CAAM -+ * or too many CAAM responses are pending to be processed. -+ * @drv_ctx: driver context for which job is to be submitted -+ * -+ * Returns caam congestion status 'true/false' -+ */ -+bool caam_drv_ctx_busy(struct caam_drv_ctx *drv_ctx); -+ -+/** -+ * caam_drv_ctx_update - Update QI driver context -+ * -+ * Invoked when shared descriptor is required to be change in driver context. -+ * -+ * @drv_ctx: driver context to be updated -+ * @sh_desc: new shared descriptor pointer to be updated in QI driver context -+ * -+ * Returns 0 on success or negative error code on failure. -+ */ -+int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc); -+ -+/** -+ * caam_drv_ctx_rel - Release a QI driver context -+ * @drv_ctx: context to be released -+ */ -+void caam_drv_ctx_rel(struct caam_drv_ctx *drv_ctx); -+ -+int caam_qi_init(struct platform_device *pdev); -+int caam_qi_shutdown(struct device *dev); -+ -+/** -+ * qi_cache_alloc - Allocate buffers from CAAM-QI cache -+ * -+ * Invoked when a user of the CAAM-QI (i.e. caamalg-qi) needs data which has -+ * to be allocated on the hotpath. Instead of using malloc, one can use the -+ * services of the CAAM QI memory cache (backed by kmem_cache). The buffers -+ * will have a size of 256B, which is sufficient for hosting 16 SG entries. -+ * -+ * @flags: flags that would be used for the equivalent malloc(..) call -+ * -+ * Returns a pointer to a retrieved buffer on success or NULL on failure. -+ */ -+void *qi_cache_alloc(gfp_t flags); -+ -+/** -+ * qi_cache_free - Frees buffers allocated from CAAM-QI cache -+ * -+ * Invoked when a user of the CAAM-QI (i.e. caamalg-qi) no longer needs -+ * the buffer previously allocated by a qi_cache_alloc call. -+ * No checking is being done, the call is a passthrough call to -+ * kmem_cache_free(...) -+ * -+ * @obj: object previously allocated using qi_cache_alloc() -+ */ -+void qi_cache_free(void *obj); -+ -+#endif /* __QI_H__ */ ---- a/drivers/crypto/caam/regs.h -+++ b/drivers/crypto/caam/regs.h -@@ -2,6 +2,7 @@ - * CAAM hardware register-level view - * - * Copyright 2008-2011 Freescale Semiconductor, Inc. -+ * Copyright 2017 NXP - */ - - #ifndef REGS_H -@@ -67,6 +68,7 @@ - */ - - extern bool caam_little_end; -+extern bool caam_imx; - - #define caam_to_cpu(len) \ - static inline u##len caam##len ## _to_cpu(u##len val) \ -@@ -154,13 +156,10 @@ static inline u64 rd_reg64(void __iomem - #else /* CONFIG_64BIT */ - static inline void wr_reg64(void __iomem *reg, u64 data) - { --#ifndef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX -- if (caam_little_end) { -+ if (!caam_imx && caam_little_end) { - wr_reg32((u32 __iomem *)(reg) + 1, data >> 32); - wr_reg32((u32 __iomem *)(reg), data); -- } else --#endif -- { -+ } else { - wr_reg32((u32 __iomem *)(reg), data >> 32); - wr_reg32((u32 __iomem *)(reg) + 1, data); - } -@@ -168,41 +167,40 @@ static inline void wr_reg64(void __iomem - - static inline u64 rd_reg64(void __iomem *reg) - { --#ifndef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX -- if (caam_little_end) -+ if (!caam_imx && caam_little_end) - return ((u64)rd_reg32((u32 __iomem *)(reg) + 1) << 32 | - (u64)rd_reg32((u32 __iomem *)(reg))); -- else --#endif -- return ((u64)rd_reg32((u32 __iomem *)(reg)) << 32 | -- (u64)rd_reg32((u32 __iomem *)(reg) + 1)); -+ -+ return ((u64)rd_reg32((u32 __iomem *)(reg)) << 32 | -+ (u64)rd_reg32((u32 __iomem *)(reg) + 1)); - } - #endif /* CONFIG_64BIT */ - -+static inline u64 cpu_to_caam_dma64(dma_addr_t value) -+{ -+ if (caam_imx) -+ return (((u64)cpu_to_caam32(lower_32_bits(value)) << 32) | -+ (u64)cpu_to_caam32(upper_32_bits(value))); -+ -+ return cpu_to_caam64(value); -+} -+ -+static inline u64 caam_dma64_to_cpu(u64 value) -+{ -+ if (caam_imx) -+ return (((u64)caam32_to_cpu(lower_32_bits(value)) << 32) | -+ (u64)caam32_to_cpu(upper_32_bits(value))); -+ -+ return caam64_to_cpu(value); -+} -+ - #ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT --#ifdef CONFIG_SOC_IMX7D --#define cpu_to_caam_dma(value) \ -- (((u64)cpu_to_caam32(lower_32_bits(value)) << 32) | \ -- (u64)cpu_to_caam32(upper_32_bits(value))) --#define caam_dma_to_cpu(value) \ -- (((u64)caam32_to_cpu(lower_32_bits(value)) << 32) | \ -- (u64)caam32_to_cpu(upper_32_bits(value))) --#else --#define cpu_to_caam_dma(value) cpu_to_caam64(value) --#define caam_dma_to_cpu(value) caam64_to_cpu(value) --#endif /* CONFIG_SOC_IMX7D */ -+#define cpu_to_caam_dma(value) cpu_to_caam_dma64(value) -+#define caam_dma_to_cpu(value) caam_dma64_to_cpu(value) - #else - #define cpu_to_caam_dma(value) cpu_to_caam32(value) - #define caam_dma_to_cpu(value) caam32_to_cpu(value) --#endif /* CONFIG_ARCH_DMA_ADDR_T_64BIT */ -- --#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_IMX --#define cpu_to_caam_dma64(value) \ -- (((u64)cpu_to_caam32(lower_32_bits(value)) << 32) | \ -- (u64)cpu_to_caam32(upper_32_bits(value))) --#else --#define cpu_to_caam_dma64(value) cpu_to_caam64(value) --#endif -+#endif /* CONFIG_ARCH_DMA_ADDR_T_64BIT */ - - /* - * jr_outentry -@@ -293,6 +291,7 @@ struct caam_perfmon { - u32 cha_rev_ls; /* CRNR - CHA Rev No. Least significant half*/ - #define CTPR_MS_QI_SHIFT 25 - #define CTPR_MS_QI_MASK (0x1ull << CTPR_MS_QI_SHIFT) -+#define CTPR_MS_DPAA2 BIT(13) - #define CTPR_MS_VIRT_EN_INCL 0x00000001 - #define CTPR_MS_VIRT_EN_POR 0x00000002 - #define CTPR_MS_PG_SZ_MASK 0x10 -@@ -628,6 +627,8 @@ struct caam_job_ring { - #define JRSTA_DECOERR_INVSIGN 0x86 - #define JRSTA_DECOERR_DSASIGN 0x87 - -+#define JRSTA_QIERR_ERROR_MASK 0x00ff -+ - #define JRSTA_CCBERR_JUMP 0x08000000 - #define JRSTA_CCBERR_INDEX_MASK 0xff00 - #define JRSTA_CCBERR_INDEX_SHIFT 8 ---- /dev/null -+++ b/drivers/crypto/caam/sg_sw_qm.h -@@ -0,0 +1,126 @@ -+/* -+ * Copyright 2013-2016 Freescale Semiconductor, Inc. -+ * Copyright 2016-2017 NXP -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the name of Freescale Semiconductor nor the -+ * names of its contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY -+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -+ * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY -+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#ifndef __SG_SW_QM_H -+#define __SG_SW_QM_H -+ -+#include <linux/fsl_qman.h> -+#include "regs.h" -+ -+static inline void cpu_to_hw_sg(struct qm_sg_entry *qm_sg_ptr) -+{ -+ dma_addr_t addr = qm_sg_ptr->opaque; -+ -+ qm_sg_ptr->opaque = cpu_to_caam64(addr); -+ qm_sg_ptr->sgt_efl = cpu_to_caam32(qm_sg_ptr->sgt_efl); -+} -+ -+static inline void __dma_to_qm_sg(struct qm_sg_entry *qm_sg_ptr, dma_addr_t dma, -+ u32 len, u16 offset) -+{ -+ qm_sg_ptr->addr = dma; -+ qm_sg_ptr->length = len; -+ qm_sg_ptr->__reserved2 = 0; -+ qm_sg_ptr->bpid = 0; -+ qm_sg_ptr->__reserved3 = 0; -+ qm_sg_ptr->offset = offset & QM_SG_OFFSET_MASK; -+ -+ cpu_to_hw_sg(qm_sg_ptr); -+} -+ -+static inline void dma_to_qm_sg_one(struct qm_sg_entry *qm_sg_ptr, -+ dma_addr_t dma, u32 len, u16 offset) -+{ -+ qm_sg_ptr->extension = 0; -+ qm_sg_ptr->final = 0; -+ __dma_to_qm_sg(qm_sg_ptr, dma, len, offset); -+} -+ -+static inline void dma_to_qm_sg_one_last(struct qm_sg_entry *qm_sg_ptr, -+ dma_addr_t dma, u32 len, u16 offset) -+{ -+ qm_sg_ptr->extension = 0; -+ qm_sg_ptr->final = 1; -+ __dma_to_qm_sg(qm_sg_ptr, dma, len, offset); -+} -+ -+static inline void dma_to_qm_sg_one_ext(struct qm_sg_entry *qm_sg_ptr, -+ dma_addr_t dma, u32 len, u16 offset) -+{ -+ qm_sg_ptr->extension = 1; -+ qm_sg_ptr->final = 0; -+ __dma_to_qm_sg(qm_sg_ptr, dma, len, offset); -+} -+ -+static inline void dma_to_qm_sg_one_last_ext(struct qm_sg_entry *qm_sg_ptr, -+ dma_addr_t dma, u32 len, -+ u16 offset) -+{ -+ qm_sg_ptr->extension = 1; -+ qm_sg_ptr->final = 1; -+ __dma_to_qm_sg(qm_sg_ptr, dma, len, offset); -+} -+ -+/* -+ * convert scatterlist to h/w link table format -+ * but does not have final bit; instead, returns last entry -+ */ -+static inline struct qm_sg_entry * -+sg_to_qm_sg(struct scatterlist *sg, int sg_count, -+ struct qm_sg_entry *qm_sg_ptr, u16 offset) -+{ -+ while (sg_count && sg) { -+ dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), -+ sg_dma_len(sg), offset); -+ qm_sg_ptr++; -+ sg = sg_next(sg); -+ sg_count--; -+ } -+ return qm_sg_ptr - 1; -+} -+ -+/* -+ * convert scatterlist to h/w link table format -+ * scatterlist must have been previously dma mapped -+ */ -+static inline void sg_to_qm_sg_last(struct scatterlist *sg, int sg_count, -+ struct qm_sg_entry *qm_sg_ptr, u16 offset) -+{ -+ qm_sg_ptr = sg_to_qm_sg(sg, sg_count, qm_sg_ptr, offset); -+ -+ qm_sg_ptr->sgt_efl = caam32_to_cpu(qm_sg_ptr->sgt_efl); -+ qm_sg_ptr->final = 1; -+ qm_sg_ptr->sgt_efl = cpu_to_caam32(qm_sg_ptr->sgt_efl); -+} -+ -+#endif /* __SG_SW_QM_H */ ---- /dev/null -+++ b/drivers/crypto/caam/sg_sw_qm2.h -@@ -0,0 +1,81 @@ -+/* -+ * Copyright 2015-2016 Freescale Semiconductor, Inc. -+ * Copyright 2017 NXP -+ * -+ * Redistribution and use in source and binary forms, with or without -+ * modification, are permitted provided that the following conditions are met: -+ * * Redistributions of source code must retain the above copyright -+ * notice, this list of conditions and the following disclaimer. -+ * * Redistributions in binary form must reproduce the above copyright -+ * notice, this list of conditions and the following disclaimer in the -+ * documentation and/or other materials provided with the distribution. -+ * * Neither the names of the above-listed copyright holders nor the -+ * names of any contributors may be used to endorse or promote products -+ * derived from this software without specific prior written permission. -+ * -+ * -+ * ALTERNATIVELY, this software may be distributed under the terms of the -+ * GNU General Public License ("GPL") as published by the Free Software -+ * Foundation, either version 2 of that License or (at your option) any -+ * later version. -+ * -+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE -+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -+ * POSSIBILITY OF SUCH DAMAGE. -+ */ -+ -+#ifndef _SG_SW_QM2_H_ -+#define _SG_SW_QM2_H_ -+ -+#include "../../../drivers/staging/fsl-mc/include/dpaa2-fd.h" -+ -+static inline void dma_to_qm_sg_one(struct dpaa2_sg_entry *qm_sg_ptr, -+ dma_addr_t dma, u32 len, u16 offset) -+{ -+ dpaa2_sg_set_addr(qm_sg_ptr, dma); -+ dpaa2_sg_set_format(qm_sg_ptr, dpaa2_sg_single); -+ dpaa2_sg_set_final(qm_sg_ptr, false); -+ dpaa2_sg_set_len(qm_sg_ptr, len); -+ dpaa2_sg_set_bpid(qm_sg_ptr, 0); -+ dpaa2_sg_set_offset(qm_sg_ptr, offset); -+} -+ -+/* -+ * convert scatterlist to h/w link table format -+ * but does not have final bit; instead, returns last entry -+ */ -+static inline struct dpaa2_sg_entry * -+sg_to_qm_sg(struct scatterlist *sg, int sg_count, -+ struct dpaa2_sg_entry *qm_sg_ptr, u16 offset) -+{ -+ while (sg_count && sg) { -+ dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), -+ sg_dma_len(sg), offset); -+ qm_sg_ptr++; -+ sg = sg_next(sg); -+ sg_count--; -+ } -+ return qm_sg_ptr - 1; -+} -+ -+/* -+ * convert scatterlist to h/w link table format -+ * scatterlist must have been previously dma mapped -+ */ -+static inline void sg_to_qm_sg_last(struct scatterlist *sg, int sg_count, -+ struct dpaa2_sg_entry *qm_sg_ptr, -+ u16 offset) -+{ -+ qm_sg_ptr = sg_to_qm_sg(sg, sg_count, qm_sg_ptr, offset); -+ dpaa2_sg_set_final(qm_sg_ptr, true); -+} -+ -+#endif /* _SG_SW_QM2_H_ */ ---- a/drivers/crypto/caam/sg_sw_sec4.h -+++ b/drivers/crypto/caam/sg_sw_sec4.h -@@ -5,9 +5,19 @@ - * - */ - -+#ifndef _SG_SW_SEC4_H_ -+#define _SG_SW_SEC4_H_ -+ -+#include "ctrl.h" - #include "regs.h" -+#include "sg_sw_qm2.h" -+#include "../../../drivers/staging/fsl-mc/include/dpaa2-fd.h" - --struct sec4_sg_entry; -+struct sec4_sg_entry { -+ u64 ptr; -+ u32 len; -+ u32 bpid_offset; -+}; - - /* - * convert single dma address to h/w link table format -@@ -15,9 +25,15 @@ struct sec4_sg_entry; - static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr, - dma_addr_t dma, u32 len, u16 offset) - { -- sec4_sg_ptr->ptr = cpu_to_caam_dma64(dma); -- sec4_sg_ptr->len = cpu_to_caam32(len); -- sec4_sg_ptr->bpid_offset = cpu_to_caam32(offset & SEC4_SG_OFFSET_MASK); -+ if (caam_dpaa2) { -+ dma_to_qm_sg_one((struct dpaa2_sg_entry *)sec4_sg_ptr, dma, len, -+ offset); -+ } else { -+ sec4_sg_ptr->ptr = cpu_to_caam_dma64(dma); -+ sec4_sg_ptr->len = cpu_to_caam32(len); -+ sec4_sg_ptr->bpid_offset = cpu_to_caam32(offset & -+ SEC4_SG_OFFSET_MASK); -+ } - #ifdef DEBUG - print_hex_dump(KERN_ERR, "sec4_sg_ptr@: ", - DUMP_PREFIX_ADDRESS, 16, 4, sec4_sg_ptr, -@@ -43,6 +59,14 @@ sg_to_sec4_sg(struct scatterlist *sg, in - return sec4_sg_ptr - 1; - } - -+static inline void sg_to_sec4_set_last(struct sec4_sg_entry *sec4_sg_ptr) -+{ -+ if (caam_dpaa2) -+ dpaa2_sg_set_final((struct dpaa2_sg_entry *)sec4_sg_ptr, true); -+ else -+ sec4_sg_ptr->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); -+} -+ - /* - * convert scatterlist to h/w link table format - * scatterlist must have been previously dma mapped -@@ -52,31 +76,7 @@ static inline void sg_to_sec4_sg_last(st - u16 offset) - { - sec4_sg_ptr = sg_to_sec4_sg(sg, sg_count, sec4_sg_ptr, offset); -- sec4_sg_ptr->len |= cpu_to_caam32(SEC4_SG_LEN_FIN); --} -- --static inline struct sec4_sg_entry *sg_to_sec4_sg_len( -- struct scatterlist *sg, unsigned int total, -- struct sec4_sg_entry *sec4_sg_ptr) --{ -- do { -- unsigned int len = min(sg_dma_len(sg), total); -- -- dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg), len, 0); -- sec4_sg_ptr++; -- sg = sg_next(sg); -- total -= len; -- } while (total); -- return sec4_sg_ptr - 1; -+ sg_to_sec4_set_last(sec4_sg_ptr); - } - --/* derive number of elements in scatterlist, but return 0 for 1 */ --static inline int sg_count(struct scatterlist *sg_list, int nbytes) --{ -- int sg_nents = sg_nents_for_len(sg_list, nbytes); -- -- if (likely(sg_nents == 1)) -- return 0; -- -- return sg_nents; --} -+#endif /* _SG_SW_SEC4_H_ */ ---- a/drivers/crypto/talitos.c -+++ b/drivers/crypto/talitos.c -@@ -1241,6 +1241,14 @@ static int ipsec_esp(struct talitos_edes - ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4], - sg_count, areq->assoclen, tbl_off, elen); - -+ /* -+ * In case of SEC 2.x+, cipher in len must include only the ciphertext, -+ * while extent is used for ICV len. -+ */ -+ if ((edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) && -+ (desc->hdr & DESC_HDR_MODE1_MDEU_CICV)) -+ desc->ptr[4].len = cpu_to_be16(cryptlen); -+ - if (ret > 1) { - tbl_off += ret; - sync_needed = true; ---- a/drivers/net/wireless/rsi/rsi_91x_usb.c -+++ b/drivers/net/wireless/rsi/rsi_91x_usb.c -@@ -516,7 +516,7 @@ err: - - /** - * rsi_disconnect() - This function performs the reverse of the probe function, -- * it deintialize the driver structure. -+ * it deinitialize the driver structure. - * @pfunction: Pointer to the USB interface structure. - * - * Return: None. ---- a/drivers/staging/wilc1000/linux_wlan.c -+++ b/drivers/staging/wilc1000/linux_wlan.c -@@ -211,7 +211,7 @@ static void deinit_irq(struct net_device - vif = netdev_priv(dev); - wilc = vif->wilc; - -- /* Deintialize IRQ */ -+ /* Deinitialize IRQ */ - if (wilc->dev_irq_num) { - free_irq(wilc->dev_irq_num, wilc); - gpio_free(wilc->gpio); ---- a/drivers/staging/wilc1000/wilc_wfi_cfgoperations.c -+++ b/drivers/staging/wilc1000/wilc_wfi_cfgoperations.c -@@ -2359,7 +2359,7 @@ int wilc_deinit_host_int(struct net_devi - del_timer_sync(&wilc_during_ip_timer); - - if (s32Error) -- netdev_err(net, "Error while deintializing host interface\n"); -+ netdev_err(net, "Error while deinitializing host interface\n"); - - return s32Error; - } ---- /dev/null -+++ b/include/crypto/acompress.h -@@ -0,0 +1,269 @@ -+/* -+ * Asynchronous Compression operations -+ * -+ * Copyright (c) 2016, Intel Corporation -+ * Authors: Weigang Li <weigang.li@intel.com> -+ * Giovanni Cabiddu <giovanni.cabiddu@intel.com> -+ * -+ * This program is free software; you can redistribute it and/or modify it -+ * under the terms of the GNU General Public License as published by the Free -+ * Software Foundation; either version 2 of the License, or (at your option) -+ * any later version. -+ * -+ */ -+#ifndef _CRYPTO_ACOMP_H -+#define _CRYPTO_ACOMP_H -+#include <linux/crypto.h> -+ -+#define CRYPTO_ACOMP_ALLOC_OUTPUT 0x00000001 -+ -+/** -+ * struct acomp_req - asynchronous (de)compression request -+ * -+ * @base: Common attributes for asynchronous crypto requests -+ * @src: Source Data -+ * @dst: Destination data -+ * @slen: Size of the input buffer -+ * @dlen: Size of the output buffer and number of bytes produced -+ * @flags: Internal flags -+ * @__ctx: Start of private context data -+ */ -+struct acomp_req { -+ struct crypto_async_request base; -+ struct scatterlist *src; -+ struct scatterlist *dst; -+ unsigned int slen; -+ unsigned int dlen; -+ u32 flags; -+ void *__ctx[] CRYPTO_MINALIGN_ATTR; -+}; -+ -+/** -+ * struct crypto_acomp - user-instantiated objects which encapsulate -+ * algorithms and core processing logic -+ * -+ * @compress: Function performs a compress operation -+ * @decompress: Function performs a de-compress operation -+ * @dst_free: Frees destination buffer if allocated inside the -+ * algorithm -+ * @reqsize: Context size for (de)compression requests -+ * @base: Common crypto API algorithm data structure -+ */ -+struct crypto_acomp { -+ int (*compress)(struct acomp_req *req); -+ int (*decompress)(struct acomp_req *req); -+ void (*dst_free)(struct scatterlist *dst); -+ unsigned int reqsize; -+ struct crypto_tfm base; -+}; -+ -+/** -+ * struct acomp_alg - asynchronous compression algorithm -+ * -+ * @compress: Function performs a compress operation -+ * @decompress: Function performs a de-compress operation -+ * @dst_free: Frees destination buffer if allocated inside the algorithm -+ * @init: Initialize the cryptographic transformation object. -+ * This function is used to initialize the cryptographic -+ * transformation object. This function is called only once at -+ * the instantiation time, right after the transformation context -+ * was allocated. In case the cryptographic hardware has some -+ * special requirements which need to be handled by software, this -+ * function shall check for the precise requirement of the -+ * transformation and put any software fallbacks in place. -+ * @exit: Deinitialize the cryptographic transformation object. This is a -+ * counterpart to @init, used to remove various changes set in -+ * @init. -+ * -+ * @reqsize: Context size for (de)compression requests -+ * @base: Common crypto API algorithm data structure -+ */ -+struct acomp_alg { -+ int (*compress)(struct acomp_req *req); -+ int (*decompress)(struct acomp_req *req); -+ void (*dst_free)(struct scatterlist *dst); -+ int (*init)(struct crypto_acomp *tfm); -+ void (*exit)(struct crypto_acomp *tfm); -+ unsigned int reqsize; -+ struct crypto_alg base; -+}; -+ -+/** -+ * DOC: Asynchronous Compression API -+ * -+ * The Asynchronous Compression API is used with the algorithms of type -+ * CRYPTO_ALG_TYPE_ACOMPRESS (listed as type "acomp" in /proc/crypto) -+ */ -+ -+/** -+ * crypto_alloc_acomp() -- allocate ACOMPRESS tfm handle -+ * @alg_name: is the cra_name / name or cra_driver_name / driver name of the -+ * compression algorithm e.g. "deflate" -+ * @type: specifies the type of the algorithm -+ * @mask: specifies the mask for the algorithm -+ * -+ * Allocate a handle for a compression algorithm. The returned struct -+ * crypto_acomp is the handle that is required for any subsequent -+ * API invocation for the compression operations. -+ * -+ * Return: allocated handle in case of success; IS_ERR() is true in case -+ * of an error, PTR_ERR() returns the error code. -+ */ -+struct crypto_acomp *crypto_alloc_acomp(const char *alg_name, u32 type, -+ u32 mask); -+ -+static inline struct crypto_tfm *crypto_acomp_tfm(struct crypto_acomp *tfm) -+{ -+ return &tfm->base; -+} -+ -+static inline struct acomp_alg *__crypto_acomp_alg(struct crypto_alg *alg) -+{ -+ return container_of(alg, struct acomp_alg, base); -+} -+ -+static inline struct crypto_acomp *__crypto_acomp_tfm(struct crypto_tfm *tfm) -+{ -+ return container_of(tfm, struct crypto_acomp, base); -+} -+ -+static inline struct acomp_alg *crypto_acomp_alg(struct crypto_acomp *tfm) -+{ -+ return __crypto_acomp_alg(crypto_acomp_tfm(tfm)->__crt_alg); -+} -+ -+static inline unsigned int crypto_acomp_reqsize(struct crypto_acomp *tfm) -+{ -+ return tfm->reqsize; -+} -+ -+static inline void acomp_request_set_tfm(struct acomp_req *req, -+ struct crypto_acomp *tfm) -+{ -+ req->base.tfm = crypto_acomp_tfm(tfm); -+} -+ -+static inline struct crypto_acomp *crypto_acomp_reqtfm(struct acomp_req *req) -+{ -+ return __crypto_acomp_tfm(req->base.tfm); -+} -+ -+/** -+ * crypto_free_acomp() -- free ACOMPRESS tfm handle -+ * -+ * @tfm: ACOMPRESS tfm handle allocated with crypto_alloc_acomp() -+ */ -+static inline void crypto_free_acomp(struct crypto_acomp *tfm) -+{ -+ crypto_destroy_tfm(tfm, crypto_acomp_tfm(tfm)); -+} -+ -+static inline int crypto_has_acomp(const char *alg_name, u32 type, u32 mask) -+{ -+ type &= ~CRYPTO_ALG_TYPE_MASK; -+ type |= CRYPTO_ALG_TYPE_ACOMPRESS; -+ mask |= CRYPTO_ALG_TYPE_MASK; -+ -+ return crypto_has_alg(alg_name, type, mask); -+} -+ -+/** -+ * acomp_request_alloc() -- allocates asynchronous (de)compression request -+ * -+ * @tfm: ACOMPRESS tfm handle allocated with crypto_alloc_acomp() -+ * -+ * Return: allocated handle in case of success or NULL in case of an error -+ */ -+struct acomp_req *acomp_request_alloc(struct crypto_acomp *tfm); -+ -+/** -+ * acomp_request_free() -- zeroize and free asynchronous (de)compression -+ * request as well as the output buffer if allocated -+ * inside the algorithm -+ * -+ * @req: request to free -+ */ -+void acomp_request_free(struct acomp_req *req); -+ -+/** -+ * acomp_request_set_callback() -- Sets an asynchronous callback -+ * -+ * Callback will be called when an asynchronous operation on a given -+ * request is finished. -+ * -+ * @req: request that the callback will be set for -+ * @flgs: specify for instance if the operation may backlog -+ * @cmlp: callback which will be called -+ * @data: private data used by the caller -+ */ -+static inline void acomp_request_set_callback(struct acomp_req *req, -+ u32 flgs, -+ crypto_completion_t cmpl, -+ void *data) -+{ -+ req->base.complete = cmpl; -+ req->base.data = data; -+ req->base.flags = flgs; -+} -+ -+/** -+ * acomp_request_set_params() -- Sets request parameters -+ * -+ * Sets parameters required by an acomp operation -+ * -+ * @req: asynchronous compress request -+ * @src: pointer to input buffer scatterlist -+ * @dst: pointer to output buffer scatterlist. If this is NULL, the -+ * acomp layer will allocate the output memory -+ * @slen: size of the input buffer -+ * @dlen: size of the output buffer. If dst is NULL, this can be used by -+ * the user to specify the maximum amount of memory to allocate -+ */ -+static inline void acomp_request_set_params(struct acomp_req *req, -+ struct scatterlist *src, -+ struct scatterlist *dst, -+ unsigned int slen, -+ unsigned int dlen) -+{ -+ req->src = src; -+ req->dst = dst; -+ req->slen = slen; -+ req->dlen = dlen; -+ -+ if (!req->dst) -+ req->flags |= CRYPTO_ACOMP_ALLOC_OUTPUT; -+} -+ -+/** -+ * crypto_acomp_compress() -- Invoke asynchronous compress operation -+ * -+ * Function invokes the asynchronous compress operation -+ * -+ * @req: asynchronous compress request -+ * -+ * Return: zero on success; error code in case of error -+ */ -+static inline int crypto_acomp_compress(struct acomp_req *req) -+{ -+ struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); -+ -+ return tfm->compress(req); -+} -+ -+/** -+ * crypto_acomp_decompress() -- Invoke asynchronous decompress operation -+ * -+ * Function invokes the asynchronous decompress operation -+ * -+ * @req: asynchronous compress request -+ * -+ * Return: zero on success; error code in case of error -+ */ -+static inline int crypto_acomp_decompress(struct acomp_req *req) -+{ -+ struct crypto_acomp *tfm = crypto_acomp_reqtfm(req); -+ -+ return tfm->decompress(req); -+} -+ -+#endif ---- /dev/null -+++ b/include/crypto/internal/acompress.h -@@ -0,0 +1,81 @@ -+/* -+ * Asynchronous Compression operations -+ * -+ * Copyright (c) 2016, Intel Corporation -+ * Authors: Weigang Li <weigang.li@intel.com> -+ * Giovanni Cabiddu <giovanni.cabiddu@intel.com> -+ * -+ * This program is free software; you can redistribute it and/or modify it -+ * under the terms of the GNU General Public License as published by the Free -+ * Software Foundation; either version 2 of the License, or (at your option) -+ * any later version. -+ * -+ */ -+#ifndef _CRYPTO_ACOMP_INT_H -+#define _CRYPTO_ACOMP_INT_H -+#include <crypto/acompress.h> -+ -+/* -+ * Transform internal helpers. -+ */ -+static inline void *acomp_request_ctx(struct acomp_req *req) -+{ -+ return req->__ctx; -+} -+ -+static inline void *acomp_tfm_ctx(struct crypto_acomp *tfm) -+{ -+ return tfm->base.__crt_ctx; -+} -+ -+static inline void acomp_request_complete(struct acomp_req *req, -+ int err) -+{ -+ req->base.complete(&req->base, err); -+} -+ -+static inline const char *acomp_alg_name(struct crypto_acomp *tfm) -+{ -+ return crypto_acomp_tfm(tfm)->__crt_alg->cra_name; -+} -+ -+static inline struct acomp_req *__acomp_request_alloc(struct crypto_acomp *tfm) -+{ -+ struct acomp_req *req; -+ -+ req = kzalloc(sizeof(*req) + crypto_acomp_reqsize(tfm), GFP_KERNEL); -+ if (likely(req)) -+ acomp_request_set_tfm(req, tfm); -+ return req; -+} -+ -+static inline void __acomp_request_free(struct acomp_req *req) -+{ -+ kzfree(req); -+} -+ -+/** -+ * crypto_register_acomp() -- Register asynchronous compression algorithm -+ * -+ * Function registers an implementation of an asynchronous -+ * compression algorithm -+ * -+ * @alg: algorithm definition -+ * -+ * Return: zero on success; error code in case of error -+ */ -+int crypto_register_acomp(struct acomp_alg *alg); -+ -+/** -+ * crypto_unregister_acomp() -- Unregister asynchronous compression algorithm -+ * -+ * Function unregisters an implementation of an asynchronous -+ * compression algorithm -+ * -+ * @alg: algorithm definition -+ * -+ * Return: zero on success; error code in case of error -+ */ -+int crypto_unregister_acomp(struct acomp_alg *alg); -+ -+#endif ---- /dev/null -+++ b/include/crypto/internal/scompress.h -@@ -0,0 +1,136 @@ -+/* -+ * Synchronous Compression operations -+ * -+ * Copyright 2015 LG Electronics Inc. -+ * Copyright (c) 2016, Intel Corporation -+ * Author: Giovanni Cabiddu <giovanni.cabiddu@intel.com> -+ * -+ * This program is free software; you can redistribute it and/or modify it -+ * under the terms of the GNU General Public License as published by the Free -+ * Software Foundation; either version 2 of the License, or (at your option) -+ * any later version. -+ * -+ */ -+#ifndef _CRYPTO_SCOMP_INT_H -+#define _CRYPTO_SCOMP_INT_H -+#include <linux/crypto.h> -+ -+#define SCOMP_SCRATCH_SIZE 131072 -+ -+struct crypto_scomp { -+ struct crypto_tfm base; -+}; -+ -+/** -+ * struct scomp_alg - synchronous compression algorithm -+ * -+ * @alloc_ctx: Function allocates algorithm specific context -+ * @free_ctx: Function frees context allocated with alloc_ctx -+ * @compress: Function performs a compress operation -+ * @decompress: Function performs a de-compress operation -+ * @init: Initialize the cryptographic transformation object. -+ * This function is used to initialize the cryptographic -+ * transformation object. This function is called only once at -+ * the instantiation time, right after the transformation context -+ * was allocated. In case the cryptographic hardware has some -+ * special requirements which need to be handled by software, this -+ * function shall check for the precise requirement of the -+ * transformation and put any software fallbacks in place. -+ * @exit: Deinitialize the cryptographic transformation object. This is a -+ * counterpart to @init, used to remove various changes set in -+ * @init. -+ * @base: Common crypto API algorithm data structure -+ */ -+struct scomp_alg { -+ void *(*alloc_ctx)(struct crypto_scomp *tfm); -+ void (*free_ctx)(struct crypto_scomp *tfm, void *ctx); -+ int (*compress)(struct crypto_scomp *tfm, const u8 *src, -+ unsigned int slen, u8 *dst, unsigned int *dlen, -+ void *ctx); -+ int (*decompress)(struct crypto_scomp *tfm, const u8 *src, -+ unsigned int slen, u8 *dst, unsigned int *dlen, -+ void *ctx); -+ struct crypto_alg base; -+}; -+ -+static inline struct scomp_alg *__crypto_scomp_alg(struct crypto_alg *alg) -+{ -+ return container_of(alg, struct scomp_alg, base); -+} -+ -+static inline struct crypto_scomp *__crypto_scomp_tfm(struct crypto_tfm *tfm) -+{ -+ return container_of(tfm, struct crypto_scomp, base); -+} -+ -+static inline struct crypto_tfm *crypto_scomp_tfm(struct crypto_scomp *tfm) -+{ -+ return &tfm->base; -+} -+ -+static inline void crypto_free_scomp(struct crypto_scomp *tfm) -+{ -+ crypto_destroy_tfm(tfm, crypto_scomp_tfm(tfm)); -+} -+ -+static inline struct scomp_alg *crypto_scomp_alg(struct crypto_scomp *tfm) -+{ -+ return __crypto_scomp_alg(crypto_scomp_tfm(tfm)->__crt_alg); -+} -+ -+static inline void *crypto_scomp_alloc_ctx(struct crypto_scomp *tfm) -+{ -+ return crypto_scomp_alg(tfm)->alloc_ctx(tfm); -+} -+ -+static inline void crypto_scomp_free_ctx(struct crypto_scomp *tfm, -+ void *ctx) -+{ -+ return crypto_scomp_alg(tfm)->free_ctx(tfm, ctx); -+} -+ -+static inline int crypto_scomp_compress(struct crypto_scomp *tfm, -+ const u8 *src, unsigned int slen, -+ u8 *dst, unsigned int *dlen, void *ctx) -+{ -+ return crypto_scomp_alg(tfm)->compress(tfm, src, slen, dst, dlen, ctx); -+} -+ -+static inline int crypto_scomp_decompress(struct crypto_scomp *tfm, -+ const u8 *src, unsigned int slen, -+ u8 *dst, unsigned int *dlen, -+ void *ctx) -+{ -+ return crypto_scomp_alg(tfm)->decompress(tfm, src, slen, dst, dlen, -+ ctx); -+} -+ -+int crypto_init_scomp_ops_async(struct crypto_tfm *tfm); -+struct acomp_req *crypto_acomp_scomp_alloc_ctx(struct acomp_req *req); -+void crypto_acomp_scomp_free_ctx(struct acomp_req *req); -+ -+/** -+ * crypto_register_scomp() -- Register synchronous compression algorithm -+ * -+ * Function registers an implementation of a synchronous -+ * compression algorithm -+ * -+ * @alg: algorithm definition -+ * -+ * Return: zero on success; error code in case of error -+ */ -+int crypto_register_scomp(struct scomp_alg *alg); -+ -+/** -+ * crypto_unregister_scomp() -- Unregister synchronous compression algorithm -+ * -+ * Function unregisters an implementation of a synchronous -+ * compression algorithm -+ * -+ * @alg: algorithm definition -+ * -+ * Return: zero on success; error code in case of error -+ */ -+int crypto_unregister_scomp(struct scomp_alg *alg); -+ -+#endif ---- a/include/linux/crypto.h -+++ b/include/linux/crypto.h -@@ -50,6 +50,8 @@ - #define CRYPTO_ALG_TYPE_SKCIPHER 0x00000005 - #define CRYPTO_ALG_TYPE_GIVCIPHER 0x00000006 - #define CRYPTO_ALG_TYPE_KPP 0x00000008 -+#define CRYPTO_ALG_TYPE_ACOMPRESS 0x0000000a -+#define CRYPTO_ALG_TYPE_SCOMPRESS 0x0000000b - #define CRYPTO_ALG_TYPE_RNG 0x0000000c - #define CRYPTO_ALG_TYPE_AKCIPHER 0x0000000d - #define CRYPTO_ALG_TYPE_DIGEST 0x0000000e -@@ -60,6 +62,7 @@ - #define CRYPTO_ALG_TYPE_HASH_MASK 0x0000000e - #define CRYPTO_ALG_TYPE_AHASH_MASK 0x0000000e - #define CRYPTO_ALG_TYPE_BLKCIPHER_MASK 0x0000000c -+#define CRYPTO_ALG_TYPE_ACOMPRESS_MASK 0x0000000e - - #define CRYPTO_ALG_LARVAL 0x00000010 - #define CRYPTO_ALG_DEAD 0x00000020 ---- a/include/uapi/linux/cryptouser.h -+++ b/include/uapi/linux/cryptouser.h -@@ -46,6 +46,7 @@ enum crypto_attr_type_t { - CRYPTOCFGA_REPORT_CIPHER, /* struct crypto_report_cipher */ - CRYPTOCFGA_REPORT_AKCIPHER, /* struct crypto_report_akcipher */ - CRYPTOCFGA_REPORT_KPP, /* struct crypto_report_kpp */ -+ CRYPTOCFGA_REPORT_ACOMP, /* struct crypto_report_acomp */ - __CRYPTOCFGA_MAX - - #define CRYPTOCFGA_MAX (__CRYPTOCFGA_MAX - 1) -@@ -112,5 +113,9 @@ struct crypto_report_kpp { - char type[CRYPTO_MAX_NAME]; - }; - -+struct crypto_report_acomp { -+ char type[CRYPTO_MAX_NAME]; -+}; -+ - #define CRYPTO_REPORT_MAXSIZE (sizeof(struct crypto_user_alg) + \ - sizeof(struct crypto_report_blkcipher)) ---- a/scripts/spelling.txt -+++ b/scripts/spelling.txt -@@ -305,6 +305,9 @@ defintion||definition - defintions||definitions - defualt||default - defult||default -+deintializing||deinitializing -+deintialize||deinitialize -+deintialized||deinitialized - deivce||device - delared||declared - delare||declare ---- a/sound/soc/amd/acp-pcm-dma.c -+++ b/sound/soc/amd/acp-pcm-dma.c -@@ -506,7 +506,7 @@ static int acp_init(void __iomem *acp_mm - return 0; - } - --/* Deintialize ACP */ -+/* Deinitialize ACP */ - static int acp_deinit(void __iomem *acp_mmio) - { - u32 val; |