diff options
Diffstat (limited to 'crypto')
62 files changed, 7021 insertions, 2335 deletions
diff --git a/crypto/Kconfig b/crypto/Kconfig index 4dfdd03e708f..e4bac29a32e7 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -23,6 +23,7 @@ comment "Crypto core or helper" config CRYPTO_FIPS bool "FIPS 200 compliance" + depends on CRYPTO_ANSI_CPRNG && !CRYPTO_MANAGER_DISABLE_TESTS help This options enables the fips boot option which is required if you want to system to operate in a FIPS 200 @@ -78,6 +79,11 @@ config CRYPTO_RNG2 config CRYPTO_PCOMP tristate + select CRYPTO_PCOMP2 + select CRYPTO_ALGAPI + +config CRYPTO_PCOMP2 + tristate select CRYPTO_ALGAPI2 config CRYPTO_MANAGER @@ -92,7 +98,15 @@ config CRYPTO_MANAGER2 select CRYPTO_AEAD2 select CRYPTO_HASH2 select CRYPTO_BLKCIPHER2 - select CRYPTO_PCOMP + select CRYPTO_PCOMP2 + +config CRYPTO_MANAGER_DISABLE_TESTS + bool "Disable run-time self tests" + default y + depends on CRYPTO_MANAGER2 + help + Disable run-time self tests that normally take place at + algorithm registration. config CRYPTO_GF128MUL tristate "GF(2^128) multiplication functions (EXPERIMENTAL)" @@ -112,6 +126,16 @@ config CRYPTO_NULL help These are 'Null' algorithms, used by IPsec, which do nothing. +config CRYPTO_PCRYPT + tristate "Parallel crypto engine (EXPERIMENTAL)" + depends on SMP && EXPERIMENTAL + select PADATA + select CRYPTO_MANAGER + select CRYPTO_AEAD + help + This converts an arbitrary crypto algorithm into a parallel + algorithm that executes in kernel threads. + config CRYPTO_WORKQUEUE tristate @@ -156,7 +180,7 @@ config CRYPTO_GCM tristate "GCM/GMAC support" select CRYPTO_CTR select CRYPTO_AEAD - select CRYPTO_GF128MUL + select CRYPTO_GHASH help Support for Galois/Counter Mode (GCM) and Galois Message Authentication Code (GMAC). Required for IPSec. @@ -267,6 +291,18 @@ config CRYPTO_XCBC http://csrc.nist.gov/encryption/modes/proposedmodes/ xcbc-mac/xcbc-mac-spec.pdf +config CRYPTO_VMAC + tristate "VMAC support" + depends on EXPERIMENTAL + select CRYPTO_HASH + select CRYPTO_MANAGER + help + VMAC is a message authentication algorithm designed for + very high speed on 64-bit architectures. + + See also: + <http://fastcrypto.org/vmac> + comment "Digest" config CRYPTO_CRC32C @@ -289,6 +325,13 @@ config CRYPTO_CRC32C_INTEL gain performance compared with software implementation. Module will be crc32c-intel. +config CRYPTO_GHASH + tristate "GHASH digest algorithm" + select CRYPTO_SHASH + select CRYPTO_GF128MUL + help + GHASH is message digest algorithm for GCM (Galois/Counter Mode). + config CRYPTO_MD4 tristate "MD4 digest algorithm" select CRYPTO_HASH @@ -321,7 +364,7 @@ config CRYPTO_RMD128 RIPEMD-160 should be used. Developed by Hans Dobbertin, Antoon Bosselaers and Bart Preneel. - See <http://home.esat.kuleuven.be/~bosselae/ripemd160.html> + See <http://homes.esat.kuleuven.be/~bosselae/ripemd160.html> config CRYPTO_RMD160 tristate "RIPEMD-160 digest algorithm" @@ -338,7 +381,7 @@ config CRYPTO_RMD160 against RIPEMD-160. Developed by Hans Dobbertin, Antoon Bosselaers and Bart Preneel. - See <http://home.esat.kuleuven.be/~bosselae/ripemd160.html> + See <http://homes.esat.kuleuven.be/~bosselae/ripemd160.html> config CRYPTO_RMD256 tristate "RIPEMD-256 digest algorithm" @@ -350,7 +393,7 @@ config CRYPTO_RMD256 (than RIPEMD-128). Developed by Hans Dobbertin, Antoon Bosselaers and Bart Preneel. - See <http://home.esat.kuleuven.be/~bosselae/ripemd160.html> + See <http://homes.esat.kuleuven.be/~bosselae/ripemd160.html> config CRYPTO_RMD320 tristate "RIPEMD-320 digest algorithm" @@ -362,7 +405,7 @@ config CRYPTO_RMD320 (than RIPEMD-160). Developed by Hans Dobbertin, Antoon Bosselaers and Bart Preneel. - See <http://home.esat.kuleuven.be/~bosselae/ripemd160.html> + See <http://homes.esat.kuleuven.be/~bosselae/ripemd160.html> config CRYPTO_SHA1 tristate "SHA1 digest algorithm" @@ -417,7 +460,16 @@ config CRYPTO_WP512 Whirlpool will be part of the ISO/IEC 10118-3:2003(E) standard See also: - <http://planeta.terra.com.br/informatica/paulobarreto/WhirlpoolPage.html> + <http://www.larc.usp.br/~pbarreto/WhirlpoolPage.html> + +config CRYPTO_GHASH_CLMUL_NI_INTEL + tristate "GHASH digest algorithm (CLMUL-NI accelerated)" + depends on (X86 || UML_X86) && 64BIT + select CRYPTO_SHASH + select CRYPTO_CRYPTD + help + GHASH is message digest algorithm for GCM (Galois/Counter Mode). + The implementation is accelerated by CLMUL-NI of Intel. comment "Ciphers" @@ -526,8 +578,8 @@ config CRYPTO_ANUBIS in the NESSIE competition. See also: - <https://www.cosic.esat.kuleuven.ac.be/nessie/reports/> - <http://planeta.terra.com.br/informatica/paulobarreto/AnubisPage.html> + <https://www.cosic.esat.kuleuven.be/nessie/reports/> + <http://www.larc.usp.br/~pbarreto/AnubisPage.html> config CRYPTO_ARC4 tristate "ARC4 cipher algorithm" @@ -606,7 +658,7 @@ config CRYPTO_KHAZAD on 32-bit processors. Khazad uses an 128 bit key size. See also: - <http://planeta.terra.com.br/informatica/paulobarreto/KhazadPage.html> + <http://www.larc.usp.br/~pbarreto/KhazadPage.html> config CRYPTO_SALSA20 tristate "Salsa20 stream cipher algorithm (EXPERIMENTAL)" @@ -780,13 +832,14 @@ comment "Random Number Generation" config CRYPTO_ANSI_CPRNG tristate "Pseudo Random Number Generation for Cryptographic modules" + default m select CRYPTO_AES select CRYPTO_RNG - select CRYPTO_FIPS help This option enables the generic pseudo random number generator for cryptographic modules. Uses the Algorithm specified in - ANSI X9.31 A.2.4 + ANSI X9.31 A.2.4. Note that this option must be enabled if + CRYPTO_FIPS is selected source "drivers/crypto/Kconfig" diff --git a/crypto/Makefile b/crypto/Makefile index 673d9f7c1bda..423b7de61f93 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -3,7 +3,7 @@ # obj-$(CONFIG_CRYPTO) += crypto.o -crypto-objs := api.o cipher.o digest.o compress.o +crypto-objs := api.o cipher.o compress.o obj-$(CONFIG_CRYPTO_WORKQUEUE) += crypto_wq.o @@ -22,17 +22,17 @@ obj-$(CONFIG_CRYPTO_BLKCIPHER2) += chainiv.o obj-$(CONFIG_CRYPTO_BLKCIPHER2) += eseqiv.o obj-$(CONFIG_CRYPTO_SEQIV) += seqiv.o -crypto_hash-objs := hash.o crypto_hash-objs += ahash.o crypto_hash-objs += shash.o obj-$(CONFIG_CRYPTO_HASH2) += crypto_hash.o -obj-$(CONFIG_CRYPTO_PCOMP) += pcompress.o +obj-$(CONFIG_CRYPTO_PCOMP2) += pcompress.o cryptomgr-objs := algboss.o testmgr.o obj-$(CONFIG_CRYPTO_MANAGER2) += cryptomgr.o obj-$(CONFIG_CRYPTO_HMAC) += hmac.o +obj-$(CONFIG_CRYPTO_VMAC) += vmac.o obj-$(CONFIG_CRYPTO_XCBC) += xcbc.o obj-$(CONFIG_CRYPTO_NULL) += crypto_null.o obj-$(CONFIG_CRYPTO_MD4) += md4.o @@ -56,11 +56,12 @@ obj-$(CONFIG_CRYPTO_XTS) += xts.o obj-$(CONFIG_CRYPTO_CTR) += ctr.o obj-$(CONFIG_CRYPTO_GCM) += gcm.o obj-$(CONFIG_CRYPTO_CCM) += ccm.o +obj-$(CONFIG_CRYPTO_PCRYPT) += pcrypt.o obj-$(CONFIG_CRYPTO_CRYPTD) += cryptd.o obj-$(CONFIG_CRYPTO_DES) += des_generic.o obj-$(CONFIG_CRYPTO_FCRYPT) += fcrypt.o obj-$(CONFIG_CRYPTO_BLOWFISH) += blowfish.o -obj-$(CONFIG_CRYPTO_TWOFISH) += twofish.o +obj-$(CONFIG_CRYPTO_TWOFISH) += twofish_generic.o obj-$(CONFIG_CRYPTO_TWOFISH_COMMON) += twofish_common.o obj-$(CONFIG_CRYPTO_SERPENT) += serpent.o obj-$(CONFIG_CRYPTO_AES) += aes_generic.o @@ -83,6 +84,7 @@ obj-$(CONFIG_CRYPTO_RNG2) += rng.o obj-$(CONFIG_CRYPTO_RNG2) += krng.o obj-$(CONFIG_CRYPTO_ANSI_CPRNG) += ansi_cprng.o obj-$(CONFIG_CRYPTO_TEST) += tcrypt.o +obj-$(CONFIG_CRYPTO_GHASH) += ghash-generic.o # # generic algorithms and the async_tx api diff --git a/crypto/ablkcipher.c b/crypto/ablkcipher.c index e11ce37c7104..a854df2a5a4b 100644 --- a/crypto/ablkcipher.c +++ b/crypto/ablkcipher.c @@ -1,6 +1,6 @@ /* * Asynchronous block chaining cipher operations. - * + * * This is the asynchronous version of blkcipher.c indicating completion * via a callback. * @@ -8,12 +8,13 @@ * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) + * Software Foundation; either version 2 of the License, or (at your option) * any later version. * */ #include <crypto/internal/skcipher.h> +#include <linux/cpumask.h> #include <linux/err.h> #include <linux/init.h> #include <linux/kernel.h> @@ -23,8 +24,287 @@ #include <linux/slab.h> #include <linux/seq_file.h> +#include <crypto/scatterwalk.h> + #include "internal.h" +static const char *skcipher_default_geniv __read_mostly; + +struct ablkcipher_buffer { + struct list_head entry; + struct scatter_walk dst; + unsigned int len; + void *data; +}; + +enum { + ABLKCIPHER_WALK_SLOW = 1 << 0, +}; + +static inline void ablkcipher_buffer_write(struct ablkcipher_buffer *p) +{ + scatterwalk_copychunks(p->data, &p->dst, p->len, 1); +} + +void __ablkcipher_walk_complete(struct ablkcipher_walk *walk) +{ + struct ablkcipher_buffer *p, *tmp; + + list_for_each_entry_safe(p, tmp, &walk->buffers, entry) { + ablkcipher_buffer_write(p); + list_del(&p->entry); + kfree(p); + } +} +EXPORT_SYMBOL_GPL(__ablkcipher_walk_complete); + +static inline void ablkcipher_queue_write(struct ablkcipher_walk *walk, + struct ablkcipher_buffer *p) +{ + p->dst = walk->out; + list_add_tail(&p->entry, &walk->buffers); +} + +/* Get a spot of the specified length that does not straddle a page. + * The caller needs to ensure that there is enough space for this operation. + */ +static inline u8 *ablkcipher_get_spot(u8 *start, unsigned int len) +{ + u8 *end_page = (u8 *)(((unsigned long)(start + len - 1)) & PAGE_MASK); + return max(start, end_page); +} + +static inline unsigned int ablkcipher_done_slow(struct ablkcipher_walk *walk, + unsigned int bsize) +{ + unsigned int n = bsize; + + for (;;) { + unsigned int len_this_page = scatterwalk_pagelen(&walk->out); + + if (len_this_page > n) + len_this_page = n; + scatterwalk_advance(&walk->out, n); + if (n == len_this_page) + break; + n -= len_this_page; + scatterwalk_start(&walk->out, scatterwalk_sg_next(walk->out.sg)); + } + + return bsize; +} + +static inline unsigned int ablkcipher_done_fast(struct ablkcipher_walk *walk, + unsigned int n) +{ + scatterwalk_advance(&walk->in, n); + scatterwalk_advance(&walk->out, n); + + return n; +} + +static int ablkcipher_walk_next(struct ablkcipher_request *req, + struct ablkcipher_walk *walk); + +int ablkcipher_walk_done(struct ablkcipher_request *req, + struct ablkcipher_walk *walk, int err) +{ + struct crypto_tfm *tfm = req->base.tfm; + unsigned int nbytes = 0; + + if (likely(err >= 0)) { + unsigned int n = walk->nbytes - err; + + if (likely(!(walk->flags & ABLKCIPHER_WALK_SLOW))) + n = ablkcipher_done_fast(walk, n); + else if (WARN_ON(err)) { + err = -EINVAL; + goto err; + } else + n = ablkcipher_done_slow(walk, n); + + nbytes = walk->total - n; + err = 0; + } + + scatterwalk_done(&walk->in, 0, nbytes); + scatterwalk_done(&walk->out, 1, nbytes); + +err: + walk->total = nbytes; + walk->nbytes = nbytes; + + if (nbytes) { + crypto_yield(req->base.flags); + return ablkcipher_walk_next(req, walk); + } + + if (walk->iv != req->info) + memcpy(req->info, walk->iv, tfm->crt_ablkcipher.ivsize); + if (walk->iv_buffer) + kfree(walk->iv_buffer); + + return err; +} +EXPORT_SYMBOL_GPL(ablkcipher_walk_done); + +static inline int ablkcipher_next_slow(struct ablkcipher_request *req, + struct ablkcipher_walk *walk, + unsigned int bsize, + unsigned int alignmask, + void **src_p, void **dst_p) +{ + unsigned aligned_bsize = ALIGN(bsize, alignmask + 1); + struct ablkcipher_buffer *p; + void *src, *dst, *base; + unsigned int n; + + n = ALIGN(sizeof(struct ablkcipher_buffer), alignmask + 1); + n += (aligned_bsize * 3 - (alignmask + 1) + + (alignmask & ~(crypto_tfm_ctx_alignment() - 1))); + + p = kmalloc(n, GFP_ATOMIC); + if (!p) + return ablkcipher_walk_done(req, walk, -ENOMEM); + + base = p + 1; + + dst = (u8 *)ALIGN((unsigned long)base, alignmask + 1); + src = dst = ablkcipher_get_spot(dst, bsize); + + p->len = bsize; + p->data = dst; + + scatterwalk_copychunks(src, &walk->in, bsize, 0); + + ablkcipher_queue_write(walk, p); + + walk->nbytes = bsize; + walk->flags |= ABLKCIPHER_WALK_SLOW; + + *src_p = src; + *dst_p = dst; + + return 0; +} + +static inline int ablkcipher_copy_iv(struct ablkcipher_walk *walk, + struct crypto_tfm *tfm, + unsigned int alignmask) +{ + unsigned bs = walk->blocksize; + unsigned int ivsize = tfm->crt_ablkcipher.ivsize; + unsigned aligned_bs = ALIGN(bs, alignmask + 1); + unsigned int size = aligned_bs * 2 + ivsize + max(aligned_bs, ivsize) - + (alignmask + 1); + u8 *iv; + + size += alignmask & ~(crypto_tfm_ctx_alignment() - 1); + walk->iv_buffer = kmalloc(size, GFP_ATOMIC); + if (!walk->iv_buffer) + return -ENOMEM; + + iv = (u8 *)ALIGN((unsigned long)walk->iv_buffer, alignmask + 1); + iv = ablkcipher_get_spot(iv, bs) + aligned_bs; + iv = ablkcipher_get_spot(iv, bs) + aligned_bs; + iv = ablkcipher_get_spot(iv, ivsize); + + walk->iv = memcpy(iv, walk->iv, ivsize); + return 0; +} + +static inline int ablkcipher_next_fast(struct ablkcipher_request *req, + struct ablkcipher_walk *walk) +{ + walk->src.page = scatterwalk_page(&walk->in); + walk->src.offset = offset_in_page(walk->in.offset); + walk->dst.page = scatterwalk_page(&walk->out); + walk->dst.offset = offset_in_page(walk->out.offset); + + return 0; +} + +static int ablkcipher_walk_next(struct ablkcipher_request *req, + struct ablkcipher_walk *walk) +{ + struct crypto_tfm *tfm = req->base.tfm; + unsigned int alignmask, bsize, n; + void *src, *dst; + int err; + + alignmask = crypto_tfm_alg_alignmask(tfm); + n = walk->total; + if (unlikely(n < crypto_tfm_alg_blocksize(tfm))) { + req->base.flags |= CRYPTO_TFM_RES_BAD_BLOCK_LEN; + return ablkcipher_walk_done(req, walk, -EINVAL); + } + + walk->flags &= ~ABLKCIPHER_WALK_SLOW; + src = dst = NULL; + + bsize = min(walk->blocksize, n); + n = scatterwalk_clamp(&walk->in, n); + n = scatterwalk_clamp(&walk->out, n); + + if (n < bsize || + !scatterwalk_aligned(&walk->in, alignmask) || + !scatterwalk_aligned(&walk->out, alignmask)) { + err = ablkcipher_next_slow(req, walk, bsize, alignmask, + &src, &dst); + goto set_phys_lowmem; + } + + walk->nbytes = n; + + return ablkcipher_next_fast(req, walk); + +set_phys_lowmem: + if (err >= 0) { + walk->src.page = virt_to_page(src); + walk->dst.page = virt_to_page(dst); + walk->src.offset = ((unsigned long)src & (PAGE_SIZE - 1)); + walk->dst.offset = ((unsigned long)dst & (PAGE_SIZE - 1)); + } + + return err; +} + +static int ablkcipher_walk_first(struct ablkcipher_request *req, + struct ablkcipher_walk *walk) +{ + struct crypto_tfm *tfm = req->base.tfm; + unsigned int alignmask; + + alignmask = crypto_tfm_alg_alignmask(tfm); + if (WARN_ON_ONCE(in_irq())) + return -EDEADLK; + + walk->nbytes = walk->total; + if (unlikely(!walk->total)) + return 0; + + walk->iv_buffer = NULL; + walk->iv = req->info; + if (unlikely(((unsigned long)walk->iv & alignmask))) { + int err = ablkcipher_copy_iv(walk, tfm, alignmask); + if (err) + return err; + } + + scatterwalk_start(&walk->in, walk->in.sg); + scatterwalk_start(&walk->out, walk->out.sg); + + return ablkcipher_walk_next(req, walk); +} + +int ablkcipher_walk_phys(struct ablkcipher_request *req, + struct ablkcipher_walk *walk) +{ + walk->blocksize = crypto_tfm_alg_blocksize(req->base.tfm); + return ablkcipher_walk_first(req, walk); +} +EXPORT_SYMBOL_GPL(ablkcipher_walk_phys); + static int setkey_unaligned(struct crypto_ablkcipher *tfm, const u8 *key, unsigned int keylen) { @@ -180,7 +460,14 @@ EXPORT_SYMBOL_GPL(crypto_givcipher_type); const char *crypto_default_geniv(const struct crypto_alg *alg) { - return alg->cra_flags & CRYPTO_ALG_ASYNC ? "eseqiv" : "chainiv"; + if (((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == + CRYPTO_ALG_TYPE_BLKCIPHER ? alg->cra_blkcipher.ivsize : + alg->cra_ablkcipher.ivsize) != + alg->cra_blocksize) + return "chainiv"; + + return alg->cra_flags & CRYPTO_ALG_ASYNC ? + "eseqiv" : skcipher_default_geniv; } static int crypto_givcipher_default(struct crypto_alg *alg, u32 type, u32 mask) @@ -201,8 +488,9 @@ static int crypto_givcipher_default(struct crypto_alg *alg, u32 type, u32 mask) int err; larval = crypto_larval_lookup(alg->cra_driver_name, + (type & ~CRYPTO_ALG_TYPE_MASK) | CRYPTO_ALG_TYPE_GIVCIPHER, - CRYPTO_ALG_TYPE_MASK); + mask | CRYPTO_ALG_TYPE_MASK); err = PTR_ERR(larval); if (IS_ERR(larval)) goto out; @@ -360,3 +648,17 @@ err: return ERR_PTR(err); } EXPORT_SYMBOL_GPL(crypto_alloc_ablkcipher); + +static int __init skcipher_module_init(void) +{ + skcipher_default_geniv = num_possible_cpus() > 1 ? + "eseqiv" : "chainiv"; + return 0; +} + +static void skcipher_module_exit(void) +{ +} + +module_init(skcipher_module_init); +module_exit(skcipher_module_exit); diff --git a/crypto/aead.c b/crypto/aead.c index d9aa733db164..6729e8ff68e7 100644 --- a/crypto/aead.c +++ b/crypto/aead.c @@ -1,13 +1,13 @@ /* * AEAD: Authenticated Encryption with Associated Data - * + * * This file provides API support for AEAD algorithms. * * Copyright (c) 2007 Herbert Xu <herbert@gondor.apana.org.au> * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) + * Software Foundation; either version 2 of the License, or (at your option) * any later version. * */ @@ -18,6 +18,7 @@ #include <linux/kernel.h> #include <linux/module.h> #include <linux/rtnetlink.h> +#include <linux/sched.h> #include <linux/slab.h> #include <linux/seq_file.h> diff --git a/crypto/aes_generic.c b/crypto/aes_generic.c index b8b66ec3883b..a68c73dae15a 100644 --- a/crypto/aes_generic.c +++ b/crypto/aes_generic.c @@ -1,4 +1,4 @@ -/* +/* * Cryptographic API. * * AES Cipher Algorithm. @@ -1127,7 +1127,7 @@ EXPORT_SYMBOL_GPL(crypto_il_tab); #define star_x(x) (((x) & 0x7f7f7f7f) << 1) ^ ((((x) & 0x80808080) >> 7) * 0x1b) -#define imix_col(y,x) do { \ +#define imix_col(y, x) do { \ u = star_x(x); \ v = star_x(u); \ w = star_x(v); \ @@ -1174,7 +1174,7 @@ EXPORT_SYMBOL_GPL(crypto_il_tab); ctx->key_enc[6 * i + 11] = t; \ } while (0) -#define loop8(i) do { \ +#define loop8tophalf(i) do { \ t = ror32(t, 8); \ t = ls_box(t) ^ rco_tab[i]; \ t ^= ctx->key_enc[8 * i]; \ @@ -1185,6 +1185,10 @@ EXPORT_SYMBOL_GPL(crypto_il_tab); ctx->key_enc[8 * i + 10] = t; \ t ^= ctx->key_enc[8 * i + 3]; \ ctx->key_enc[8 * i + 11] = t; \ +} while (0) + +#define loop8(i) do { \ + loop8tophalf(i); \ t = ctx->key_enc[8 * i + 4] ^ ls_box(t); \ ctx->key_enc[8 * i + 12] = t; \ t ^= ctx->key_enc[8 * i + 5]; \ @@ -1245,8 +1249,9 @@ int crypto_aes_expand_key(struct crypto_aes_ctx *ctx, const u8 *in_key, ctx->key_enc[5] = le32_to_cpu(key[5]); ctx->key_enc[6] = le32_to_cpu(key[6]); t = ctx->key_enc[7] = le32_to_cpu(key[7]); - for (i = 0; i < 7; ++i) + for (i = 0; i < 6; ++i) loop8(i); + loop8tophalf(i); break; } diff --git a/crypto/ahash.c b/crypto/ahash.c index f3476374f764..f669822a7a44 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -24,6 +24,19 @@ #include "internal.h" +struct ahash_request_priv { + crypto_completion_t complete; + void *data; + u8 *result; + void *ubuf[] CRYPTO_MINALIGN_ATTR; +}; + +static inline struct ahash_alg *crypto_ahash_alg(struct crypto_ahash *hash) +{ + return container_of(crypto_hash_alg_common(hash), struct ahash_alg, + halg); +} + static int hash_walk_next(struct crypto_hash_walk *walk) { unsigned int alignmask = walk->alignmask; @@ -34,8 +47,11 @@ static int hash_walk_next(struct crypto_hash_walk *walk) walk->data = crypto_kmap(walk->pg, 0); walk->data += offset; - if (offset & alignmask) - nbytes = alignmask + 1 - (offset & alignmask); + if (offset & alignmask) { + unsigned int unaligned = alignmask + 1 - (offset & alignmask); + if (nbytes > unaligned) + nbytes = unaligned; + } walk->entrylen -= nbytes; return nbytes; @@ -65,7 +81,6 @@ int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err) walk->data -= walk->offset; if (nbytes && walk->offset & alignmask && !err) { - walk->offset += alignmask - 1; walk->offset = ALIGN(walk->offset, alignmask + 1); walk->data += walk->offset; @@ -132,36 +147,34 @@ int crypto_hash_walk_first_compat(struct hash_desc *hdesc, static int ahash_setkey_unaligned(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { - struct ahash_alg *ahash = crypto_ahash_alg(tfm); unsigned long alignmask = crypto_ahash_alignmask(tfm); int ret; u8 *buffer, *alignbuffer; unsigned long absize; absize = keylen + alignmask; - buffer = kmalloc(absize, GFP_ATOMIC); + buffer = kmalloc(absize, GFP_KERNEL); if (!buffer) return -ENOMEM; alignbuffer = (u8 *)ALIGN((unsigned long)buffer, alignmask + 1); memcpy(alignbuffer, key, keylen); - ret = ahash->setkey(tfm, alignbuffer, keylen); - memset(alignbuffer, 0, keylen); - kfree(buffer); + ret = tfm->setkey(tfm, alignbuffer, keylen); + kzfree(buffer); return ret; } -static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key, +int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { - struct ahash_alg *ahash = crypto_ahash_alg(tfm); unsigned long alignmask = crypto_ahash_alignmask(tfm); if ((unsigned long)key & alignmask) return ahash_setkey_unaligned(tfm, key, keylen); - return ahash->setkey(tfm, key, keylen); + return tfm->setkey(tfm, key, keylen); } +EXPORT_SYMBOL_GPL(crypto_ahash_setkey); static int ahash_nosetkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) @@ -169,44 +182,221 @@ static int ahash_nosetkey(struct crypto_ahash *tfm, const u8 *key, return -ENOSYS; } -int crypto_ahash_import(struct ahash_request *req, const u8 *in) +static inline unsigned int ahash_align_buffer_size(unsigned len, + unsigned long mask) +{ + return len + (mask & ~(crypto_tfm_ctx_alignment() - 1)); +} + +static void ahash_op_unaligned_finish(struct ahash_request *req, int err) +{ + struct ahash_request_priv *priv = req->priv; + + if (err == -EINPROGRESS) + return; + + if (!err) + memcpy(priv->result, req->result, + crypto_ahash_digestsize(crypto_ahash_reqtfm(req))); + + kzfree(priv); +} + +static void ahash_op_unaligned_done(struct crypto_async_request *req, int err) +{ + struct ahash_request *areq = req->data; + struct ahash_request_priv *priv = areq->priv; + crypto_completion_t complete = priv->complete; + void *data = priv->data; + + ahash_op_unaligned_finish(areq, err); + + complete(data, err); +} + +static int ahash_op_unaligned(struct ahash_request *req, + int (*op)(struct ahash_request *)) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ahash_alg *alg = crypto_ahash_alg(tfm); + unsigned long alignmask = crypto_ahash_alignmask(tfm); + unsigned int ds = crypto_ahash_digestsize(tfm); + struct ahash_request_priv *priv; + int err; + + priv = kmalloc(sizeof(*priv) + ahash_align_buffer_size(ds, alignmask), + (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC); + if (!priv) + return -ENOMEM; - memcpy(ahash_request_ctx(req), in, crypto_ahash_reqsize(tfm)); + priv->result = req->result; + priv->complete = req->base.complete; + priv->data = req->base.data; - if (alg->reinit) - alg->reinit(req); + req->result = PTR_ALIGN((u8 *)priv->ubuf, alignmask + 1); + req->base.complete = ahash_op_unaligned_done; + req->base.data = req; + req->priv = priv; - return 0; + err = op(req); + ahash_op_unaligned_finish(req, err); + + return err; } -EXPORT_SYMBOL_GPL(crypto_ahash_import); -static unsigned int crypto_ahash_ctxsize(struct crypto_alg *alg, u32 type, - u32 mask) +static int crypto_ahash_op(struct ahash_request *req, + int (*op)(struct ahash_request *)) { - return alg->cra_ctxsize; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + unsigned long alignmask = crypto_ahash_alignmask(tfm); + + if ((unsigned long)req->result & alignmask) + return ahash_op_unaligned(req, op); + + return op(req); } -static int crypto_init_ahash_ops(struct crypto_tfm *tfm, u32 type, u32 mask) +int crypto_ahash_final(struct ahash_request *req) { - struct ahash_alg *alg = &tfm->__crt_alg->cra_ahash; - struct ahash_tfm *crt = &tfm->crt_ahash; + return crypto_ahash_op(req, crypto_ahash_reqtfm(req)->final); +} +EXPORT_SYMBOL_GPL(crypto_ahash_final); - if (alg->digestsize > PAGE_SIZE / 8) - return -EINVAL; +int crypto_ahash_finup(struct ahash_request *req) +{ + return crypto_ahash_op(req, crypto_ahash_reqtfm(req)->finup); +} +EXPORT_SYMBOL_GPL(crypto_ahash_finup); + +int crypto_ahash_digest(struct ahash_request *req) +{ + return crypto_ahash_op(req, crypto_ahash_reqtfm(req)->digest); +} +EXPORT_SYMBOL_GPL(crypto_ahash_digest); + +static void ahash_def_finup_finish2(struct ahash_request *req, int err) +{ + struct ahash_request_priv *priv = req->priv; + + if (err == -EINPROGRESS) + return; + + if (!err) + memcpy(priv->result, req->result, + crypto_ahash_digestsize(crypto_ahash_reqtfm(req))); + + kzfree(priv); +} + +static void ahash_def_finup_done2(struct crypto_async_request *req, int err) +{ + struct ahash_request *areq = req->data; + struct ahash_request_priv *priv = areq->priv; + crypto_completion_t complete = priv->complete; + void *data = priv->data; + + ahash_def_finup_finish2(areq, err); + + complete(data, err); +} + +static int ahash_def_finup_finish1(struct ahash_request *req, int err) +{ + if (err) + goto out; + + req->base.complete = ahash_def_finup_done2; + req->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + err = crypto_ahash_reqtfm(req)->final(req); + +out: + ahash_def_finup_finish2(req, err); + return err; +} + +static void ahash_def_finup_done1(struct crypto_async_request *req, int err) +{ + struct ahash_request *areq = req->data; + struct ahash_request_priv *priv = areq->priv; + crypto_completion_t complete = priv->complete; + void *data = priv->data; + + err = ahash_def_finup_finish1(areq, err); + + complete(data, err); +} + +static int ahash_def_finup(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + unsigned long alignmask = crypto_ahash_alignmask(tfm); + unsigned int ds = crypto_ahash_digestsize(tfm); + struct ahash_request_priv *priv; - crt->init = alg->init; - crt->update = alg->update; - crt->final = alg->final; - crt->digest = alg->digest; - crt->setkey = alg->setkey ? ahash_setkey : ahash_nosetkey; - crt->digestsize = alg->digestsize; + priv = kmalloc(sizeof(*priv) + ahash_align_buffer_size(ds, alignmask), + (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC); + if (!priv) + return -ENOMEM; + + priv->result = req->result; + priv->complete = req->base.complete; + priv->data = req->base.data; + + req->result = PTR_ALIGN((u8 *)priv->ubuf, alignmask + 1); + req->base.complete = ahash_def_finup_done1; + req->base.data = req; + req->priv = priv; + + return ahash_def_finup_finish1(req, tfm->update(req)); +} + +static int ahash_no_export(struct ahash_request *req, void *out) +{ + return -ENOSYS; +} + +static int ahash_no_import(struct ahash_request *req, const void *in) +{ + return -ENOSYS; +} + +static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) +{ + struct crypto_ahash *hash = __crypto_ahash_cast(tfm); + struct ahash_alg *alg = crypto_ahash_alg(hash); + + hash->setkey = ahash_nosetkey; + hash->export = ahash_no_export; + hash->import = ahash_no_import; + + if (tfm->__crt_alg->cra_type != &crypto_ahash_type) + return crypto_init_shash_ops_async(tfm); + + hash->init = alg->init; + hash->update = alg->update; + hash->final = alg->final; + hash->finup = alg->finup ?: ahash_def_finup; + hash->digest = alg->digest; + + if (alg->setkey) + hash->setkey = alg->setkey; + if (alg->export) + hash->export = alg->export; + if (alg->import) + hash->import = alg->import; return 0; } +static unsigned int crypto_ahash_extsize(struct crypto_alg *alg) +{ + if (alg->cra_type == &crypto_ahash_type) + return alg->cra_ctxsize; + + return sizeof(struct crypto_shash *); +} + static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg) __attribute__ ((unused)); static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg) @@ -215,17 +405,101 @@ static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg) seq_printf(m, "async : %s\n", alg->cra_flags & CRYPTO_ALG_ASYNC ? "yes" : "no"); seq_printf(m, "blocksize : %u\n", alg->cra_blocksize); - seq_printf(m, "digestsize : %u\n", alg->cra_ahash.digestsize); + seq_printf(m, "digestsize : %u\n", + __crypto_hash_alg_common(alg)->digestsize); } const struct crypto_type crypto_ahash_type = { - .ctxsize = crypto_ahash_ctxsize, - .init = crypto_init_ahash_ops, + .extsize = crypto_ahash_extsize, + .init_tfm = crypto_ahash_init_tfm, #ifdef CONFIG_PROC_FS .show = crypto_ahash_show, #endif + .maskclear = ~CRYPTO_ALG_TYPE_MASK, + .maskset = CRYPTO_ALG_TYPE_AHASH_MASK, + .type = CRYPTO_ALG_TYPE_AHASH, + .tfmsize = offsetof(struct crypto_ahash, base), }; EXPORT_SYMBOL_GPL(crypto_ahash_type); +struct crypto_ahash *crypto_alloc_ahash(const char *alg_name, u32 type, + u32 mask) +{ + return crypto_alloc_tfm(alg_name, &crypto_ahash_type, type, mask); +} +EXPORT_SYMBOL_GPL(crypto_alloc_ahash); + +static int ahash_prepare_alg(struct ahash_alg *alg) +{ + struct crypto_alg *base = &alg->halg.base; + + if (alg->halg.digestsize > PAGE_SIZE / 8 || + alg->halg.statesize > PAGE_SIZE / 8) + return -EINVAL; + + base->cra_type = &crypto_ahash_type; + base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK; + base->cra_flags |= CRYPTO_ALG_TYPE_AHASH; + + return 0; +} + +int crypto_register_ahash(struct ahash_alg *alg) +{ + struct crypto_alg *base = &alg->halg.base; + int err; + + err = ahash_prepare_alg(alg); + if (err) + return err; + + return crypto_register_alg(base); +} +EXPORT_SYMBOL_GPL(crypto_register_ahash); + +int crypto_unregister_ahash(struct ahash_alg *alg) +{ + return crypto_unregister_alg(&alg->halg.base); +} +EXPORT_SYMBOL_GPL(crypto_unregister_ahash); + +int ahash_register_instance(struct crypto_template *tmpl, + struct ahash_instance *inst) +{ + int err; + + err = ahash_prepare_alg(&inst->alg); + if (err) + return err; + + return crypto_register_instance(tmpl, ahash_crypto_instance(inst)); +} +EXPORT_SYMBOL_GPL(ahash_register_instance); + +void ahash_free_instance(struct crypto_instance *inst) +{ + crypto_drop_spawn(crypto_instance_ctx(inst)); + kfree(ahash_instance(inst)); +} +EXPORT_SYMBOL_GPL(ahash_free_instance); + +int crypto_init_ahash_spawn(struct crypto_ahash_spawn *spawn, + struct hash_alg_common *alg, + struct crypto_instance *inst) +{ + return crypto_init_spawn2(&spawn->base, &alg->base, inst, + &crypto_ahash_type); +} +EXPORT_SYMBOL_GPL(crypto_init_ahash_spawn); + +struct hash_alg_common *ahash_attr_alg(struct rtattr *rta, u32 type, u32 mask) +{ + struct crypto_alg *alg; + + alg = crypto_attr_alg2(rta, &crypto_ahash_type, type, mask); + return IS_ERR(alg) ? ERR_CAST(alg) : __crypto_hash_alg_common(alg); +} +EXPORT_SYMBOL_GPL(ahash_attr_alg); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Asynchronous cryptographic hash type"); diff --git a/crypto/algapi.c b/crypto/algapi.c index 56c62e2858d5..c3cf1a69a47a 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -17,6 +17,7 @@ #include <linux/list.h> #include <linux/module.h> #include <linux/rtnetlink.h> +#include <linux/slab.h> #include <linux/string.h> #include "internal.h" @@ -81,16 +82,35 @@ static void crypto_destroy_instance(struct crypto_alg *alg) crypto_tmpl_put(tmpl); } +static struct list_head *crypto_more_spawns(struct crypto_alg *alg, + struct list_head *stack, + struct list_head *top, + struct list_head *secondary_spawns) +{ + struct crypto_spawn *spawn, *n; + + if (list_empty(stack)) + return NULL; + + spawn = list_first_entry(stack, struct crypto_spawn, list); + n = list_entry(spawn->list.next, struct crypto_spawn, list); + + if (spawn->alg && &n->list != stack && !n->alg) + n->alg = (n->list.next == stack) ? alg : + &list_entry(n->list.next, struct crypto_spawn, + list)->inst->alg; + + list_move(&spawn->list, secondary_spawns); + + return &n->list == stack ? top : &n->inst->alg.cra_users; +} + static void crypto_remove_spawn(struct crypto_spawn *spawn, - struct list_head *list, - struct list_head *secondary_spawns) + struct list_head *list) { struct crypto_instance *inst = spawn->inst; struct crypto_template *tmpl = inst->tmpl; - list_del_init(&spawn->list); - spawn->alg = NULL; - if (crypto_is_dead(&inst->alg)) return; @@ -106,25 +126,55 @@ static void crypto_remove_spawn(struct crypto_spawn *spawn, hlist_del(&inst->list); inst->alg.cra_destroy = crypto_destroy_instance; - list_splice(&inst->alg.cra_users, secondary_spawns); + BUG_ON(!list_empty(&inst->alg.cra_users)); } -static void crypto_remove_spawns(struct list_head *spawns, - struct list_head *list, u32 new_type) +static void crypto_remove_spawns(struct crypto_alg *alg, + struct list_head *list, + struct crypto_alg *nalg) { + u32 new_type = (nalg ?: alg)->cra_flags; struct crypto_spawn *spawn, *n; LIST_HEAD(secondary_spawns); + struct list_head *spawns; + LIST_HEAD(stack); + LIST_HEAD(top); + spawns = &alg->cra_users; list_for_each_entry_safe(spawn, n, spawns, list) { if ((spawn->alg->cra_flags ^ new_type) & spawn->mask) continue; - crypto_remove_spawn(spawn, list, &secondary_spawns); + list_move(&spawn->list, &top); } - while (!list_empty(&secondary_spawns)) { - list_for_each_entry_safe(spawn, n, &secondary_spawns, list) - crypto_remove_spawn(spawn, list, &secondary_spawns); + spawns = ⊤ + do { + while (!list_empty(spawns)) { + struct crypto_instance *inst; + + spawn = list_first_entry(spawns, struct crypto_spawn, + list); + inst = spawn->inst; + + BUG_ON(&inst->alg == alg); + + list_move(&spawn->list, &stack); + + if (&inst->alg == nalg) + break; + + spawn->alg = NULL; + spawns = &inst->alg.cra_users; + } + } while ((spawns = crypto_more_spawns(alg, &stack, &top, + &secondary_spawns))); + + list_for_each_entry_safe(spawn, n, &secondary_spawns, list) { + if (spawn->alg) + list_move(&spawn->list, &spawn->alg->cra_users); + else + crypto_remove_spawn(spawn, list); } } @@ -181,7 +231,7 @@ static struct crypto_larval *__crypto_register_alg(struct crypto_alg *alg) list_add(&alg->cra_list, &crypto_alg_list); list_add(&larval->alg.cra_list, &crypto_alg_list); -out: +out: return larval; free_larval: @@ -258,7 +308,7 @@ found: q->cra_priority > alg->cra_priority) continue; - crypto_remove_spawns(&q->cra_users, &list, alg->cra_flags); + crypto_remove_spawns(q, &list, alg); } complete: @@ -330,7 +380,7 @@ static int crypto_remove_alg(struct crypto_alg *alg, struct list_head *list) crypto_notify(CRYPTO_MSG_ALG_UNREGISTER, alg); list_del_init(&alg->cra_list); - crypto_remove_spawns(&alg->cra_users, list, alg->cra_flags); + crypto_remove_spawns(alg, list, NULL); return 0; } @@ -339,7 +389,7 @@ int crypto_unregister_alg(struct crypto_alg *alg) { int ret; LIST_HEAD(list); - + down_write(&crypto_alg_sem); ret = crypto_remove_alg(alg, &list); up_write(&crypto_alg_sem); @@ -488,20 +538,38 @@ int crypto_init_spawn(struct crypto_spawn *spawn, struct crypto_alg *alg, } EXPORT_SYMBOL_GPL(crypto_init_spawn); +int crypto_init_spawn2(struct crypto_spawn *spawn, struct crypto_alg *alg, + struct crypto_instance *inst, + const struct crypto_type *frontend) +{ + int err = -EINVAL; + + if ((alg->cra_flags ^ frontend->type) & frontend->maskset) + goto out; + + spawn->frontend = frontend; + err = crypto_init_spawn(spawn, alg, inst, frontend->maskset); + +out: + return err; +} +EXPORT_SYMBOL_GPL(crypto_init_spawn2); + void crypto_drop_spawn(struct crypto_spawn *spawn) { + if (!spawn->alg) + return; + down_write(&crypto_alg_sem); list_del(&spawn->list); up_write(&crypto_alg_sem); } EXPORT_SYMBOL_GPL(crypto_drop_spawn); -struct crypto_tfm *crypto_spawn_tfm(struct crypto_spawn *spawn, u32 type, - u32 mask) +static struct crypto_alg *crypto_spawn_alg(struct crypto_spawn *spawn) { struct crypto_alg *alg; struct crypto_alg *alg2; - struct crypto_tfm *tfm; down_read(&crypto_alg_sem); alg = spawn->alg; @@ -516,6 +584,19 @@ struct crypto_tfm *crypto_spawn_tfm(struct crypto_spawn *spawn, u32 type, return ERR_PTR(-EAGAIN); } + return alg; +} + +struct crypto_tfm *crypto_spawn_tfm(struct crypto_spawn *spawn, u32 type, + u32 mask) +{ + struct crypto_alg *alg; + struct crypto_tfm *tfm; + + alg = crypto_spawn_alg(spawn); + if (IS_ERR(alg)) + return ERR_CAST(alg); + tfm = ERR_PTR(-EINVAL); if (unlikely((alg->cra_flags ^ type) & mask)) goto out_put_alg; @@ -532,6 +613,27 @@ out_put_alg: } EXPORT_SYMBOL_GPL(crypto_spawn_tfm); +void *crypto_spawn_tfm2(struct crypto_spawn *spawn) +{ + struct crypto_alg *alg; + struct crypto_tfm *tfm; + + alg = crypto_spawn_alg(spawn); + if (IS_ERR(alg)) + return ERR_CAST(alg); + + tfm = crypto_create_tfm(alg, spawn->frontend); + if (IS_ERR(tfm)) + goto out_put_alg; + + return tfm; + +out_put_alg: + crypto_mod_put(alg); + return tfm; +} +EXPORT_SYMBOL_GPL(crypto_spawn_tfm2); + int crypto_register_notifier(struct notifier_block *nb) { return blocking_notifier_chain_register(&crypto_chain, nb); @@ -595,7 +697,9 @@ const char *crypto_attr_alg_name(struct rtattr *rta) } EXPORT_SYMBOL_GPL(crypto_attr_alg_name); -struct crypto_alg *crypto_attr_alg(struct rtattr *rta, u32 type, u32 mask) +struct crypto_alg *crypto_attr_alg2(struct rtattr *rta, + const struct crypto_type *frontend, + u32 type, u32 mask) { const char *name; int err; @@ -605,9 +709,9 @@ struct crypto_alg *crypto_attr_alg(struct rtattr *rta, u32 type, u32 mask) if (IS_ERR(name)) return ERR_PTR(err); - return crypto_alg_mod_lookup(name, type, mask); + return crypto_find_alg(name, frontend, type, mask); } -EXPORT_SYMBOL_GPL(crypto_attr_alg); +EXPORT_SYMBOL_GPL(crypto_attr_alg2); int crypto_attr_u32(struct rtattr *rta, u32 *num) { @@ -627,17 +731,20 @@ int crypto_attr_u32(struct rtattr *rta, u32 *num) } EXPORT_SYMBOL_GPL(crypto_attr_u32); -struct crypto_instance *crypto_alloc_instance(const char *name, - struct crypto_alg *alg) +void *crypto_alloc_instance2(const char *name, struct crypto_alg *alg, + unsigned int head) { struct crypto_instance *inst; - struct crypto_spawn *spawn; + char *p; int err; - inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); - if (!inst) + p = kzalloc(head + sizeof(*inst) + sizeof(struct crypto_spawn), + GFP_KERNEL); + if (!p) return ERR_PTR(-ENOMEM); + inst = (void *)(p + head); + err = -ENAMETOOLONG; if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", name, alg->cra_name) >= CRYPTO_MAX_ALG_NAME) @@ -647,6 +754,25 @@ struct crypto_instance *crypto_alloc_instance(const char *name, name, alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto err_free_inst; + return p; + +err_free_inst: + kfree(p); + return ERR_PTR(err); +} +EXPORT_SYMBOL_GPL(crypto_alloc_instance2); + +struct crypto_instance *crypto_alloc_instance(const char *name, + struct crypto_alg *alg) +{ + struct crypto_instance *inst; + struct crypto_spawn *spawn; + int err; + + inst = crypto_alloc_instance2(name, alg, 0); + if (IS_ERR(inst)) + goto out; + spawn = crypto_instance_ctx(inst); err = crypto_init_spawn(spawn, alg, inst, CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC); @@ -658,7 +784,10 @@ struct crypto_instance *crypto_alloc_instance(const char *name, err_free_inst: kfree(inst); - return ERR_PTR(err); + inst = ERR_PTR(err); + +out: + return inst; } EXPORT_SYMBOL_GPL(crypto_alloc_instance); @@ -692,7 +821,7 @@ out: } EXPORT_SYMBOL_GPL(crypto_enqueue_request); -struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue) +void *__crypto_dequeue_request(struct crypto_queue *queue, unsigned int offset) { struct list_head *request; @@ -707,7 +836,14 @@ struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue) request = queue->list.next; list_del(request); - return list_entry(request, struct crypto_async_request, list); + return (char *)list_entry(request, struct crypto_async_request, list) - + offset; +} +EXPORT_SYMBOL_GPL(__crypto_dequeue_request); + +struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue) +{ + return __crypto_dequeue_request(queue, 0); } EXPORT_SYMBOL_GPL(crypto_dequeue_request); diff --git a/crypto/algboss.c b/crypto/algboss.c index 9908dd830c26..791d194958fa 100644 --- a/crypto/algboss.c +++ b/crypto/algboss.c @@ -19,6 +19,7 @@ #include <linux/notifier.h> #include <linux/rtnetlink.h> #include <linux/sched.h> +#include <linux/slab.h> #include <linux/string.h> #include "internal.h" @@ -68,6 +69,11 @@ static int cryptomgr_probe(void *data) goto err; do { + if (tmpl->create) { + err = tmpl->create(tmpl, param->tb); + continue; + } + inst = tmpl->alloc(param->tb); if (IS_ERR(inst)) err = PTR_ERR(inst); @@ -206,6 +212,10 @@ static int cryptomgr_test(void *data) u32 type = param->type; int err = 0; +#ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS + goto skiptest; +#endif + if (type & CRYPTO_ALG_TESTED) goto skiptest; diff --git a/crypto/ansi_cprng.c b/crypto/ansi_cprng.c index d80ed4c1e009..2bc332142849 100644 --- a/crypto/ansi_cprng.c +++ b/crypto/ansi_cprng.c @@ -85,7 +85,7 @@ static void xor_vectors(unsigned char *in1, unsigned char *in2, * Returns DEFAULT_BLK_SZ bytes of random data per call * returns 0 if generation succeded, <0 if something went wrong */ -static int _get_more_prng_bytes(struct prng_context *ctx) +static int _get_more_prng_bytes(struct prng_context *ctx, int cont_test) { int i; unsigned char tmp[DEFAULT_BLK_SZ]; @@ -132,7 +132,7 @@ static int _get_more_prng_bytes(struct prng_context *ctx) */ if (!memcmp(ctx->rand_data, ctx->last_rand_data, DEFAULT_BLK_SZ)) { - if (fips_enabled) { + if (cont_test) { panic("cprng %p Failed repetition check!\n", ctx); } @@ -185,18 +185,15 @@ static int _get_more_prng_bytes(struct prng_context *ctx) } /* Our exported functions */ -static int get_prng_bytes(char *buf, size_t nbytes, struct prng_context *ctx) +static int get_prng_bytes(char *buf, size_t nbytes, struct prng_context *ctx, + int do_cont_test) { - unsigned long flags; unsigned char *ptr = buf; unsigned int byte_count = (unsigned int)nbytes; int err; - if (nbytes < 0) - return -EINVAL; - - spin_lock_irqsave(&ctx->prng_lock, flags); + spin_lock_bh(&ctx->prng_lock); err = -EINVAL; if (ctx->flags & PRNG_NEED_RESET) @@ -221,7 +218,7 @@ static int get_prng_bytes(char *buf, size_t nbytes, struct prng_context *ctx) remainder: if (ctx->rand_data_valid == DEFAULT_BLK_SZ) { - if (_get_more_prng_bytes(ctx) < 0) { + if (_get_more_prng_bytes(ctx, do_cont_test) < 0) { memset(buf, 0, nbytes); err = -EINVAL; goto done; @@ -248,7 +245,7 @@ empty_rbuf: */ for (; byte_count >= DEFAULT_BLK_SZ; byte_count -= DEFAULT_BLK_SZ) { if (ctx->rand_data_valid == DEFAULT_BLK_SZ) { - if (_get_more_prng_bytes(ctx) < 0) { + if (_get_more_prng_bytes(ctx, do_cont_test) < 0) { memset(buf, 0, nbytes); err = -EINVAL; goto done; @@ -268,7 +265,7 @@ empty_rbuf: goto remainder; done: - spin_unlock_irqrestore(&ctx->prng_lock, flags); + spin_unlock_bh(&ctx->prng_lock); dbgprint(KERN_CRIT "returning %d from get_prng_bytes in context %p\n", err, ctx); return err; @@ -284,10 +281,9 @@ static int reset_prng_context(struct prng_context *ctx, unsigned char *V, unsigned char *DT) { int ret; - int rc = -EINVAL; unsigned char *prng_key; - spin_lock(&ctx->prng_lock); + spin_lock_bh(&ctx->prng_lock); ctx->flags |= PRNG_NEED_RESET; prng_key = (key != NULL) ? key : (unsigned char *)DEFAULT_PRNG_KEY; @@ -308,34 +304,20 @@ static int reset_prng_context(struct prng_context *ctx, memset(ctx->rand_data, 0, DEFAULT_BLK_SZ); memset(ctx->last_rand_data, 0, DEFAULT_BLK_SZ); - if (ctx->tfm) - crypto_free_cipher(ctx->tfm); - - ctx->tfm = crypto_alloc_cipher("aes", 0, 0); - if (IS_ERR(ctx->tfm)) { - dbgprint(KERN_CRIT "Failed to alloc tfm for context %p\n", - ctx); - ctx->tfm = NULL; - goto out; - } - ctx->rand_data_valid = DEFAULT_BLK_SZ; ret = crypto_cipher_setkey(ctx->tfm, prng_key, klen); if (ret) { dbgprint(KERN_CRIT "PRNG: setkey() failed flags=%x\n", crypto_cipher_get_flags(ctx->tfm)); - crypto_free_cipher(ctx->tfm); goto out; } - rc = 0; + ret = 0; ctx->flags &= ~PRNG_NEED_RESET; out: - spin_unlock(&ctx->prng_lock); - - return rc; - + spin_unlock_bh(&ctx->prng_lock); + return ret; } static int cprng_init(struct crypto_tfm *tfm) @@ -343,6 +325,12 @@ static int cprng_init(struct crypto_tfm *tfm) struct prng_context *ctx = crypto_tfm_ctx(tfm); spin_lock_init(&ctx->prng_lock); + ctx->tfm = crypto_alloc_cipher("aes", 0, 0); + if (IS_ERR(ctx->tfm)) { + dbgprint(KERN_CRIT "Failed to alloc tfm for context %p\n", + ctx); + return PTR_ERR(ctx->tfm); + } if (reset_prng_context(ctx, NULL, DEFAULT_PRNG_KSZ, NULL, NULL) < 0) return -EINVAL; @@ -366,7 +354,7 @@ static int cprng_get_random(struct crypto_rng *tfm, u8 *rdata, { struct prng_context *prng = crypto_rng_ctx(tfm); - return get_prng_bytes(rdata, dlen, prng); + return get_prng_bytes(rdata, dlen, prng, 0); } /* @@ -414,26 +402,79 @@ static struct crypto_alg rng_alg = { } }; +#ifdef CONFIG_CRYPTO_FIPS +static int fips_cprng_get_random(struct crypto_rng *tfm, u8 *rdata, + unsigned int dlen) +{ + struct prng_context *prng = crypto_rng_ctx(tfm); + + return get_prng_bytes(rdata, dlen, prng, 1); +} + +static int fips_cprng_reset(struct crypto_rng *tfm, u8 *seed, unsigned int slen) +{ + u8 rdata[DEFAULT_BLK_SZ]; + int rc; + + struct prng_context *prng = crypto_rng_ctx(tfm); + + rc = cprng_reset(tfm, seed, slen); + + if (!rc) + goto out; + + /* this primes our continuity test */ + rc = get_prng_bytes(rdata, DEFAULT_BLK_SZ, prng, 0); + prng->rand_data_valid = DEFAULT_BLK_SZ; + +out: + return rc; +} + +static struct crypto_alg fips_rng_alg = { + .cra_name = "fips(ansi_cprng)", + .cra_driver_name = "fips_ansi_cprng", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_RNG, + .cra_ctxsize = sizeof(struct prng_context), + .cra_type = &crypto_rng_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(rng_alg.cra_list), + .cra_init = cprng_init, + .cra_exit = cprng_exit, + .cra_u = { + .rng = { + .rng_make_random = fips_cprng_get_random, + .rng_reset = fips_cprng_reset, + .seedsize = DEFAULT_PRNG_KSZ + 2*DEFAULT_BLK_SZ, + } + } +}; +#endif /* Module initalization */ static int __init prng_mod_init(void) { - int ret = 0; + int rc = 0; - if (fips_enabled) - rng_alg.cra_priority += 200; + rc = crypto_register_alg(&rng_alg); +#ifdef CONFIG_CRYPTO_FIPS + if (rc) + goto out; - ret = crypto_register_alg(&rng_alg); + rc = crypto_register_alg(&fips_rng_alg); - if (ret) - goto out; out: - return 0; +#endif + return rc; } static void __exit prng_mod_fini(void) { crypto_unregister_alg(&rng_alg); +#ifdef CONFIG_CRYPTO_FIPS + crypto_unregister_alg(&fips_rng_alg); +#endif return; } diff --git a/crypto/anubis.c b/crypto/anubis.c index e42c3a8ba4aa..77530d571c96 100644 --- a/crypto/anubis.c +++ b/crypto/anubis.c @@ -469,14 +469,13 @@ static int anubis_setkey(struct crypto_tfm *tfm, const u8 *in_key, u32 kappa[ANUBIS_MAX_N]; u32 inter[ANUBIS_MAX_N]; - switch (key_len) - { + switch (key_len) { case 16: case 20: case 24: case 28: case 32: case 36: case 40: break; default: *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; - return - EINVAL; + return -EINVAL; } ctx->key_len = key_len * 8; @@ -530,23 +529,24 @@ static int anubis_setkey(struct crypto_tfm *tfm, const u8 *in_key, /* * compute kappa^{r+1} from kappa^r: */ - if (r == R) { + if (r == R) break; - } for (i = 0; i < N; i++) { int j = i; inter[i] = T0[(kappa[j--] >> 24) ]; - if (j < 0) j = N - 1; + if (j < 0) + j = N - 1; inter[i] ^= T1[(kappa[j--] >> 16) & 0xff]; - if (j < 0) j = N - 1; + if (j < 0) + j = N - 1; inter[i] ^= T2[(kappa[j--] >> 8) & 0xff]; - if (j < 0) j = N - 1; + if (j < 0) + j = N - 1; inter[i] ^= T3[(kappa[j ] ) & 0xff]; } kappa[0] = inter[0] ^ rc[r]; - for (i = 1; i < N; i++) { + for (i = 1; i < N; i++) kappa[i] = inter[i]; - } } /* @@ -690,7 +690,7 @@ static struct crypto_alg anubis_alg = { static int __init anubis_mod_init(void) { int ret = 0; - + ret = crypto_register_alg(&anubis_alg); return ret; } diff --git a/crypto/api.c b/crypto/api.c index d5944f92b416..033a7147e5eb 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -10,7 +10,7 @@ * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) + * Software Foundation; either version 2 of the License, or (at your option) * any later version. * */ @@ -285,21 +285,14 @@ static int crypto_init_ops(struct crypto_tfm *tfm, u32 type, u32 mask) switch (crypto_tfm_alg_type(tfm)) { case CRYPTO_ALG_TYPE_CIPHER: return crypto_init_cipher_ops(tfm); - - case CRYPTO_ALG_TYPE_DIGEST: - if ((mask & CRYPTO_ALG_TYPE_HASH_MASK) != - CRYPTO_ALG_TYPE_HASH_MASK) - return crypto_init_digest_ops_async(tfm); - else - return crypto_init_digest_ops(tfm); case CRYPTO_ALG_TYPE_COMPRESS: return crypto_init_compress_ops(tfm); - + default: break; } - + BUG(); return -EINVAL; } @@ -318,18 +311,13 @@ static void crypto_exit_ops(struct crypto_tfm *tfm) case CRYPTO_ALG_TYPE_CIPHER: crypto_exit_cipher_ops(tfm); break; - - case CRYPTO_ALG_TYPE_DIGEST: - crypto_exit_digest_ops(tfm); - break; - + case CRYPTO_ALG_TYPE_COMPRESS: crypto_exit_compress_ops(tfm); break; - + default: BUG(); - } } @@ -349,11 +337,7 @@ static unsigned int crypto_ctxsize(struct crypto_alg *alg, u32 type, u32 mask) case CRYPTO_ALG_TYPE_CIPHER: len += crypto_cipher_ctxsize(alg); break; - - case CRYPTO_ALG_TYPE_DIGEST: - len += crypto_digest_ctxsize(alg); - break; - + case CRYPTO_ALG_TYPE_COMPRESS: len += crypto_compress_ctxsize(alg); break; @@ -472,7 +456,7 @@ void *crypto_create_tfm(struct crypto_alg *alg, int err = -ENOMEM; tfmsize = frontend->tfmsize; - total = tfmsize + sizeof(*tfm) + frontend->extsize(alg, frontend); + total = tfmsize + sizeof(*tfm) + frontend->extsize(alg); mem = kzalloc(total, GFP_KERNEL); if (mem == NULL) @@ -481,7 +465,7 @@ void *crypto_create_tfm(struct crypto_alg *alg, tfm = (struct crypto_tfm *)(mem + tfmsize); tfm->__crt_alg = alg; - err = frontend->init_tfm(tfm, frontend); + err = frontend->init_tfm(tfm); if (err) goto out_free_tfm; @@ -503,6 +487,27 @@ out: } EXPORT_SYMBOL_GPL(crypto_create_tfm); +struct crypto_alg *crypto_find_alg(const char *alg_name, + const struct crypto_type *frontend, + u32 type, u32 mask) +{ + struct crypto_alg *(*lookup)(const char *name, u32 type, u32 mask) = + crypto_alg_mod_lookup; + + if (frontend) { + type &= frontend->maskclear; + mask &= frontend->maskclear; + type |= frontend->type; + mask |= frontend->maskset; + + if (frontend->lookup) + lookup = frontend->lookup; + } + + return lookup(alg_name, type, mask); +} +EXPORT_SYMBOL_GPL(crypto_find_alg); + /* * crypto_alloc_tfm - Locate algorithm and allocate transform * @alg_name: Name of algorithm @@ -526,21 +531,13 @@ EXPORT_SYMBOL_GPL(crypto_create_tfm); void *crypto_alloc_tfm(const char *alg_name, const struct crypto_type *frontend, u32 type, u32 mask) { - struct crypto_alg *(*lookup)(const char *name, u32 type, u32 mask); void *tfm; int err; - type &= frontend->maskclear; - mask &= frontend->maskclear; - type |= frontend->type; - mask |= frontend->maskset; - - lookup = frontend->lookup ?: crypto_alg_mod_lookup; - for (;;) { struct crypto_alg *alg; - alg = lookup(alg_name, type, mask); + alg = crypto_find_alg(alg_name, frontend, type, mask); if (IS_ERR(alg)) { err = PTR_ERR(alg); goto err; @@ -595,12 +592,12 @@ int crypto_has_alg(const char *name, u32 type, u32 mask) { int ret = 0; struct crypto_alg *alg = crypto_alg_mod_lookup(name, type, mask); - + if (!IS_ERR(alg)) { crypto_mod_put(alg); ret = 1; } - + return ret; } EXPORT_SYMBOL_GPL(crypto_has_alg); diff --git a/crypto/async_tx/Kconfig b/crypto/async_tx/Kconfig index d8fb39145986..1b11abbb5c91 100644 --- a/crypto/async_tx/Kconfig +++ b/crypto/async_tx/Kconfig @@ -14,3 +14,18 @@ config ASYNC_MEMSET tristate select ASYNC_CORE +config ASYNC_PQ + tristate + select ASYNC_CORE + +config ASYNC_RAID6_RECOV + tristate + select ASYNC_CORE + select ASYNC_PQ + select ASYNC_XOR + +config ASYNC_TX_DISABLE_PQ_VAL_DMA + bool + +config ASYNC_TX_DISABLE_XOR_VAL_DMA + bool diff --git a/crypto/async_tx/Makefile b/crypto/async_tx/Makefile index 27baa7d52fbc..d1e0e6f72bc1 100644 --- a/crypto/async_tx/Makefile +++ b/crypto/async_tx/Makefile @@ -2,3 +2,6 @@ obj-$(CONFIG_ASYNC_CORE) += async_tx.o obj-$(CONFIG_ASYNC_MEMCPY) += async_memcpy.o obj-$(CONFIG_ASYNC_MEMSET) += async_memset.o obj-$(CONFIG_ASYNC_XOR) += async_xor.o +obj-$(CONFIG_ASYNC_PQ) += async_pq.o +obj-$(CONFIG_ASYNC_RAID6_RECOV) += async_raid6_recov.o +obj-$(CONFIG_ASYNC_RAID6_TEST) += raid6test.o diff --git a/crypto/async_tx/async_memcpy.c b/crypto/async_tx/async_memcpy.c index ddccfb01c416..518c22bd9562 100644 --- a/crypto/async_tx/async_memcpy.c +++ b/crypto/async_tx/async_memcpy.c @@ -33,28 +33,31 @@ * async_memcpy - attempt to copy memory with a dma engine. * @dest: destination page * @src: src page - * @offset: offset in pages to start transaction + * @dest_offset: offset into 'dest' to start transaction + * @src_offset: offset into 'src' to start transaction * @len: length in bytes - * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK, - * @depend_tx: memcpy depends on the result of this transaction - * @cb_fn: function to call when the memcpy completes - * @cb_param: parameter to pass to the callback routine + * @submit: submission / completion modifiers + * + * honored flags: ASYNC_TX_ACK */ struct dma_async_tx_descriptor * async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset, - unsigned int src_offset, size_t len, enum async_tx_flags flags, - struct dma_async_tx_descriptor *depend_tx, - dma_async_tx_callback cb_fn, void *cb_param) + unsigned int src_offset, size_t len, + struct async_submit_ctl *submit) { - struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_MEMCPY, + struct dma_chan *chan = async_tx_find_channel(submit, DMA_MEMCPY, &dest, 1, &src, 1, len); struct dma_device *device = chan ? chan->device : NULL; struct dma_async_tx_descriptor *tx = NULL; - if (device) { + if (device && is_dma_copy_aligned(device, src_offset, dest_offset, len)) { dma_addr_t dma_dest, dma_src; - unsigned long dma_prep_flags = cb_fn ? DMA_PREP_INTERRUPT : 0; + unsigned long dma_prep_flags = 0; + if (submit->cb_fn) + dma_prep_flags |= DMA_PREP_INTERRUPT; + if (submit->flags & ASYNC_TX_FENCE) + dma_prep_flags |= DMA_PREP_FENCE; dma_dest = dma_map_page(device->dev, dest, dest_offset, len, DMA_FROM_DEVICE); @@ -67,42 +70,29 @@ async_memcpy(struct page *dest, struct page *src, unsigned int dest_offset, if (tx) { pr_debug("%s: (async) len: %zu\n", __func__, len); - async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param); + async_tx_submit(chan, tx, submit); } else { void *dest_buf, *src_buf; pr_debug("%s: (sync) len: %zu\n", __func__, len); /* wait for any prerequisite operations */ - async_tx_quiesce(&depend_tx); + async_tx_quiesce(&submit->depend_tx); dest_buf = kmap_atomic(dest, KM_USER0) + dest_offset; src_buf = kmap_atomic(src, KM_USER1) + src_offset; memcpy(dest_buf, src_buf, len); - kunmap_atomic(dest_buf, KM_USER0); kunmap_atomic(src_buf, KM_USER1); + kunmap_atomic(dest_buf, KM_USER0); - async_tx_sync_epilog(cb_fn, cb_param); + async_tx_sync_epilog(submit); } return tx; } EXPORT_SYMBOL_GPL(async_memcpy); -static int __init async_memcpy_init(void) -{ - return 0; -} - -static void __exit async_memcpy_exit(void) -{ - do { } while (0); -} - -module_init(async_memcpy_init); -module_exit(async_memcpy_exit); - MODULE_AUTHOR("Intel Corporation"); MODULE_DESCRIPTION("asynchronous memcpy api"); MODULE_LICENSE("GPL"); diff --git a/crypto/async_tx/async_memset.c b/crypto/async_tx/async_memset.c index 5b5eb99bb244..58e4a8752aee 100644 --- a/crypto/async_tx/async_memset.c +++ b/crypto/async_tx/async_memset.c @@ -35,26 +35,26 @@ * @val: fill value * @offset: offset in pages to start transaction * @len: length in bytes - * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK - * @depend_tx: memset depends on the result of this transaction - * @cb_fn: function to call when the memcpy completes - * @cb_param: parameter to pass to the callback routine + * + * honored flags: ASYNC_TX_ACK */ struct dma_async_tx_descriptor * -async_memset(struct page *dest, int val, unsigned int offset, - size_t len, enum async_tx_flags flags, - struct dma_async_tx_descriptor *depend_tx, - dma_async_tx_callback cb_fn, void *cb_param) +async_memset(struct page *dest, int val, unsigned int offset, size_t len, + struct async_submit_ctl *submit) { - struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_MEMSET, + struct dma_chan *chan = async_tx_find_channel(submit, DMA_MEMSET, &dest, 1, NULL, 0, len); struct dma_device *device = chan ? chan->device : NULL; struct dma_async_tx_descriptor *tx = NULL; - if (device) { + if (device && is_dma_fill_aligned(device, offset, 0, len)) { dma_addr_t dma_dest; - unsigned long dma_prep_flags = cb_fn ? DMA_PREP_INTERRUPT : 0; + unsigned long dma_prep_flags = 0; + if (submit->cb_fn) + dma_prep_flags |= DMA_PREP_INTERRUPT; + if (submit->flags & ASYNC_TX_FENCE) + dma_prep_flags |= DMA_PREP_FENCE; dma_dest = dma_map_page(device->dev, dest, offset, len, DMA_FROM_DEVICE); @@ -64,38 +64,25 @@ async_memset(struct page *dest, int val, unsigned int offset, if (tx) { pr_debug("%s: (async) len: %zu\n", __func__, len); - async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param); + async_tx_submit(chan, tx, submit); } else { /* run the memset synchronously */ void *dest_buf; pr_debug("%s: (sync) len: %zu\n", __func__, len); - dest_buf = (void *) (((char *) page_address(dest)) + offset); + dest_buf = page_address(dest) + offset; /* wait for any prerequisite operations */ - async_tx_quiesce(&depend_tx); + async_tx_quiesce(&submit->depend_tx); memset(dest_buf, val, len); - async_tx_sync_epilog(cb_fn, cb_param); + async_tx_sync_epilog(submit); } return tx; } EXPORT_SYMBOL_GPL(async_memset); -static int __init async_memset_init(void) -{ - return 0; -} - -static void __exit async_memset_exit(void) -{ - do { } while (0); -} - -module_init(async_memset_init); -module_exit(async_memset_exit); - MODULE_AUTHOR("Intel Corporation"); MODULE_DESCRIPTION("asynchronous memset api"); MODULE_LICENSE("GPL"); diff --git a/crypto/async_tx/async_pq.c b/crypto/async_tx/async_pq.c new file mode 100644 index 000000000000..fdd8257d35d9 --- /dev/null +++ b/crypto/async_tx/async_pq.c @@ -0,0 +1,416 @@ +/* + * Copyright(c) 2007 Yuri Tikhonov <yur@emcraft.com> + * Copyright(c) 2009 Intel Corporation + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 + * Temple Place - Suite 330, Boston, MA 02111-1307, USA. + * + * The full GNU General Public License is included in this distribution in the + * file called COPYING. + */ +#include <linux/kernel.h> +#include <linux/interrupt.h> +#include <linux/dma-mapping.h> +#include <linux/raid/pq.h> +#include <linux/async_tx.h> +#include <linux/gfp.h> + +/** + * pq_scribble_page - space to hold throwaway P or Q buffer for + * synchronous gen_syndrome + */ +static struct page *pq_scribble_page; + +/* the struct page *blocks[] parameter passed to async_gen_syndrome() + * and async_syndrome_val() contains the 'P' destination address at + * blocks[disks-2] and the 'Q' destination address at blocks[disks-1] + * + * note: these are macros as they are used as lvalues + */ +#define P(b, d) (b[d-2]) +#define Q(b, d) (b[d-1]) + +/** + * do_async_gen_syndrome - asynchronously calculate P and/or Q + */ +static __async_inline struct dma_async_tx_descriptor * +do_async_gen_syndrome(struct dma_chan *chan, struct page **blocks, + const unsigned char *scfs, unsigned int offset, int disks, + size_t len, dma_addr_t *dma_src, + struct async_submit_ctl *submit) +{ + struct dma_async_tx_descriptor *tx = NULL; + struct dma_device *dma = chan->device; + enum dma_ctrl_flags dma_flags = 0; + enum async_tx_flags flags_orig = submit->flags; + dma_async_tx_callback cb_fn_orig = submit->cb_fn; + dma_async_tx_callback cb_param_orig = submit->cb_param; + int src_cnt = disks - 2; + unsigned char coefs[src_cnt]; + unsigned short pq_src_cnt; + dma_addr_t dma_dest[2]; + int src_off = 0; + int idx; + int i; + + /* DMAs use destinations as sources, so use BIDIRECTIONAL mapping */ + if (P(blocks, disks)) + dma_dest[0] = dma_map_page(dma->dev, P(blocks, disks), offset, + len, DMA_BIDIRECTIONAL); + else + dma_flags |= DMA_PREP_PQ_DISABLE_P; + if (Q(blocks, disks)) + dma_dest[1] = dma_map_page(dma->dev, Q(blocks, disks), offset, + len, DMA_BIDIRECTIONAL); + else + dma_flags |= DMA_PREP_PQ_DISABLE_Q; + + /* convert source addresses being careful to collapse 'empty' + * sources and update the coefficients accordingly + */ + for (i = 0, idx = 0; i < src_cnt; i++) { + if (blocks[i] == NULL) + continue; + dma_src[idx] = dma_map_page(dma->dev, blocks[i], offset, len, + DMA_TO_DEVICE); + coefs[idx] = scfs[i]; + idx++; + } + src_cnt = idx; + + while (src_cnt > 0) { + submit->flags = flags_orig; + pq_src_cnt = min(src_cnt, dma_maxpq(dma, dma_flags)); + /* if we are submitting additional pqs, leave the chain open, + * clear the callback parameters, and leave the destination + * buffers mapped + */ + if (src_cnt > pq_src_cnt) { + submit->flags &= ~ASYNC_TX_ACK; + submit->flags |= ASYNC_TX_FENCE; + dma_flags |= DMA_COMPL_SKIP_DEST_UNMAP; + submit->cb_fn = NULL; + submit->cb_param = NULL; + } else { + dma_flags &= ~DMA_COMPL_SKIP_DEST_UNMAP; + submit->cb_fn = cb_fn_orig; + submit->cb_param = cb_param_orig; + if (cb_fn_orig) + dma_flags |= DMA_PREP_INTERRUPT; + } + if (submit->flags & ASYNC_TX_FENCE) + dma_flags |= DMA_PREP_FENCE; + + /* Since we have clobbered the src_list we are committed + * to doing this asynchronously. Drivers force forward + * progress in case they can not provide a descriptor + */ + for (;;) { + tx = dma->device_prep_dma_pq(chan, dma_dest, + &dma_src[src_off], + pq_src_cnt, + &coefs[src_off], len, + dma_flags); + if (likely(tx)) + break; + async_tx_quiesce(&submit->depend_tx); + dma_async_issue_pending(chan); + } + + async_tx_submit(chan, tx, submit); + submit->depend_tx = tx; + + /* drop completed sources */ + src_cnt -= pq_src_cnt; + src_off += pq_src_cnt; + + dma_flags |= DMA_PREP_CONTINUE; + } + + return tx; +} + +/** + * do_sync_gen_syndrome - synchronously calculate a raid6 syndrome + */ +static void +do_sync_gen_syndrome(struct page **blocks, unsigned int offset, int disks, + size_t len, struct async_submit_ctl *submit) +{ + void **srcs; + int i; + + if (submit->scribble) + srcs = submit->scribble; + else + srcs = (void **) blocks; + + for (i = 0; i < disks; i++) { + if (blocks[i] == NULL) { + BUG_ON(i > disks - 3); /* P or Q can't be zero */ + srcs[i] = (void*)raid6_empty_zero_page; + } else + srcs[i] = page_address(blocks[i]) + offset; + } + raid6_call.gen_syndrome(disks, len, srcs); + async_tx_sync_epilog(submit); +} + +/** + * async_gen_syndrome - asynchronously calculate a raid6 syndrome + * @blocks: source blocks from idx 0..disks-3, P @ disks-2 and Q @ disks-1 + * @offset: common offset into each block (src and dest) to start transaction + * @disks: number of blocks (including missing P or Q, see below) + * @len: length of operation in bytes + * @submit: submission/completion modifiers + * + * General note: This routine assumes a field of GF(2^8) with a + * primitive polynomial of 0x11d and a generator of {02}. + * + * 'disks' note: callers can optionally omit either P or Q (but not + * both) from the calculation by setting blocks[disks-2] or + * blocks[disks-1] to NULL. When P or Q is omitted 'len' must be <= + * PAGE_SIZE as a temporary buffer of this size is used in the + * synchronous path. 'disks' always accounts for both destination + * buffers. If any source buffers (blocks[i] where i < disks - 2) are + * set to NULL those buffers will be replaced with the raid6_zero_page + * in the synchronous path and omitted in the hardware-asynchronous + * path. + * + * 'blocks' note: if submit->scribble is NULL then the contents of + * 'blocks' may be overwritten to perform address conversions + * (dma_map_page() or page_address()). + */ +struct dma_async_tx_descriptor * +async_gen_syndrome(struct page **blocks, unsigned int offset, int disks, + size_t len, struct async_submit_ctl *submit) +{ + int src_cnt = disks - 2; + struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ, + &P(blocks, disks), 2, + blocks, src_cnt, len); + struct dma_device *device = chan ? chan->device : NULL; + dma_addr_t *dma_src = NULL; + + BUG_ON(disks > 255 || !(P(blocks, disks) || Q(blocks, disks))); + + if (submit->scribble) + dma_src = submit->scribble; + else if (sizeof(dma_addr_t) <= sizeof(struct page *)) + dma_src = (dma_addr_t *) blocks; + + if (dma_src && device && + (src_cnt <= dma_maxpq(device, 0) || + dma_maxpq(device, DMA_PREP_CONTINUE) > 0) && + is_dma_pq_aligned(device, offset, 0, len)) { + /* run the p+q asynchronously */ + pr_debug("%s: (async) disks: %d len: %zu\n", + __func__, disks, len); + return do_async_gen_syndrome(chan, blocks, raid6_gfexp, offset, + disks, len, dma_src, submit); + } + + /* run the pq synchronously */ + pr_debug("%s: (sync) disks: %d len: %zu\n", __func__, disks, len); + + /* wait for any prerequisite operations */ + async_tx_quiesce(&submit->depend_tx); + + if (!P(blocks, disks)) { + P(blocks, disks) = pq_scribble_page; + BUG_ON(len + offset > PAGE_SIZE); + } + if (!Q(blocks, disks)) { + Q(blocks, disks) = pq_scribble_page; + BUG_ON(len + offset > PAGE_SIZE); + } + do_sync_gen_syndrome(blocks, offset, disks, len, submit); + + return NULL; +} +EXPORT_SYMBOL_GPL(async_gen_syndrome); + +static inline struct dma_chan * +pq_val_chan(struct async_submit_ctl *submit, struct page **blocks, int disks, size_t len) +{ + #ifdef CONFIG_ASYNC_TX_DISABLE_PQ_VAL_DMA + return NULL; + #endif + return async_tx_find_channel(submit, DMA_PQ_VAL, NULL, 0, blocks, + disks, len); +} + +/** + * async_syndrome_val - asynchronously validate a raid6 syndrome + * @blocks: source blocks from idx 0..disks-3, P @ disks-2 and Q @ disks-1 + * @offset: common offset into each block (src and dest) to start transaction + * @disks: number of blocks (including missing P or Q, see below) + * @len: length of operation in bytes + * @pqres: on val failure SUM_CHECK_P_RESULT and/or SUM_CHECK_Q_RESULT are set + * @spare: temporary result buffer for the synchronous case + * @submit: submission / completion modifiers + * + * The same notes from async_gen_syndrome apply to the 'blocks', + * and 'disks' parameters of this routine. The synchronous path + * requires a temporary result buffer and submit->scribble to be + * specified. + */ +struct dma_async_tx_descriptor * +async_syndrome_val(struct page **blocks, unsigned int offset, int disks, + size_t len, enum sum_check_flags *pqres, struct page *spare, + struct async_submit_ctl *submit) +{ + struct dma_chan *chan = pq_val_chan(submit, blocks, disks, len); + struct dma_device *device = chan ? chan->device : NULL; + struct dma_async_tx_descriptor *tx; + unsigned char coefs[disks-2]; + enum dma_ctrl_flags dma_flags = submit->cb_fn ? DMA_PREP_INTERRUPT : 0; + dma_addr_t *dma_src = NULL; + int src_cnt = 0; + + BUG_ON(disks < 4); + + if (submit->scribble) + dma_src = submit->scribble; + else if (sizeof(dma_addr_t) <= sizeof(struct page *)) + dma_src = (dma_addr_t *) blocks; + + if (dma_src && device && disks <= dma_maxpq(device, 0) && + is_dma_pq_aligned(device, offset, 0, len)) { + struct device *dev = device->dev; + dma_addr_t *pq = &dma_src[disks-2]; + int i; + + pr_debug("%s: (async) disks: %d len: %zu\n", + __func__, disks, len); + if (!P(blocks, disks)) + dma_flags |= DMA_PREP_PQ_DISABLE_P; + else + pq[0] = dma_map_page(dev, P(blocks, disks), + offset, len, + DMA_TO_DEVICE); + if (!Q(blocks, disks)) + dma_flags |= DMA_PREP_PQ_DISABLE_Q; + else + pq[1] = dma_map_page(dev, Q(blocks, disks), + offset, len, + DMA_TO_DEVICE); + + if (submit->flags & ASYNC_TX_FENCE) + dma_flags |= DMA_PREP_FENCE; + for (i = 0; i < disks-2; i++) + if (likely(blocks[i])) { + dma_src[src_cnt] = dma_map_page(dev, blocks[i], + offset, len, + DMA_TO_DEVICE); + coefs[src_cnt] = raid6_gfexp[i]; + src_cnt++; + } + + for (;;) { + tx = device->device_prep_dma_pq_val(chan, pq, dma_src, + src_cnt, + coefs, + len, pqres, + dma_flags); + if (likely(tx)) + break; + async_tx_quiesce(&submit->depend_tx); + dma_async_issue_pending(chan); + } + async_tx_submit(chan, tx, submit); + + return tx; + } else { + struct page *p_src = P(blocks, disks); + struct page *q_src = Q(blocks, disks); + enum async_tx_flags flags_orig = submit->flags; + dma_async_tx_callback cb_fn_orig = submit->cb_fn; + void *scribble = submit->scribble; + void *cb_param_orig = submit->cb_param; + void *p, *q, *s; + + pr_debug("%s: (sync) disks: %d len: %zu\n", + __func__, disks, len); + + /* caller must provide a temporary result buffer and + * allow the input parameters to be preserved + */ + BUG_ON(!spare || !scribble); + + /* wait for any prerequisite operations */ + async_tx_quiesce(&submit->depend_tx); + + /* recompute p and/or q into the temporary buffer and then + * check to see the result matches the current value + */ + tx = NULL; + *pqres = 0; + if (p_src) { + init_async_submit(submit, ASYNC_TX_XOR_ZERO_DST, NULL, + NULL, NULL, scribble); + tx = async_xor(spare, blocks, offset, disks-2, len, submit); + async_tx_quiesce(&tx); + p = page_address(p_src) + offset; + s = page_address(spare) + offset; + *pqres |= !!memcmp(p, s, len) << SUM_CHECK_P; + } + + if (q_src) { + P(blocks, disks) = NULL; + Q(blocks, disks) = spare; + init_async_submit(submit, 0, NULL, NULL, NULL, scribble); + tx = async_gen_syndrome(blocks, offset, disks, len, submit); + async_tx_quiesce(&tx); + q = page_address(q_src) + offset; + s = page_address(spare) + offset; + *pqres |= !!memcmp(q, s, len) << SUM_CHECK_Q; + } + + /* restore P, Q and submit */ + P(blocks, disks) = p_src; + Q(blocks, disks) = q_src; + + submit->cb_fn = cb_fn_orig; + submit->cb_param = cb_param_orig; + submit->flags = flags_orig; + async_tx_sync_epilog(submit); + + return NULL; + } +} +EXPORT_SYMBOL_GPL(async_syndrome_val); + +static int __init async_pq_init(void) +{ + pq_scribble_page = alloc_page(GFP_KERNEL); + + if (pq_scribble_page) + return 0; + + pr_err("%s: failed to allocate required spare page\n", __func__); + + return -ENOMEM; +} + +static void __exit async_pq_exit(void) +{ + put_page(pq_scribble_page); +} + +module_init(async_pq_init); +module_exit(async_pq_exit); + +MODULE_DESCRIPTION("asynchronous raid6 syndrome generation/validation"); +MODULE_LICENSE("GPL"); diff --git a/crypto/async_tx/async_raid6_recov.c b/crypto/async_tx/async_raid6_recov.c new file mode 100644 index 000000000000..ce038d861eb9 --- /dev/null +++ b/crypto/async_tx/async_raid6_recov.c @@ -0,0 +1,505 @@ +/* + * Asynchronous RAID-6 recovery calculations ASYNC_TX API. + * Copyright(c) 2009 Intel Corporation + * + * based on raid6recov.c: + * Copyright 2002 H. Peter Anvin + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 51 + * Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + * + */ +#include <linux/kernel.h> +#include <linux/interrupt.h> +#include <linux/dma-mapping.h> +#include <linux/raid/pq.h> +#include <linux/async_tx.h> + +static struct dma_async_tx_descriptor * +async_sum_product(struct page *dest, struct page **srcs, unsigned char *coef, + size_t len, struct async_submit_ctl *submit) +{ + struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ, + &dest, 1, srcs, 2, len); + struct dma_device *dma = chan ? chan->device : NULL; + const u8 *amul, *bmul; + u8 ax, bx; + u8 *a, *b, *c; + + if (dma) { + dma_addr_t dma_dest[2]; + dma_addr_t dma_src[2]; + struct device *dev = dma->dev; + struct dma_async_tx_descriptor *tx; + enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P; + + if (submit->flags & ASYNC_TX_FENCE) + dma_flags |= DMA_PREP_FENCE; + dma_dest[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL); + dma_src[0] = dma_map_page(dev, srcs[0], 0, len, DMA_TO_DEVICE); + dma_src[1] = dma_map_page(dev, srcs[1], 0, len, DMA_TO_DEVICE); + tx = dma->device_prep_dma_pq(chan, dma_dest, dma_src, 2, coef, + len, dma_flags); + if (tx) { + async_tx_submit(chan, tx, submit); + return tx; + } + + /* could not get a descriptor, unmap and fall through to + * the synchronous path + */ + dma_unmap_page(dev, dma_dest[1], len, DMA_BIDIRECTIONAL); + dma_unmap_page(dev, dma_src[0], len, DMA_TO_DEVICE); + dma_unmap_page(dev, dma_src[1], len, DMA_TO_DEVICE); + } + + /* run the operation synchronously */ + async_tx_quiesce(&submit->depend_tx); + amul = raid6_gfmul[coef[0]]; + bmul = raid6_gfmul[coef[1]]; + a = page_address(srcs[0]); + b = page_address(srcs[1]); + c = page_address(dest); + + while (len--) { + ax = amul[*a++]; + bx = bmul[*b++]; + *c++ = ax ^ bx; + } + + return NULL; +} + +static struct dma_async_tx_descriptor * +async_mult(struct page *dest, struct page *src, u8 coef, size_t len, + struct async_submit_ctl *submit) +{ + struct dma_chan *chan = async_tx_find_channel(submit, DMA_PQ, + &dest, 1, &src, 1, len); + struct dma_device *dma = chan ? chan->device : NULL; + const u8 *qmul; /* Q multiplier table */ + u8 *d, *s; + + if (dma) { + dma_addr_t dma_dest[2]; + dma_addr_t dma_src[1]; + struct device *dev = dma->dev; + struct dma_async_tx_descriptor *tx; + enum dma_ctrl_flags dma_flags = DMA_PREP_PQ_DISABLE_P; + + if (submit->flags & ASYNC_TX_FENCE) + dma_flags |= DMA_PREP_FENCE; + dma_dest[1] = dma_map_page(dev, dest, 0, len, DMA_BIDIRECTIONAL); + dma_src[0] = dma_map_page(dev, src, 0, len, DMA_TO_DEVICE); + tx = dma->device_prep_dma_pq(chan, dma_dest, dma_src, 1, &coef, + len, dma_flags); + if (tx) { + async_tx_submit(chan, tx, submit); + return tx; + } + + /* could not get a descriptor, unmap and fall through to + * the synchronous path + */ + dma_unmap_page(dev, dma_dest[1], len, DMA_BIDIRECTIONAL); + dma_unmap_page(dev, dma_src[0], len, DMA_TO_DEVICE); + } + + /* no channel available, or failed to allocate a descriptor, so + * perform the operation synchronously + */ + async_tx_quiesce(&submit->depend_tx); + qmul = raid6_gfmul[coef]; + d = page_address(dest); + s = page_address(src); + + while (len--) + *d++ = qmul[*s++]; + + return NULL; +} + +static struct dma_async_tx_descriptor * +__2data_recov_4(int disks, size_t bytes, int faila, int failb, + struct page **blocks, struct async_submit_ctl *submit) +{ + struct dma_async_tx_descriptor *tx = NULL; + struct page *p, *q, *a, *b; + struct page *srcs[2]; + unsigned char coef[2]; + enum async_tx_flags flags = submit->flags; + dma_async_tx_callback cb_fn = submit->cb_fn; + void *cb_param = submit->cb_param; + void *scribble = submit->scribble; + + p = blocks[disks-2]; + q = blocks[disks-1]; + + a = blocks[faila]; + b = blocks[failb]; + + /* in the 4 disk case P + Pxy == P and Q + Qxy == Q */ + /* Dx = A*(P+Pxy) + B*(Q+Qxy) */ + srcs[0] = p; + srcs[1] = q; + coef[0] = raid6_gfexi[failb-faila]; + coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]]; + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); + tx = async_sum_product(b, srcs, coef, bytes, submit); + + /* Dy = P+Pxy+Dx */ + srcs[0] = p; + srcs[1] = b; + init_async_submit(submit, flags | ASYNC_TX_XOR_ZERO_DST, tx, cb_fn, + cb_param, scribble); + tx = async_xor(a, srcs, 0, 2, bytes, submit); + + return tx; + +} + +static struct dma_async_tx_descriptor * +__2data_recov_5(int disks, size_t bytes, int faila, int failb, + struct page **blocks, struct async_submit_ctl *submit) +{ + struct dma_async_tx_descriptor *tx = NULL; + struct page *p, *q, *g, *dp, *dq; + struct page *srcs[2]; + unsigned char coef[2]; + enum async_tx_flags flags = submit->flags; + dma_async_tx_callback cb_fn = submit->cb_fn; + void *cb_param = submit->cb_param; + void *scribble = submit->scribble; + int good_srcs, good, i; + + good_srcs = 0; + good = -1; + for (i = 0; i < disks-2; i++) { + if (blocks[i] == NULL) + continue; + if (i == faila || i == failb) + continue; + good = i; + good_srcs++; + } + BUG_ON(good_srcs > 1); + + p = blocks[disks-2]; + q = blocks[disks-1]; + g = blocks[good]; + + /* Compute syndrome with zero for the missing data pages + * Use the dead data pages as temporary storage for delta p and + * delta q + */ + dp = blocks[faila]; + dq = blocks[failb]; + + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); + tx = async_memcpy(dp, g, 0, 0, bytes, submit); + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); + tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit); + + /* compute P + Pxy */ + srcs[0] = dp; + srcs[1] = p; + init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, + NULL, NULL, scribble); + tx = async_xor(dp, srcs, 0, 2, bytes, submit); + + /* compute Q + Qxy */ + srcs[0] = dq; + srcs[1] = q; + init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, + NULL, NULL, scribble); + tx = async_xor(dq, srcs, 0, 2, bytes, submit); + + /* Dx = A*(P+Pxy) + B*(Q+Qxy) */ + srcs[0] = dp; + srcs[1] = dq; + coef[0] = raid6_gfexi[failb-faila]; + coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]]; + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); + tx = async_sum_product(dq, srcs, coef, bytes, submit); + + /* Dy = P+Pxy+Dx */ + srcs[0] = dp; + srcs[1] = dq; + init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn, + cb_param, scribble); + tx = async_xor(dp, srcs, 0, 2, bytes, submit); + + return tx; +} + +static struct dma_async_tx_descriptor * +__2data_recov_n(int disks, size_t bytes, int faila, int failb, + struct page **blocks, struct async_submit_ctl *submit) +{ + struct dma_async_tx_descriptor *tx = NULL; + struct page *p, *q, *dp, *dq; + struct page *srcs[2]; + unsigned char coef[2]; + enum async_tx_flags flags = submit->flags; + dma_async_tx_callback cb_fn = submit->cb_fn; + void *cb_param = submit->cb_param; + void *scribble = submit->scribble; + + p = blocks[disks-2]; + q = blocks[disks-1]; + + /* Compute syndrome with zero for the missing data pages + * Use the dead data pages as temporary storage for + * delta p and delta q + */ + dp = blocks[faila]; + blocks[faila] = NULL; + blocks[disks-2] = dp; + dq = blocks[failb]; + blocks[failb] = NULL; + blocks[disks-1] = dq; + + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); + tx = async_gen_syndrome(blocks, 0, disks, bytes, submit); + + /* Restore pointer table */ + blocks[faila] = dp; + blocks[failb] = dq; + blocks[disks-2] = p; + blocks[disks-1] = q; + + /* compute P + Pxy */ + srcs[0] = dp; + srcs[1] = p; + init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, + NULL, NULL, scribble); + tx = async_xor(dp, srcs, 0, 2, bytes, submit); + + /* compute Q + Qxy */ + srcs[0] = dq; + srcs[1] = q; + init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, + NULL, NULL, scribble); + tx = async_xor(dq, srcs, 0, 2, bytes, submit); + + /* Dx = A*(P+Pxy) + B*(Q+Qxy) */ + srcs[0] = dp; + srcs[1] = dq; + coef[0] = raid6_gfexi[failb-faila]; + coef[1] = raid6_gfinv[raid6_gfexp[faila]^raid6_gfexp[failb]]; + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); + tx = async_sum_product(dq, srcs, coef, bytes, submit); + + /* Dy = P+Pxy+Dx */ + srcs[0] = dp; + srcs[1] = dq; + init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn, + cb_param, scribble); + tx = async_xor(dp, srcs, 0, 2, bytes, submit); + + return tx; +} + +/** + * async_raid6_2data_recov - asynchronously calculate two missing data blocks + * @disks: number of disks in the RAID-6 array + * @bytes: block size + * @faila: first failed drive index + * @failb: second failed drive index + * @blocks: array of source pointers where the last two entries are p and q + * @submit: submission/completion modifiers + */ +struct dma_async_tx_descriptor * +async_raid6_2data_recov(int disks, size_t bytes, int faila, int failb, + struct page **blocks, struct async_submit_ctl *submit) +{ + void *scribble = submit->scribble; + int non_zero_srcs, i; + + BUG_ON(faila == failb); + if (failb < faila) + swap(faila, failb); + + pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes); + + /* if a dma resource is not available or a scribble buffer is not + * available punt to the synchronous path. In the 'dma not + * available' case be sure to use the scribble buffer to + * preserve the content of 'blocks' as the caller intended. + */ + if (!async_dma_find_channel(DMA_PQ) || !scribble) { + void **ptrs = scribble ? scribble : (void **) blocks; + + async_tx_quiesce(&submit->depend_tx); + for (i = 0; i < disks; i++) + if (blocks[i] == NULL) + ptrs[i] = (void *) raid6_empty_zero_page; + else + ptrs[i] = page_address(blocks[i]); + + raid6_2data_recov(disks, bytes, faila, failb, ptrs); + + async_tx_sync_epilog(submit); + + return NULL; + } + + non_zero_srcs = 0; + for (i = 0; i < disks-2 && non_zero_srcs < 4; i++) + if (blocks[i]) + non_zero_srcs++; + switch (non_zero_srcs) { + case 0: + case 1: + /* There must be at least 2 sources - the failed devices. */ + BUG(); + + case 2: + /* dma devices do not uniformly understand a zero source pq + * operation (in contrast to the synchronous case), so + * explicitly handle the special case of a 4 disk array with + * both data disks missing. + */ + return __2data_recov_4(disks, bytes, faila, failb, blocks, submit); + case 3: + /* dma devices do not uniformly understand a single + * source pq operation (in contrast to the synchronous + * case), so explicitly handle the special case of a 5 disk + * array with 2 of 3 data disks missing. + */ + return __2data_recov_5(disks, bytes, faila, failb, blocks, submit); + default: + return __2data_recov_n(disks, bytes, faila, failb, blocks, submit); + } +} +EXPORT_SYMBOL_GPL(async_raid6_2data_recov); + +/** + * async_raid6_datap_recov - asynchronously calculate a data and the 'p' block + * @disks: number of disks in the RAID-6 array + * @bytes: block size + * @faila: failed drive index + * @blocks: array of source pointers where the last two entries are p and q + * @submit: submission/completion modifiers + */ +struct dma_async_tx_descriptor * +async_raid6_datap_recov(int disks, size_t bytes, int faila, + struct page **blocks, struct async_submit_ctl *submit) +{ + struct dma_async_tx_descriptor *tx = NULL; + struct page *p, *q, *dq; + u8 coef; + enum async_tx_flags flags = submit->flags; + dma_async_tx_callback cb_fn = submit->cb_fn; + void *cb_param = submit->cb_param; + void *scribble = submit->scribble; + int good_srcs, good, i; + struct page *srcs[2]; + + pr_debug("%s: disks: %d len: %zu\n", __func__, disks, bytes); + + /* if a dma resource is not available or a scribble buffer is not + * available punt to the synchronous path. In the 'dma not + * available' case be sure to use the scribble buffer to + * preserve the content of 'blocks' as the caller intended. + */ + if (!async_dma_find_channel(DMA_PQ) || !scribble) { + void **ptrs = scribble ? scribble : (void **) blocks; + + async_tx_quiesce(&submit->depend_tx); + for (i = 0; i < disks; i++) + if (blocks[i] == NULL) + ptrs[i] = (void*)raid6_empty_zero_page; + else + ptrs[i] = page_address(blocks[i]); + + raid6_datap_recov(disks, bytes, faila, ptrs); + + async_tx_sync_epilog(submit); + + return NULL; + } + + good_srcs = 0; + good = -1; + for (i = 0; i < disks-2; i++) { + if (i == faila) + continue; + if (blocks[i]) { + good = i; + good_srcs++; + if (good_srcs > 1) + break; + } + } + BUG_ON(good_srcs == 0); + + p = blocks[disks-2]; + q = blocks[disks-1]; + + /* Compute syndrome with zero for the missing data page + * Use the dead data page as temporary storage for delta q + */ + dq = blocks[faila]; + blocks[faila] = NULL; + blocks[disks-1] = dq; + + /* in the 4-disk case we only need to perform a single source + * multiplication with the one good data block. + */ + if (good_srcs == 1) { + struct page *g = blocks[good]; + + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, + scribble); + tx = async_memcpy(p, g, 0, 0, bytes, submit); + + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, + scribble); + tx = async_mult(dq, g, raid6_gfexp[good], bytes, submit); + } else { + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, + scribble); + tx = async_gen_syndrome(blocks, 0, disks, bytes, submit); + } + + /* Restore pointer table */ + blocks[faila] = dq; + blocks[disks-1] = q; + + /* calculate g^{-faila} */ + coef = raid6_gfinv[raid6_gfexp[faila]]; + + srcs[0] = dq; + srcs[1] = q; + init_async_submit(submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx, + NULL, NULL, scribble); + tx = async_xor(dq, srcs, 0, 2, bytes, submit); + + init_async_submit(submit, ASYNC_TX_FENCE, tx, NULL, NULL, scribble); + tx = async_mult(dq, dq, coef, bytes, submit); + + srcs[0] = p; + srcs[1] = dq; + init_async_submit(submit, flags | ASYNC_TX_XOR_DROP_DST, tx, cb_fn, + cb_param, scribble); + tx = async_xor(p, srcs, 0, 2, bytes, submit); + + return tx; +} +EXPORT_SYMBOL_GPL(async_raid6_datap_recov); + +MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>"); +MODULE_DESCRIPTION("asynchronous RAID-6 recovery api"); +MODULE_LICENSE("GPL"); diff --git a/crypto/async_tx/async_tx.c b/crypto/async_tx/async_tx.c index 06eb6cc09fef..7f2c00a45205 100644 --- a/crypto/async_tx/async_tx.c +++ b/crypto/async_tx/async_tx.c @@ -42,16 +42,21 @@ static void __exit async_tx_exit(void) async_dmaengine_put(); } +module_init(async_tx_init); +module_exit(async_tx_exit); + /** * __async_tx_find_channel - find a channel to carry out the operation or let * the transaction execute synchronously - * @depend_tx: transaction dependency + * @submit: transaction dependency and submission modifiers * @tx_type: transaction type */ struct dma_chan * -__async_tx_find_channel(struct dma_async_tx_descriptor *depend_tx, - enum dma_transaction_type tx_type) +__async_tx_find_channel(struct async_submit_ctl *submit, + enum dma_transaction_type tx_type) { + struct dma_async_tx_descriptor *depend_tx = submit->depend_tx; + /* see if we can keep the chain on one channel */ if (depend_tx && dma_has_cap(tx_type, depend_tx->chan->device->cap_mask)) @@ -59,17 +64,6 @@ __async_tx_find_channel(struct dma_async_tx_descriptor *depend_tx, return async_dma_find_channel(tx_type); } EXPORT_SYMBOL_GPL(__async_tx_find_channel); -#else -static int __init async_tx_init(void) -{ - printk(KERN_INFO "async_tx: api initialized (sync-only)\n"); - return 0; -} - -static void __exit async_tx_exit(void) -{ - do { } while (0); -} #endif @@ -83,24 +77,23 @@ static void async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx, struct dma_async_tx_descriptor *tx) { - struct dma_chan *chan; - struct dma_device *device; + struct dma_chan *chan = depend_tx->chan; + struct dma_device *device = chan->device; struct dma_async_tx_descriptor *intr_tx = (void *) ~0; /* first check to see if we can still append to depend_tx */ - spin_lock_bh(&depend_tx->lock); - if (depend_tx->parent && depend_tx->chan == tx->chan) { - tx->parent = depend_tx; - depend_tx->next = tx; + txd_lock(depend_tx); + if (txd_parent(depend_tx) && depend_tx->chan == tx->chan) { + txd_chain(depend_tx, tx); intr_tx = NULL; } - spin_unlock_bh(&depend_tx->lock); + txd_unlock(depend_tx); - if (!intr_tx) + /* attached dependency, flush the parent channel */ + if (!intr_tx) { + device->device_issue_pending(chan); return; - - chan = depend_tx->chan; - device = chan->device; + } /* see if we can schedule an interrupt * otherwise poll for completion @@ -113,27 +106,26 @@ async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx, if (intr_tx) { intr_tx->callback = NULL; intr_tx->callback_param = NULL; - tx->parent = intr_tx; - /* safe to set ->next outside the lock since we know we are + /* safe to chain outside the lock since we know we are * not submitted yet */ - intr_tx->next = tx; + txd_chain(intr_tx, tx); /* check if we need to append */ - spin_lock_bh(&depend_tx->lock); - if (depend_tx->parent) { - intr_tx->parent = depend_tx; - depend_tx->next = intr_tx; + txd_lock(depend_tx); + if (txd_parent(depend_tx)) { + txd_chain(depend_tx, intr_tx); async_tx_ack(intr_tx); intr_tx = NULL; } - spin_unlock_bh(&depend_tx->lock); + txd_unlock(depend_tx); if (intr_tx) { - intr_tx->parent = NULL; + txd_clear_parent(intr_tx); intr_tx->tx_submit(intr_tx); async_tx_ack(intr_tx); } + device->device_issue_pending(chan); } else { if (dma_wait_for_async_tx(depend_tx) == DMA_ERROR) panic("%s: DMA_ERROR waiting for depend_tx\n", @@ -144,13 +136,14 @@ async_tx_channel_switch(struct dma_async_tx_descriptor *depend_tx, /** - * submit_disposition - while holding depend_tx->lock we must avoid submitting - * new operations to prevent a circular locking dependency with - * drivers that already hold a channel lock when calling - * async_tx_run_dependencies. + * submit_disposition - flags for routing an incoming operation * @ASYNC_TX_SUBMITTED: we were able to append the new operation under the lock * @ASYNC_TX_CHANNEL_SWITCH: when the lock is dropped schedule a channel switch * @ASYNC_TX_DIRECT_SUBMIT: when the lock is dropped submit directly + * + * while holding depend_tx->lock we must avoid submitting new operations + * to prevent a circular locking dependency with drivers that already + * hold a channel lock when calling async_tx_run_dependencies. */ enum submit_disposition { ASYNC_TX_SUBMITTED, @@ -160,11 +153,12 @@ enum submit_disposition { void async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx, - enum async_tx_flags flags, struct dma_async_tx_descriptor *depend_tx, - dma_async_tx_callback cb_fn, void *cb_param) + struct async_submit_ctl *submit) { - tx->callback = cb_fn; - tx->callback_param = cb_param; + struct dma_async_tx_descriptor *depend_tx = submit->depend_tx; + + tx->callback = submit->cb_fn; + tx->callback_param = submit->cb_param; if (depend_tx) { enum submit_disposition s; @@ -175,21 +169,20 @@ async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx, * 2/ dependencies are 1:1 i.e. two transactions can * not depend on the same parent */ - BUG_ON(async_tx_test_ack(depend_tx) || depend_tx->next || - tx->parent); + BUG_ON(async_tx_test_ack(depend_tx) || txd_next(depend_tx) || + txd_parent(tx)); /* the lock prevents async_tx_run_dependencies from missing * the setting of ->next when ->parent != NULL */ - spin_lock_bh(&depend_tx->lock); - if (depend_tx->parent) { + txd_lock(depend_tx); + if (txd_parent(depend_tx)) { /* we have a parent so we can not submit directly * if we are staying on the same channel: append * else: channel switch */ if (depend_tx->chan == chan) { - tx->parent = depend_tx; - depend_tx->next = tx; + txd_chain(depend_tx, tx); s = ASYNC_TX_SUBMITTED; } else s = ASYNC_TX_CHANNEL_SWITCH; @@ -202,7 +195,7 @@ async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx, else s = ASYNC_TX_CHANNEL_SWITCH; } - spin_unlock_bh(&depend_tx->lock); + txd_unlock(depend_tx); switch (s) { case ASYNC_TX_SUBMITTED: @@ -211,39 +204,38 @@ async_tx_submit(struct dma_chan *chan, struct dma_async_tx_descriptor *tx, async_tx_channel_switch(depend_tx, tx); break; case ASYNC_TX_DIRECT_SUBMIT: - tx->parent = NULL; + txd_clear_parent(tx); tx->tx_submit(tx); break; } } else { - tx->parent = NULL; + txd_clear_parent(tx); tx->tx_submit(tx); } - if (flags & ASYNC_TX_ACK) + if (submit->flags & ASYNC_TX_ACK) async_tx_ack(tx); - if (depend_tx && (flags & ASYNC_TX_DEP_ACK)) + if (depend_tx) async_tx_ack(depend_tx); } EXPORT_SYMBOL_GPL(async_tx_submit); /** - * async_trigger_callback - schedules the callback function to be run after - * any dependent operations have been completed. - * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK - * @depend_tx: 'callback' requires the completion of this transaction - * @cb_fn: function to call after depend_tx completes - * @cb_param: parameter to pass to the callback routine + * async_trigger_callback - schedules the callback function to be run + * @submit: submission and completion parameters + * + * honored flags: ASYNC_TX_ACK + * + * The callback is run after any dependent operations have completed. */ struct dma_async_tx_descriptor * -async_trigger_callback(enum async_tx_flags flags, - struct dma_async_tx_descriptor *depend_tx, - dma_async_tx_callback cb_fn, void *cb_param) +async_trigger_callback(struct async_submit_ctl *submit) { struct dma_chan *chan; struct dma_device *device; struct dma_async_tx_descriptor *tx; + struct dma_async_tx_descriptor *depend_tx = submit->depend_tx; if (depend_tx) { chan = depend_tx->chan; @@ -262,14 +254,14 @@ async_trigger_callback(enum async_tx_flags flags, if (tx) { pr_debug("%s: (async)\n", __func__); - async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param); + async_tx_submit(chan, tx, submit); } else { pr_debug("%s: (sync)\n", __func__); /* wait for any prerequisite operations */ - async_tx_quiesce(&depend_tx); + async_tx_quiesce(&submit->depend_tx); - async_tx_sync_epilog(cb_fn, cb_param); + async_tx_sync_epilog(submit); } return tx; @@ -295,9 +287,6 @@ void async_tx_quiesce(struct dma_async_tx_descriptor **tx) } EXPORT_SYMBOL_GPL(async_tx_quiesce); -module_init(async_tx_init); -module_exit(async_tx_exit); - MODULE_AUTHOR("Intel Corporation"); MODULE_DESCRIPTION("Asynchronous Bulk Memory Transactions API"); MODULE_LICENSE("GPL"); diff --git a/crypto/async_tx/async_xor.c b/crypto/async_tx/async_xor.c index 95fe2c8d6c51..079ae8ca590b 100644 --- a/crypto/async_tx/async_xor.c +++ b/crypto/async_tx/async_xor.c @@ -33,55 +33,57 @@ /* do_async_xor - dma map the pages and perform the xor with an engine */ static __async_inline struct dma_async_tx_descriptor * do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list, - unsigned int offset, int src_cnt, size_t len, - enum async_tx_flags flags, - struct dma_async_tx_descriptor *depend_tx, - dma_async_tx_callback cb_fn, void *cb_param) + unsigned int offset, int src_cnt, size_t len, dma_addr_t *dma_src, + struct async_submit_ctl *submit) { struct dma_device *dma = chan->device; - dma_addr_t *dma_src = (dma_addr_t *) src_list; struct dma_async_tx_descriptor *tx = NULL; int src_off = 0; int i; - dma_async_tx_callback _cb_fn; - void *_cb_param; - enum async_tx_flags async_flags; + dma_async_tx_callback cb_fn_orig = submit->cb_fn; + void *cb_param_orig = submit->cb_param; + enum async_tx_flags flags_orig = submit->flags; enum dma_ctrl_flags dma_flags; - int xor_src_cnt; + int xor_src_cnt = 0; dma_addr_t dma_dest; /* map the dest bidrectional in case it is re-used as a source */ dma_dest = dma_map_page(dma->dev, dest, offset, len, DMA_BIDIRECTIONAL); for (i = 0; i < src_cnt; i++) { /* only map the dest once */ + if (!src_list[i]) + continue; if (unlikely(src_list[i] == dest)) { - dma_src[i] = dma_dest; + dma_src[xor_src_cnt++] = dma_dest; continue; } - dma_src[i] = dma_map_page(dma->dev, src_list[i], offset, - len, DMA_TO_DEVICE); + dma_src[xor_src_cnt++] = dma_map_page(dma->dev, src_list[i], offset, + len, DMA_TO_DEVICE); } + src_cnt = xor_src_cnt; while (src_cnt) { - async_flags = flags; + submit->flags = flags_orig; dma_flags = 0; - xor_src_cnt = min(src_cnt, dma->max_xor); + xor_src_cnt = min(src_cnt, (int)dma->max_xor); /* if we are submitting additional xors, leave the chain open, * clear the callback parameters, and leave the destination * buffer mapped */ if (src_cnt > xor_src_cnt) { - async_flags &= ~ASYNC_TX_ACK; + submit->flags &= ~ASYNC_TX_ACK; + submit->flags |= ASYNC_TX_FENCE; dma_flags = DMA_COMPL_SKIP_DEST_UNMAP; - _cb_fn = NULL; - _cb_param = NULL; + submit->cb_fn = NULL; + submit->cb_param = NULL; } else { - _cb_fn = cb_fn; - _cb_param = cb_param; + submit->cb_fn = cb_fn_orig; + submit->cb_param = cb_param_orig; } - if (_cb_fn) + if (submit->cb_fn) dma_flags |= DMA_PREP_INTERRUPT; - + if (submit->flags & ASYNC_TX_FENCE) + dma_flags |= DMA_PREP_FENCE; /* Since we have clobbered the src_list we are committed * to doing this asynchronously. Drivers force forward progress * in case they can not provide a descriptor @@ -90,7 +92,7 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list, xor_src_cnt, len, dma_flags); if (unlikely(!tx)) - async_tx_quiesce(&depend_tx); + async_tx_quiesce(&submit->depend_tx); /* spin wait for the preceeding transactions to complete */ while (unlikely(!tx)) { @@ -101,11 +103,8 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list, dma_flags); } - async_tx_submit(chan, tx, async_flags, depend_tx, _cb_fn, - _cb_param); - - depend_tx = tx; - flags |= ASYNC_TX_DEP_ACK; + async_tx_submit(chan, tx, submit); + submit->depend_tx = tx; if (src_cnt > xor_src_cnt) { /* drop completed sources */ @@ -124,23 +123,28 @@ do_async_xor(struct dma_chan *chan, struct page *dest, struct page **src_list, static void do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset, - int src_cnt, size_t len, enum async_tx_flags flags, - dma_async_tx_callback cb_fn, void *cb_param) + int src_cnt, size_t len, struct async_submit_ctl *submit) { int i; - int xor_src_cnt; + int xor_src_cnt = 0; int src_off = 0; void *dest_buf; - void **srcs = (void **) src_list; + void **srcs; - /* reuse the 'src_list' array to convert to buffer pointers */ - for (i = 0; i < src_cnt; i++) - srcs[i] = page_address(src_list[i]) + offset; + if (submit->scribble) + srcs = submit->scribble; + else + srcs = (void **) src_list; + /* convert to buffer pointers */ + for (i = 0; i < src_cnt; i++) + if (src_list[i]) + srcs[xor_src_cnt++] = page_address(src_list[i]) + offset; + src_cnt = xor_src_cnt; /* set destination address */ dest_buf = page_address(dest) + offset; - if (flags & ASYNC_TX_XOR_ZERO_DST) + if (submit->flags & ASYNC_TX_XOR_ZERO_DST) memset(dest_buf, 0, len); while (src_cnt > 0) { @@ -153,61 +157,70 @@ do_sync_xor(struct page *dest, struct page **src_list, unsigned int offset, src_off += xor_src_cnt; } - async_tx_sync_epilog(cb_fn, cb_param); + async_tx_sync_epilog(submit); } /** * async_xor - attempt to xor a set of blocks with a dma engine. - * xor_blocks always uses the dest as a source so the ASYNC_TX_XOR_ZERO_DST - * flag must be set to not include dest data in the calculation. The - * assumption with dma eninges is that they only use the destination - * buffer as a source when it is explicity specified in the source list. * @dest: destination page - * @src_list: array of source pages (if the dest is also a source it must be - * at index zero). The contents of this array may be overwritten. - * @offset: offset in pages to start transaction + * @src_list: array of source pages + * @offset: common src/dst offset to start transaction * @src_cnt: number of source pages * @len: length in bytes - * @flags: ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DEST, - * ASYNC_TX_ACK, ASYNC_TX_DEP_ACK - * @depend_tx: xor depends on the result of this transaction. - * @cb_fn: function to call when the xor completes - * @cb_param: parameter to pass to the callback routine + * @submit: submission / completion modifiers + * + * honored flags: ASYNC_TX_ACK, ASYNC_TX_XOR_ZERO_DST, ASYNC_TX_XOR_DROP_DST + * + * xor_blocks always uses the dest as a source so the + * ASYNC_TX_XOR_ZERO_DST flag must be set to not include dest data in + * the calculation. The assumption with dma eninges is that they only + * use the destination buffer as a source when it is explicity specified + * in the source list. + * + * src_list note: if the dest is also a source it must be at index zero. + * The contents of this array will be overwritten if a scribble region + * is not specified. */ struct dma_async_tx_descriptor * async_xor(struct page *dest, struct page **src_list, unsigned int offset, - int src_cnt, size_t len, enum async_tx_flags flags, - struct dma_async_tx_descriptor *depend_tx, - dma_async_tx_callback cb_fn, void *cb_param) + int src_cnt, size_t len, struct async_submit_ctl *submit) { - struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_XOR, + struct dma_chan *chan = async_tx_find_channel(submit, DMA_XOR, &dest, 1, src_list, src_cnt, len); + dma_addr_t *dma_src = NULL; + BUG_ON(src_cnt <= 1); - if (chan) { + if (submit->scribble) + dma_src = submit->scribble; + else if (sizeof(dma_addr_t) <= sizeof(struct page *)) + dma_src = (dma_addr_t *) src_list; + + if (dma_src && chan && is_dma_xor_aligned(chan->device, offset, 0, len)) { /* run the xor asynchronously */ pr_debug("%s (async): len: %zu\n", __func__, len); return do_async_xor(chan, dest, src_list, offset, src_cnt, len, - flags, depend_tx, cb_fn, cb_param); + dma_src, submit); } else { /* run the xor synchronously */ pr_debug("%s (sync): len: %zu\n", __func__, len); + WARN_ONCE(chan, "%s: no space for dma address conversion\n", + __func__); /* in the sync case the dest is an implied source * (assumes the dest is the first source) */ - if (flags & ASYNC_TX_XOR_DROP_DST) { + if (submit->flags & ASYNC_TX_XOR_DROP_DST) { src_cnt--; src_list++; } /* wait for any prerequisite operations */ - async_tx_quiesce(&depend_tx); + async_tx_quiesce(&submit->depend_tx); - do_sync_xor(dest, src_list, offset, src_cnt, len, - flags, cb_fn, cb_param); + do_sync_xor(dest, src_list, offset, src_cnt, len, submit); return NULL; } @@ -221,105 +234,104 @@ static int page_is_zero(struct page *p, unsigned int offset, size_t len) memcmp(a, a + 4, len - 4) == 0); } +static inline struct dma_chan * +xor_val_chan(struct async_submit_ctl *submit, struct page *dest, + struct page **src_list, int src_cnt, size_t len) +{ + #ifdef CONFIG_ASYNC_TX_DISABLE_XOR_VAL_DMA + return NULL; + #endif + return async_tx_find_channel(submit, DMA_XOR_VAL, &dest, 1, src_list, + src_cnt, len); +} + /** - * async_xor_zero_sum - attempt a xor parity check with a dma engine. + * async_xor_val - attempt a xor parity check with a dma engine. * @dest: destination page used if the xor is performed synchronously - * @src_list: array of source pages. The dest page must be listed as a source - * at index zero. The contents of this array may be overwritten. + * @src_list: array of source pages * @offset: offset in pages to start transaction * @src_cnt: number of source pages * @len: length in bytes * @result: 0 if sum == 0 else non-zero - * @flags: ASYNC_TX_ACK, ASYNC_TX_DEP_ACK - * @depend_tx: xor depends on the result of this transaction. - * @cb_fn: function to call when the xor completes - * @cb_param: parameter to pass to the callback routine + * @submit: submission / completion modifiers + * + * honored flags: ASYNC_TX_ACK + * + * src_list note: if the dest is also a source it must be at index zero. + * The contents of this array will be overwritten if a scribble region + * is not specified. */ struct dma_async_tx_descriptor * -async_xor_zero_sum(struct page *dest, struct page **src_list, - unsigned int offset, int src_cnt, size_t len, - u32 *result, enum async_tx_flags flags, - struct dma_async_tx_descriptor *depend_tx, - dma_async_tx_callback cb_fn, void *cb_param) +async_xor_val(struct page *dest, struct page **src_list, unsigned int offset, + int src_cnt, size_t len, enum sum_check_flags *result, + struct async_submit_ctl *submit) { - struct dma_chan *chan = async_tx_find_channel(depend_tx, DMA_ZERO_SUM, - &dest, 1, src_list, - src_cnt, len); + struct dma_chan *chan = xor_val_chan(submit, dest, src_list, src_cnt, len); struct dma_device *device = chan ? chan->device : NULL; struct dma_async_tx_descriptor *tx = NULL; + dma_addr_t *dma_src = NULL; BUG_ON(src_cnt <= 1); - if (device && src_cnt <= device->max_xor) { - dma_addr_t *dma_src = (dma_addr_t *) src_list; - unsigned long dma_prep_flags = cb_fn ? DMA_PREP_INTERRUPT : 0; + if (submit->scribble) + dma_src = submit->scribble; + else if (sizeof(dma_addr_t) <= sizeof(struct page *)) + dma_src = (dma_addr_t *) src_list; + + if (dma_src && device && src_cnt <= device->max_xor && + is_dma_xor_aligned(device, offset, 0, len)) { + unsigned long dma_prep_flags = 0; int i; pr_debug("%s: (async) len: %zu\n", __func__, len); + if (submit->cb_fn) + dma_prep_flags |= DMA_PREP_INTERRUPT; + if (submit->flags & ASYNC_TX_FENCE) + dma_prep_flags |= DMA_PREP_FENCE; for (i = 0; i < src_cnt; i++) dma_src[i] = dma_map_page(device->dev, src_list[i], offset, len, DMA_TO_DEVICE); - tx = device->device_prep_dma_zero_sum(chan, dma_src, src_cnt, - len, result, - dma_prep_flags); + tx = device->device_prep_dma_xor_val(chan, dma_src, src_cnt, + len, result, + dma_prep_flags); if (unlikely(!tx)) { - async_tx_quiesce(&depend_tx); + async_tx_quiesce(&submit->depend_tx); while (!tx) { dma_async_issue_pending(chan); - tx = device->device_prep_dma_zero_sum(chan, + tx = device->device_prep_dma_xor_val(chan, dma_src, src_cnt, len, result, dma_prep_flags); } } - async_tx_submit(chan, tx, flags, depend_tx, cb_fn, cb_param); + async_tx_submit(chan, tx, submit); } else { - unsigned long xor_flags = flags; + enum async_tx_flags flags_orig = submit->flags; pr_debug("%s: (sync) len: %zu\n", __func__, len); + WARN_ONCE(device && src_cnt <= device->max_xor, + "%s: no space for dma address conversion\n", + __func__); - xor_flags |= ASYNC_TX_XOR_DROP_DST; - xor_flags &= ~ASYNC_TX_ACK; + submit->flags |= ASYNC_TX_XOR_DROP_DST; + submit->flags &= ~ASYNC_TX_ACK; - tx = async_xor(dest, src_list, offset, src_cnt, len, xor_flags, - depend_tx, NULL, NULL); + tx = async_xor(dest, src_list, offset, src_cnt, len, submit); async_tx_quiesce(&tx); - *result = page_is_zero(dest, offset, len) ? 0 : 1; + *result = !page_is_zero(dest, offset, len) << SUM_CHECK_P; - async_tx_sync_epilog(cb_fn, cb_param); + async_tx_sync_epilog(submit); + submit->flags = flags_orig; } return tx; } -EXPORT_SYMBOL_GPL(async_xor_zero_sum); - -static int __init async_xor_init(void) -{ - #ifdef CONFIG_DMA_ENGINE - /* To conserve stack space the input src_list (array of page pointers) - * is reused to hold the array of dma addresses passed to the driver. - * This conversion is only possible when dma_addr_t is less than the - * the size of a pointer. HIGHMEM64G is known to violate this - * assumption. - */ - BUILD_BUG_ON(sizeof(dma_addr_t) > sizeof(struct page *)); - #endif - - return 0; -} - -static void __exit async_xor_exit(void) -{ - do { } while (0); -} - -module_init(async_xor_init); -module_exit(async_xor_exit); +EXPORT_SYMBOL_GPL(async_xor_val); MODULE_AUTHOR("Intel Corporation"); MODULE_DESCRIPTION("asynchronous xor/xor-zero-sum api"); diff --git a/crypto/async_tx/raid6test.c b/crypto/async_tx/raid6test.c new file mode 100644 index 000000000000..c1321935ebcc --- /dev/null +++ b/crypto/async_tx/raid6test.c @@ -0,0 +1,248 @@ +/* + * asynchronous raid6 recovery self test + * Copyright (c) 2009, Intel Corporation. + * + * based on drivers/md/raid6test/test.c: + * Copyright 2002-2007 H. Peter Anvin + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + * + */ +#include <linux/async_tx.h> +#include <linux/gfp.h> +#include <linux/random.h> + +#undef pr +#define pr(fmt, args...) pr_info("raid6test: " fmt, ##args) + +#define NDISKS 16 /* Including P and Q */ + +static struct page *dataptrs[NDISKS]; +static addr_conv_t addr_conv[NDISKS]; +static struct page *data[NDISKS+3]; +static struct page *spare; +static struct page *recovi; +static struct page *recovj; + +static void callback(void *param) +{ + struct completion *cmp = param; + + complete(cmp); +} + +static void makedata(int disks) +{ + int i, j; + + for (i = 0; i < disks; i++) { + for (j = 0; j < PAGE_SIZE/sizeof(u32); j += sizeof(u32)) { + u32 *p = page_address(data[i]) + j; + + *p = random32(); + } + + dataptrs[i] = data[i]; + } +} + +static char disk_type(int d, int disks) +{ + if (d == disks - 2) + return 'P'; + else if (d == disks - 1) + return 'Q'; + else + return 'D'; +} + +/* Recover two failed blocks. */ +static void raid6_dual_recov(int disks, size_t bytes, int faila, int failb, struct page **ptrs) +{ + struct async_submit_ctl submit; + struct completion cmp; + struct dma_async_tx_descriptor *tx = NULL; + enum sum_check_flags result = ~0; + + if (faila > failb) + swap(faila, failb); + + if (failb == disks-1) { + if (faila == disks-2) { + /* P+Q failure. Just rebuild the syndrome. */ + init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv); + tx = async_gen_syndrome(ptrs, 0, disks, bytes, &submit); + } else { + struct page *blocks[disks]; + struct page *dest; + int count = 0; + int i; + + /* data+Q failure. Reconstruct data from P, + * then rebuild syndrome + */ + for (i = disks; i-- ; ) { + if (i == faila || i == failb) + continue; + blocks[count++] = ptrs[i]; + } + dest = ptrs[faila]; + init_async_submit(&submit, ASYNC_TX_XOR_ZERO_DST, NULL, + NULL, NULL, addr_conv); + tx = async_xor(dest, blocks, 0, count, bytes, &submit); + + init_async_submit(&submit, 0, tx, NULL, NULL, addr_conv); + tx = async_gen_syndrome(ptrs, 0, disks, bytes, &submit); + } + } else { + if (failb == disks-2) { + /* data+P failure. */ + init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv); + tx = async_raid6_datap_recov(disks, bytes, faila, ptrs, &submit); + } else { + /* data+data failure. */ + init_async_submit(&submit, 0, NULL, NULL, NULL, addr_conv); + tx = async_raid6_2data_recov(disks, bytes, faila, failb, ptrs, &submit); + } + } + init_completion(&cmp); + init_async_submit(&submit, ASYNC_TX_ACK, tx, callback, &cmp, addr_conv); + tx = async_syndrome_val(ptrs, 0, disks, bytes, &result, spare, &submit); + async_tx_issue_pending(tx); + + if (wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)) == 0) + pr("%s: timeout! (faila: %d failb: %d disks: %d)\n", + __func__, faila, failb, disks); + + if (result != 0) + pr("%s: validation failure! faila: %d failb: %d sum_check_flags: %x\n", + __func__, faila, failb, result); +} + +static int test_disks(int i, int j, int disks) +{ + int erra, errb; + + memset(page_address(recovi), 0xf0, PAGE_SIZE); + memset(page_address(recovj), 0xba, PAGE_SIZE); + + dataptrs[i] = recovi; + dataptrs[j] = recovj; + + raid6_dual_recov(disks, PAGE_SIZE, i, j, dataptrs); + + erra = memcmp(page_address(data[i]), page_address(recovi), PAGE_SIZE); + errb = memcmp(page_address(data[j]), page_address(recovj), PAGE_SIZE); + + pr("%s(%d, %d): faila=%3d(%c) failb=%3d(%c) %s\n", + __func__, i, j, i, disk_type(i, disks), j, disk_type(j, disks), + (!erra && !errb) ? "OK" : !erra ? "ERRB" : !errb ? "ERRA" : "ERRAB"); + + dataptrs[i] = data[i]; + dataptrs[j] = data[j]; + + return erra || errb; +} + +static int test(int disks, int *tests) +{ + struct dma_async_tx_descriptor *tx; + struct async_submit_ctl submit; + struct completion cmp; + int err = 0; + int i, j; + + recovi = data[disks]; + recovj = data[disks+1]; + spare = data[disks+2]; + + makedata(disks); + + /* Nuke syndromes */ + memset(page_address(data[disks-2]), 0xee, PAGE_SIZE); + memset(page_address(data[disks-1]), 0xee, PAGE_SIZE); + + /* Generate assumed good syndrome */ + init_completion(&cmp); + init_async_submit(&submit, ASYNC_TX_ACK, NULL, callback, &cmp, addr_conv); + tx = async_gen_syndrome(dataptrs, 0, disks, PAGE_SIZE, &submit); + async_tx_issue_pending(tx); + + if (wait_for_completion_timeout(&cmp, msecs_to_jiffies(3000)) == 0) { + pr("error: initial gen_syndrome(%d) timed out\n", disks); + return 1; + } + + pr("testing the %d-disk case...\n", disks); + for (i = 0; i < disks-1; i++) + for (j = i+1; j < disks; j++) { + (*tests)++; + err += test_disks(i, j, disks); + } + + return err; +} + + +static int raid6_test(void) +{ + int err = 0; + int tests = 0; + int i; + + for (i = 0; i < NDISKS+3; i++) { + data[i] = alloc_page(GFP_KERNEL); + if (!data[i]) { + while (i--) + put_page(data[i]); + return -ENOMEM; + } + } + + /* the 4-disk and 5-disk cases are special for the recovery code */ + if (NDISKS > 4) + err += test(4, &tests); + if (NDISKS > 5) + err += test(5, &tests); + /* the 11 and 12 disk cases are special for ioatdma (p-disabled + * q-continuation without extended descriptor) + */ + if (NDISKS > 12) { + err += test(11, &tests); + err += test(12, &tests); + } + err += test(NDISKS, &tests); + + pr("\n"); + pr("complete (%d tests, %d failure%s)\n", + tests, err, err == 1 ? "" : "s"); + + for (i = 0; i < NDISKS+3; i++) + put_page(data[i]); + + return 0; +} + +static void raid6_test_exit(void) +{ +} + +/* when compiled-in wait for drivers to load first (assumes dma drivers + * are also compliled-in) + */ +late_initcall(raid6_test); +module_exit(raid6_test_exit); +MODULE_AUTHOR("Dan Williams <dan.j.williams@intel.com>"); +MODULE_DESCRIPTION("asynchronous RAID-6 recovery self tests"); +MODULE_LICENSE("GPL"); diff --git a/crypto/authenc.c b/crypto/authenc.c index 5793b64c81a8..a5a22cfcd07b 100644 --- a/crypto/authenc.c +++ b/crypto/authenc.c @@ -23,24 +23,42 @@ #include <linux/slab.h> #include <linux/spinlock.h> +typedef u8 *(*authenc_ahash_t)(struct aead_request *req, unsigned int flags); + struct authenc_instance_ctx { - struct crypto_spawn auth; + struct crypto_ahash_spawn auth; struct crypto_skcipher_spawn enc; }; struct crypto_authenc_ctx { - spinlock_t auth_lock; - struct crypto_hash *auth; + unsigned int reqoff; + struct crypto_ahash *auth; struct crypto_ablkcipher *enc; }; +struct authenc_request_ctx { + unsigned int cryptlen; + struct scatterlist *sg; + struct scatterlist asg[2]; + struct scatterlist cipher[2]; + crypto_completion_t complete; + crypto_completion_t update_complete; + char tail[]; +}; + +static void authenc_request_complete(struct aead_request *req, int err) +{ + if (err != -EINPROGRESS) + aead_request_complete(req, err); +} + static int crypto_authenc_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen) { unsigned int authkeylen; unsigned int enckeylen; struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); - struct crypto_hash *auth = ctx->auth; + struct crypto_ahash *auth = ctx->auth; struct crypto_ablkcipher *enc = ctx->enc; struct rtattr *rta = (void *)key; struct crypto_authenc_key_param *param; @@ -64,11 +82,11 @@ static int crypto_authenc_setkey(struct crypto_aead *authenc, const u8 *key, authkeylen = keylen - enckeylen; - crypto_hash_clear_flags(auth, CRYPTO_TFM_REQ_MASK); - crypto_hash_set_flags(auth, crypto_aead_get_flags(authenc) & + crypto_ahash_clear_flags(auth, CRYPTO_TFM_REQ_MASK); + crypto_ahash_set_flags(auth, crypto_aead_get_flags(authenc) & CRYPTO_TFM_REQ_MASK); - err = crypto_hash_setkey(auth, key, authkeylen); - crypto_aead_set_flags(authenc, crypto_hash_get_flags(auth) & + err = crypto_ahash_setkey(auth, key, authkeylen); + crypto_aead_set_flags(authenc, crypto_ahash_get_flags(auth) & CRYPTO_TFM_RES_MASK); if (err) @@ -103,40 +121,202 @@ static void authenc_chain(struct scatterlist *head, struct scatterlist *sg, sg_mark_end(head); } -static u8 *crypto_authenc_hash(struct aead_request *req, unsigned int flags, - struct scatterlist *cipher, - unsigned int cryptlen) +static void authenc_geniv_ahash_update_done(struct crypto_async_request *areq, + int err) +{ + struct aead_request *req = areq->data; + struct crypto_aead *authenc = crypto_aead_reqtfm(req); + struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); + struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); + struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); + + if (err) + goto out; + + ahash_request_set_crypt(ahreq, areq_ctx->sg, ahreq->result, + areq_ctx->cryptlen); + ahash_request_set_callback(ahreq, aead_request_flags(req) & + CRYPTO_TFM_REQ_MAY_SLEEP, + areq_ctx->complete, req); + + err = crypto_ahash_finup(ahreq); + if (err) + goto out; + + scatterwalk_map_and_copy(ahreq->result, areq_ctx->sg, + areq_ctx->cryptlen, + crypto_aead_authsize(authenc), 1); + +out: + authenc_request_complete(req, err); +} + +static void authenc_geniv_ahash_done(struct crypto_async_request *areq, int err) +{ + struct aead_request *req = areq->data; + struct crypto_aead *authenc = crypto_aead_reqtfm(req); + struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); + struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); + struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); + + if (err) + goto out; + + scatterwalk_map_and_copy(ahreq->result, areq_ctx->sg, + areq_ctx->cryptlen, + crypto_aead_authsize(authenc), 1); + +out: + aead_request_complete(req, err); +} + +static void authenc_verify_ahash_update_done(struct crypto_async_request *areq, + int err) +{ + u8 *ihash; + unsigned int authsize; + struct ablkcipher_request *abreq; + struct aead_request *req = areq->data; + struct crypto_aead *authenc = crypto_aead_reqtfm(req); + struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); + struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); + struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); + unsigned int cryptlen = req->cryptlen; + + if (err) + goto out; + + ahash_request_set_crypt(ahreq, areq_ctx->sg, ahreq->result, + areq_ctx->cryptlen); + ahash_request_set_callback(ahreq, aead_request_flags(req) & + CRYPTO_TFM_REQ_MAY_SLEEP, + areq_ctx->complete, req); + + err = crypto_ahash_finup(ahreq); + if (err) + goto out; + + authsize = crypto_aead_authsize(authenc); + cryptlen -= authsize; + ihash = ahreq->result + authsize; + scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen, + authsize, 0); + + err = memcmp(ihash, ahreq->result, authsize) ? -EBADMSG : 0; + if (err) + goto out; + + abreq = aead_request_ctx(req); + ablkcipher_request_set_tfm(abreq, ctx->enc); + ablkcipher_request_set_callback(abreq, aead_request_flags(req), + req->base.complete, req->base.data); + ablkcipher_request_set_crypt(abreq, req->src, req->dst, + cryptlen, req->iv); + + err = crypto_ablkcipher_decrypt(abreq); + +out: + authenc_request_complete(req, err); +} + +static void authenc_verify_ahash_done(struct crypto_async_request *areq, + int err) { + u8 *ihash; + unsigned int authsize; + struct ablkcipher_request *abreq; + struct aead_request *req = areq->data; struct crypto_aead *authenc = crypto_aead_reqtfm(req); struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); - struct crypto_hash *auth = ctx->auth; - struct hash_desc desc = { - .tfm = auth, - .flags = aead_request_flags(req) & flags, - }; - u8 *hash = aead_request_ctx(req); + struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); + struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); + unsigned int cryptlen = req->cryptlen; + + if (err) + goto out; + + authsize = crypto_aead_authsize(authenc); + cryptlen -= authsize; + ihash = ahreq->result + authsize; + scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen, + authsize, 0); + + err = memcmp(ihash, ahreq->result, authsize) ? -EBADMSG : 0; + if (err) + goto out; + + abreq = aead_request_ctx(req); + ablkcipher_request_set_tfm(abreq, ctx->enc); + ablkcipher_request_set_callback(abreq, aead_request_flags(req), + req->base.complete, req->base.data); + ablkcipher_request_set_crypt(abreq, req->src, req->dst, + cryptlen, req->iv); + + err = crypto_ablkcipher_decrypt(abreq); + +out: + authenc_request_complete(req, err); +} + +static u8 *crypto_authenc_ahash_fb(struct aead_request *req, unsigned int flags) +{ + struct crypto_aead *authenc = crypto_aead_reqtfm(req); + struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); + struct crypto_ahash *auth = ctx->auth; + struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); + struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); + u8 *hash = areq_ctx->tail; int err; - hash = (u8 *)ALIGN((unsigned long)hash + crypto_hash_alignmask(auth), - crypto_hash_alignmask(auth) + 1); + hash = (u8 *)ALIGN((unsigned long)hash + crypto_ahash_alignmask(auth), + crypto_ahash_alignmask(auth) + 1); + + ahash_request_set_tfm(ahreq, auth); - spin_lock_bh(&ctx->auth_lock); - err = crypto_hash_init(&desc); + err = crypto_ahash_init(ahreq); if (err) - goto auth_unlock; + return ERR_PTR(err); + + ahash_request_set_crypt(ahreq, req->assoc, hash, req->assoclen); + ahash_request_set_callback(ahreq, aead_request_flags(req) & flags, + areq_ctx->update_complete, req); - err = crypto_hash_update(&desc, req->assoc, req->assoclen); + err = crypto_ahash_update(ahreq); if (err) - goto auth_unlock; + return ERR_PTR(err); - err = crypto_hash_update(&desc, cipher, cryptlen); + ahash_request_set_crypt(ahreq, areq_ctx->sg, hash, + areq_ctx->cryptlen); + ahash_request_set_callback(ahreq, aead_request_flags(req) & flags, + areq_ctx->complete, req); + + err = crypto_ahash_finup(ahreq); if (err) - goto auth_unlock; + return ERR_PTR(err); - err = crypto_hash_final(&desc, hash); -auth_unlock: - spin_unlock_bh(&ctx->auth_lock); + return hash; +} +static u8 *crypto_authenc_ahash(struct aead_request *req, unsigned int flags) +{ + struct crypto_aead *authenc = crypto_aead_reqtfm(req); + struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); + struct crypto_ahash *auth = ctx->auth; + struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); + struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); + u8 *hash = areq_ctx->tail; + int err; + + hash = (u8 *)ALIGN((unsigned long)hash + crypto_ahash_alignmask(auth), + crypto_ahash_alignmask(auth) + 1); + + ahash_request_set_tfm(ahreq, auth); + ahash_request_set_crypt(ahreq, areq_ctx->sg, hash, + areq_ctx->cryptlen); + ahash_request_set_callback(ahreq, aead_request_flags(req) & flags, + areq_ctx->complete, req); + + err = crypto_ahash_digest(ahreq); if (err) return ERR_PTR(err); @@ -147,11 +327,15 @@ static int crypto_authenc_genicv(struct aead_request *req, u8 *iv, unsigned int flags) { struct crypto_aead *authenc = crypto_aead_reqtfm(req); + struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); struct scatterlist *dst = req->dst; - struct scatterlist cipher[2]; - struct page *dstp; + struct scatterlist *assoc = req->assoc; + struct scatterlist *cipher = areq_ctx->cipher; + struct scatterlist *asg = areq_ctx->asg; unsigned int ivsize = crypto_aead_ivsize(authenc); - unsigned int cryptlen; + unsigned int cryptlen = req->cryptlen; + authenc_ahash_t authenc_ahash_fn = crypto_authenc_ahash_fb; + struct page *dstp; u8 *vdst; u8 *hash; @@ -163,10 +347,25 @@ static int crypto_authenc_genicv(struct aead_request *req, u8 *iv, sg_set_buf(cipher, iv, ivsize); authenc_chain(cipher, dst, vdst == iv + ivsize); dst = cipher; + cryptlen += ivsize; + } + + if (sg_is_last(assoc)) { + authenc_ahash_fn = crypto_authenc_ahash; + sg_init_table(asg, 2); + sg_set_page(asg, sg_page(assoc), assoc->length, assoc->offset); + authenc_chain(asg, dst, 0); + dst = asg; + cryptlen += req->assoclen; } - cryptlen = req->cryptlen + ivsize; - hash = crypto_authenc_hash(req, flags, dst, cryptlen); + areq_ctx->cryptlen = cryptlen; + areq_ctx->sg = dst; + + areq_ctx->complete = authenc_geniv_ahash_done; + areq_ctx->update_complete = authenc_geniv_ahash_update_done; + + hash = authenc_ahash_fn(req, flags); if (IS_ERR(hash)) return PTR_ERR(hash); @@ -190,18 +389,20 @@ static void crypto_authenc_encrypt_done(struct crypto_async_request *req, err = crypto_authenc_genicv(areq, iv, 0); } - aead_request_complete(areq, err); + authenc_request_complete(areq, err); } static int crypto_authenc_encrypt(struct aead_request *req) { struct crypto_aead *authenc = crypto_aead_reqtfm(req); struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); - struct ablkcipher_request *abreq = aead_request_ctx(req); + struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); struct crypto_ablkcipher *enc = ctx->enc; struct scatterlist *dst = req->dst; unsigned int cryptlen = req->cryptlen; - u8 *iv = (u8 *)(abreq + 1) + crypto_ablkcipher_reqsize(enc); + struct ablkcipher_request *abreq = (void *)(areq_ctx->tail + + ctx->reqoff); + u8 *iv = (u8 *)abreq - crypto_ablkcipher_ivsize(enc); int err; ablkcipher_request_set_tfm(abreq, enc); @@ -229,7 +430,7 @@ static void crypto_authenc_givencrypt_done(struct crypto_async_request *req, err = crypto_authenc_genicv(areq, greq->giv, 0); } - aead_request_complete(areq, err); + authenc_request_complete(areq, err); } static int crypto_authenc_givencrypt(struct aead_givcrypt_request *req) @@ -256,33 +457,40 @@ static int crypto_authenc_givencrypt(struct aead_givcrypt_request *req) } static int crypto_authenc_verify(struct aead_request *req, - struct scatterlist *cipher, - unsigned int cryptlen) + authenc_ahash_t authenc_ahash_fn) { struct crypto_aead *authenc = crypto_aead_reqtfm(req); + struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); u8 *ohash; u8 *ihash; unsigned int authsize; - ohash = crypto_authenc_hash(req, CRYPTO_TFM_REQ_MAY_SLEEP, cipher, - cryptlen); + areq_ctx->complete = authenc_verify_ahash_done; + areq_ctx->update_complete = authenc_verify_ahash_update_done; + + ohash = authenc_ahash_fn(req, CRYPTO_TFM_REQ_MAY_SLEEP); if (IS_ERR(ohash)) return PTR_ERR(ohash); authsize = crypto_aead_authsize(authenc); ihash = ohash + authsize; - scatterwalk_map_and_copy(ihash, cipher, cryptlen, authsize, 0); - return memcmp(ihash, ohash, authsize) ? -EBADMSG: 0; + scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen, + authsize, 0); + return memcmp(ihash, ohash, authsize) ? -EBADMSG : 0; } static int crypto_authenc_iverify(struct aead_request *req, u8 *iv, unsigned int cryptlen) { struct crypto_aead *authenc = crypto_aead_reqtfm(req); + struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); struct scatterlist *src = req->src; - struct scatterlist cipher[2]; - struct page *srcp; + struct scatterlist *assoc = req->assoc; + struct scatterlist *cipher = areq_ctx->cipher; + struct scatterlist *asg = areq_ctx->asg; unsigned int ivsize = crypto_aead_ivsize(authenc); + authenc_ahash_t authenc_ahash_fn = crypto_authenc_ahash_fb; + struct page *srcp; u8 *vsrc; srcp = sg_page(src); @@ -293,9 +501,22 @@ static int crypto_authenc_iverify(struct aead_request *req, u8 *iv, sg_set_buf(cipher, iv, ivsize); authenc_chain(cipher, src, vsrc == iv + ivsize); src = cipher; + cryptlen += ivsize; + } + + if (sg_is_last(assoc)) { + authenc_ahash_fn = crypto_authenc_ahash; + sg_init_table(asg, 2); + sg_set_page(asg, sg_page(assoc), assoc->length, assoc->offset); + authenc_chain(asg, src, 0); + src = asg; + cryptlen += req->assoclen; } - return crypto_authenc_verify(req, src, cryptlen + ivsize); + areq_ctx->cryptlen = cryptlen; + areq_ctx->sg = src; + + return crypto_authenc_verify(req, authenc_ahash_fn); } static int crypto_authenc_decrypt(struct aead_request *req) @@ -326,38 +547,42 @@ static int crypto_authenc_decrypt(struct aead_request *req) static int crypto_authenc_init_tfm(struct crypto_tfm *tfm) { - struct crypto_instance *inst = (void *)tfm->__crt_alg; + struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); struct authenc_instance_ctx *ictx = crypto_instance_ctx(inst); struct crypto_authenc_ctx *ctx = crypto_tfm_ctx(tfm); - struct crypto_hash *auth; + struct crypto_ahash *auth; struct crypto_ablkcipher *enc; int err; - auth = crypto_spawn_hash(&ictx->auth); + auth = crypto_spawn_ahash(&ictx->auth); if (IS_ERR(auth)) return PTR_ERR(auth); enc = crypto_spawn_skcipher(&ictx->enc); err = PTR_ERR(enc); if (IS_ERR(enc)) - goto err_free_hash; + goto err_free_ahash; ctx->auth = auth; ctx->enc = enc; - tfm->crt_aead.reqsize = max_t(unsigned int, - (crypto_hash_alignmask(auth) & - ~(crypto_tfm_ctx_alignment() - 1)) + - crypto_hash_digestsize(auth) * 2, - sizeof(struct skcipher_givcrypt_request) + - crypto_ablkcipher_reqsize(enc) + - crypto_ablkcipher_ivsize(enc)); - spin_lock_init(&ctx->auth_lock); + ctx->reqoff = ALIGN(2 * crypto_ahash_digestsize(auth) + + crypto_ahash_alignmask(auth), + crypto_ahash_alignmask(auth) + 1) + + crypto_ablkcipher_ivsize(enc); + + tfm->crt_aead.reqsize = sizeof(struct authenc_request_ctx) + + ctx->reqoff + + max_t(unsigned int, + crypto_ahash_reqsize(auth) + + sizeof(struct ahash_request), + sizeof(struct skcipher_givcrypt_request) + + crypto_ablkcipher_reqsize(enc)); return 0; -err_free_hash: - crypto_free_hash(auth); +err_free_ahash: + crypto_free_ahash(auth); return err; } @@ -365,7 +590,7 @@ static void crypto_authenc_exit_tfm(struct crypto_tfm *tfm) { struct crypto_authenc_ctx *ctx = crypto_tfm_ctx(tfm); - crypto_free_hash(ctx->auth); + crypto_free_ahash(ctx->auth); crypto_free_ablkcipher(ctx->enc); } @@ -373,7 +598,8 @@ static struct crypto_instance *crypto_authenc_alloc(struct rtattr **tb) { struct crypto_attr_type *algt; struct crypto_instance *inst; - struct crypto_alg *auth; + struct hash_alg_common *auth; + struct crypto_alg *auth_base; struct crypto_alg *enc; struct authenc_instance_ctx *ctx; const char *enc_name; @@ -387,10 +613,12 @@ static struct crypto_instance *crypto_authenc_alloc(struct rtattr **tb) if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) return ERR_PTR(-EINVAL); - auth = crypto_attr_alg(tb[1], CRYPTO_ALG_TYPE_HASH, - CRYPTO_ALG_TYPE_HASH_MASK); + auth = ahash_attr_alg(tb[1], CRYPTO_ALG_TYPE_HASH, + CRYPTO_ALG_TYPE_AHASH_MASK); if (IS_ERR(auth)) - return ERR_PTR(PTR_ERR(auth)); + return ERR_CAST(auth); + + auth_base = &auth->base; enc_name = crypto_attr_alg_name(tb[2]); err = PTR_ERR(enc_name); @@ -404,7 +632,7 @@ static struct crypto_instance *crypto_authenc_alloc(struct rtattr **tb) ctx = crypto_instance_ctx(inst); - err = crypto_init_spawn(&ctx->auth, auth, inst, CRYPTO_ALG_TYPE_MASK); + err = crypto_init_ahash_spawn(&ctx->auth, auth, inst); if (err) goto err_free_inst; @@ -419,28 +647,25 @@ static struct crypto_instance *crypto_authenc_alloc(struct rtattr **tb) err = -ENAMETOOLONG; if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, - "authenc(%s,%s)", auth->cra_name, enc->cra_name) >= + "authenc(%s,%s)", auth_base->cra_name, enc->cra_name) >= CRYPTO_MAX_ALG_NAME) goto err_drop_enc; if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, - "authenc(%s,%s)", auth->cra_driver_name, + "authenc(%s,%s)", auth_base->cra_driver_name, enc->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto err_drop_enc; inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD; inst->alg.cra_flags |= enc->cra_flags & CRYPTO_ALG_ASYNC; - inst->alg.cra_priority = enc->cra_priority * 10 + auth->cra_priority; + inst->alg.cra_priority = enc->cra_priority * + 10 + auth_base->cra_priority; inst->alg.cra_blocksize = enc->cra_blocksize; - inst->alg.cra_alignmask = auth->cra_alignmask | enc->cra_alignmask; + inst->alg.cra_alignmask = auth_base->cra_alignmask | enc->cra_alignmask; inst->alg.cra_type = &crypto_aead_type; inst->alg.cra_aead.ivsize = enc->cra_ablkcipher.ivsize; - inst->alg.cra_aead.maxauthsize = auth->cra_type == &crypto_hash_type ? - auth->cra_hash.digestsize : - auth->cra_type ? - __crypto_shash_alg(auth)->digestsize : - auth->cra_digest.dia_digestsize; + inst->alg.cra_aead.maxauthsize = auth->digestsize; inst->alg.cra_ctxsize = sizeof(struct crypto_authenc_ctx); @@ -453,13 +678,13 @@ static struct crypto_instance *crypto_authenc_alloc(struct rtattr **tb) inst->alg.cra_aead.givencrypt = crypto_authenc_givencrypt; out: - crypto_mod_put(auth); + crypto_mod_put(auth_base); return inst; err_drop_enc: crypto_drop_skcipher(&ctx->enc); err_drop_auth: - crypto_drop_spawn(&ctx->auth); + crypto_drop_ahash(&ctx->auth); err_free_inst: kfree(inst); out_put_auth: @@ -472,7 +697,7 @@ static void crypto_authenc_free(struct crypto_instance *inst) struct authenc_instance_ctx *ctx = crypto_instance_ctx(inst); crypto_drop_skcipher(&ctx->enc); - crypto_drop_spawn(&ctx->auth); + crypto_drop_ahash(&ctx->auth); kfree(inst); } diff --git a/crypto/blkcipher.c b/crypto/blkcipher.c index 90d26c91f4e9..7a7219266e3c 100644 --- a/crypto/blkcipher.c +++ b/crypto/blkcipher.c @@ -89,9 +89,9 @@ static inline unsigned int blkcipher_done_fast(struct blkcipher_walk *walk, memcpy(walk->dst.virt.addr, walk->page, n); blkcipher_unmap_dst(walk); } else if (!(walk->flags & BLKCIPHER_WALK_PHYS)) { - blkcipher_unmap_src(walk); if (walk->flags & BLKCIPHER_WALK_DIFF) blkcipher_unmap_dst(walk); + blkcipher_unmap_src(walk); } scatterwalk_advance(&walk->in, n); diff --git a/crypto/blowfish.c b/crypto/blowfish.c index 6f5b48731922..a67d52ee0580 100644 --- a/crypto/blowfish.c +++ b/crypto/blowfish.c @@ -1,4 +1,4 @@ -/* +/* * Cryptographic API. * * Blowfish Cipher Algorithm, by Bruce Schneier. @@ -299,7 +299,7 @@ static const u32 bf_sbox[256 * 4] = { 0xb74e6132, 0xce77e25b, 0x578fdfe3, 0x3ac372e6, }; -/* +/* * Round loop unrolling macros, S is a pointer to a S-Box array * organized in 4 unsigned longs at a row. */ @@ -315,7 +315,7 @@ static const u32 bf_sbox[256 * 4] = { /* * The blowfish encipher, processes 64-bit blocks. - * NOTE: This function MUSTN'T respect endianess + * NOTE: This function MUSTN'T respect endianess */ static void encrypt_block(struct bf_ctx *bctx, u32 *dst, u32 *src) { @@ -395,7 +395,7 @@ static void bf_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) out_blk[1] = cpu_to_be32(yl); } -/* +/* * Calculates the blowfish S and P boxes for encryption and decryption. */ static int bf_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen) @@ -417,10 +417,10 @@ static int bf_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen) /* Actual subkey generation */ for (j = 0, i = 0; i < 16 + 2; i++) { - temp = (((u32 )key[j] << 24) | - ((u32 )key[(j + 1) % keylen] << 16) | - ((u32 )key[(j + 2) % keylen] << 8) | - ((u32 )key[(j + 3) % keylen])); + temp = (((u32)key[j] << 24) | + ((u32)key[(j + 1) % keylen] << 16) | + ((u32)key[(j + 2) % keylen] << 8) | + ((u32)key[(j + 3) % keylen])); P[i] = P[i] ^ temp; j = (j + 4) % keylen; @@ -444,7 +444,7 @@ static int bf_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen) S[count + 1] = data[1]; } } - + /* Bruce says not to bother with the weak key check. */ return 0; } diff --git a/crypto/camellia.c b/crypto/camellia.c index 964635d163f4..64cff46ea5e4 100644 --- a/crypto/camellia.c +++ b/crypto/camellia.c @@ -39,271 +39,271 @@ #include <asm/unaligned.h> static const u32 camellia_sp1110[256] = { - 0x70707000,0x82828200,0x2c2c2c00,0xececec00, - 0xb3b3b300,0x27272700,0xc0c0c000,0xe5e5e500, - 0xe4e4e400,0x85858500,0x57575700,0x35353500, - 0xeaeaea00,0x0c0c0c00,0xaeaeae00,0x41414100, - 0x23232300,0xefefef00,0x6b6b6b00,0x93939300, - 0x45454500,0x19191900,0xa5a5a500,0x21212100, - 0xededed00,0x0e0e0e00,0x4f4f4f00,0x4e4e4e00, - 0x1d1d1d00,0x65656500,0x92929200,0xbdbdbd00, - 0x86868600,0xb8b8b800,0xafafaf00,0x8f8f8f00, - 0x7c7c7c00,0xebebeb00,0x1f1f1f00,0xcecece00, - 0x3e3e3e00,0x30303000,0xdcdcdc00,0x5f5f5f00, - 0x5e5e5e00,0xc5c5c500,0x0b0b0b00,0x1a1a1a00, - 0xa6a6a600,0xe1e1e100,0x39393900,0xcacaca00, - 0xd5d5d500,0x47474700,0x5d5d5d00,0x3d3d3d00, - 0xd9d9d900,0x01010100,0x5a5a5a00,0xd6d6d600, - 0x51515100,0x56565600,0x6c6c6c00,0x4d4d4d00, - 0x8b8b8b00,0x0d0d0d00,0x9a9a9a00,0x66666600, - 0xfbfbfb00,0xcccccc00,0xb0b0b000,0x2d2d2d00, - 0x74747400,0x12121200,0x2b2b2b00,0x20202000, - 0xf0f0f000,0xb1b1b100,0x84848400,0x99999900, - 0xdfdfdf00,0x4c4c4c00,0xcbcbcb00,0xc2c2c200, - 0x34343400,0x7e7e7e00,0x76767600,0x05050500, - 0x6d6d6d00,0xb7b7b700,0xa9a9a900,0x31313100, - 0xd1d1d100,0x17171700,0x04040400,0xd7d7d700, - 0x14141400,0x58585800,0x3a3a3a00,0x61616100, - 0xdedede00,0x1b1b1b00,0x11111100,0x1c1c1c00, - 0x32323200,0x0f0f0f00,0x9c9c9c00,0x16161600, - 0x53535300,0x18181800,0xf2f2f200,0x22222200, - 0xfefefe00,0x44444400,0xcfcfcf00,0xb2b2b200, - 0xc3c3c300,0xb5b5b500,0x7a7a7a00,0x91919100, - 0x24242400,0x08080800,0xe8e8e800,0xa8a8a800, - 0x60606000,0xfcfcfc00,0x69696900,0x50505000, - 0xaaaaaa00,0xd0d0d000,0xa0a0a000,0x7d7d7d00, - 0xa1a1a100,0x89898900,0x62626200,0x97979700, - 0x54545400,0x5b5b5b00,0x1e1e1e00,0x95959500, - 0xe0e0e000,0xffffff00,0x64646400,0xd2d2d200, - 0x10101000,0xc4c4c400,0x00000000,0x48484800, - 0xa3a3a300,0xf7f7f700,0x75757500,0xdbdbdb00, - 0x8a8a8a00,0x03030300,0xe6e6e600,0xdadada00, - 0x09090900,0x3f3f3f00,0xdddddd00,0x94949400, - 0x87878700,0x5c5c5c00,0x83838300,0x02020200, - 0xcdcdcd00,0x4a4a4a00,0x90909000,0x33333300, - 0x73737300,0x67676700,0xf6f6f600,0xf3f3f300, - 0x9d9d9d00,0x7f7f7f00,0xbfbfbf00,0xe2e2e200, - 0x52525200,0x9b9b9b00,0xd8d8d800,0x26262600, - 0xc8c8c800,0x37373700,0xc6c6c600,0x3b3b3b00, - 0x81818100,0x96969600,0x6f6f6f00,0x4b4b4b00, - 0x13131300,0xbebebe00,0x63636300,0x2e2e2e00, - 0xe9e9e900,0x79797900,0xa7a7a700,0x8c8c8c00, - 0x9f9f9f00,0x6e6e6e00,0xbcbcbc00,0x8e8e8e00, - 0x29292900,0xf5f5f500,0xf9f9f900,0xb6b6b600, - 0x2f2f2f00,0xfdfdfd00,0xb4b4b400,0x59595900, - 0x78787800,0x98989800,0x06060600,0x6a6a6a00, - 0xe7e7e700,0x46464600,0x71717100,0xbababa00, - 0xd4d4d400,0x25252500,0xababab00,0x42424200, - 0x88888800,0xa2a2a200,0x8d8d8d00,0xfafafa00, - 0x72727200,0x07070700,0xb9b9b900,0x55555500, - 0xf8f8f800,0xeeeeee00,0xacacac00,0x0a0a0a00, - 0x36363600,0x49494900,0x2a2a2a00,0x68686800, - 0x3c3c3c00,0x38383800,0xf1f1f100,0xa4a4a400, - 0x40404000,0x28282800,0xd3d3d300,0x7b7b7b00, - 0xbbbbbb00,0xc9c9c900,0x43434300,0xc1c1c100, - 0x15151500,0xe3e3e300,0xadadad00,0xf4f4f400, - 0x77777700,0xc7c7c700,0x80808000,0x9e9e9e00, + 0x70707000, 0x82828200, 0x2c2c2c00, 0xececec00, + 0xb3b3b300, 0x27272700, 0xc0c0c000, 0xe5e5e500, + 0xe4e4e400, 0x85858500, 0x57575700, 0x35353500, + 0xeaeaea00, 0x0c0c0c00, 0xaeaeae00, 0x41414100, + 0x23232300, 0xefefef00, 0x6b6b6b00, 0x93939300, + 0x45454500, 0x19191900, 0xa5a5a500, 0x21212100, + 0xededed00, 0x0e0e0e00, 0x4f4f4f00, 0x4e4e4e00, + 0x1d1d1d00, 0x65656500, 0x92929200, 0xbdbdbd00, + 0x86868600, 0xb8b8b800, 0xafafaf00, 0x8f8f8f00, + 0x7c7c7c00, 0xebebeb00, 0x1f1f1f00, 0xcecece00, + 0x3e3e3e00, 0x30303000, 0xdcdcdc00, 0x5f5f5f00, + 0x5e5e5e00, 0xc5c5c500, 0x0b0b0b00, 0x1a1a1a00, + 0xa6a6a600, 0xe1e1e100, 0x39393900, 0xcacaca00, + 0xd5d5d500, 0x47474700, 0x5d5d5d00, 0x3d3d3d00, + 0xd9d9d900, 0x01010100, 0x5a5a5a00, 0xd6d6d600, + 0x51515100, 0x56565600, 0x6c6c6c00, 0x4d4d4d00, + 0x8b8b8b00, 0x0d0d0d00, 0x9a9a9a00, 0x66666600, + 0xfbfbfb00, 0xcccccc00, 0xb0b0b000, 0x2d2d2d00, + 0x74747400, 0x12121200, 0x2b2b2b00, 0x20202000, + 0xf0f0f000, 0xb1b1b100, 0x84848400, 0x99999900, + 0xdfdfdf00, 0x4c4c4c00, 0xcbcbcb00, 0xc2c2c200, + 0x34343400, 0x7e7e7e00, 0x76767600, 0x05050500, + 0x6d6d6d00, 0xb7b7b700, 0xa9a9a900, 0x31313100, + 0xd1d1d100, 0x17171700, 0x04040400, 0xd7d7d700, + 0x14141400, 0x58585800, 0x3a3a3a00, 0x61616100, + 0xdedede00, 0x1b1b1b00, 0x11111100, 0x1c1c1c00, + 0x32323200, 0x0f0f0f00, 0x9c9c9c00, 0x16161600, + 0x53535300, 0x18181800, 0xf2f2f200, 0x22222200, + 0xfefefe00, 0x44444400, 0xcfcfcf00, 0xb2b2b200, + 0xc3c3c300, 0xb5b5b500, 0x7a7a7a00, 0x91919100, + 0x24242400, 0x08080800, 0xe8e8e800, 0xa8a8a800, + 0x60606000, 0xfcfcfc00, 0x69696900, 0x50505000, + 0xaaaaaa00, 0xd0d0d000, 0xa0a0a000, 0x7d7d7d00, + 0xa1a1a100, 0x89898900, 0x62626200, 0x97979700, + 0x54545400, 0x5b5b5b00, 0x1e1e1e00, 0x95959500, + 0xe0e0e000, 0xffffff00, 0x64646400, 0xd2d2d200, + 0x10101000, 0xc4c4c400, 0x00000000, 0x48484800, + 0xa3a3a300, 0xf7f7f700, 0x75757500, 0xdbdbdb00, + 0x8a8a8a00, 0x03030300, 0xe6e6e600, 0xdadada00, + 0x09090900, 0x3f3f3f00, 0xdddddd00, 0x94949400, + 0x87878700, 0x5c5c5c00, 0x83838300, 0x02020200, + 0xcdcdcd00, 0x4a4a4a00, 0x90909000, 0x33333300, + 0x73737300, 0x67676700, 0xf6f6f600, 0xf3f3f300, + 0x9d9d9d00, 0x7f7f7f00, 0xbfbfbf00, 0xe2e2e200, + 0x52525200, 0x9b9b9b00, 0xd8d8d800, 0x26262600, + 0xc8c8c800, 0x37373700, 0xc6c6c600, 0x3b3b3b00, + 0x81818100, 0x96969600, 0x6f6f6f00, 0x4b4b4b00, + 0x13131300, 0xbebebe00, 0x63636300, 0x2e2e2e00, + 0xe9e9e900, 0x79797900, 0xa7a7a700, 0x8c8c8c00, + 0x9f9f9f00, 0x6e6e6e00, 0xbcbcbc00, 0x8e8e8e00, + 0x29292900, 0xf5f5f500, 0xf9f9f900, 0xb6b6b600, + 0x2f2f2f00, 0xfdfdfd00, 0xb4b4b400, 0x59595900, + 0x78787800, 0x98989800, 0x06060600, 0x6a6a6a00, + 0xe7e7e700, 0x46464600, 0x71717100, 0xbababa00, + 0xd4d4d400, 0x25252500, 0xababab00, 0x42424200, + 0x88888800, 0xa2a2a200, 0x8d8d8d00, 0xfafafa00, + 0x72727200, 0x07070700, 0xb9b9b900, 0x55555500, + 0xf8f8f800, 0xeeeeee00, 0xacacac00, 0x0a0a0a00, + 0x36363600, 0x49494900, 0x2a2a2a00, 0x68686800, + 0x3c3c3c00, 0x38383800, 0xf1f1f100, 0xa4a4a400, + 0x40404000, 0x28282800, 0xd3d3d300, 0x7b7b7b00, + 0xbbbbbb00, 0xc9c9c900, 0x43434300, 0xc1c1c100, + 0x15151500, 0xe3e3e300, 0xadadad00, 0xf4f4f400, + 0x77777700, 0xc7c7c700, 0x80808000, 0x9e9e9e00, }; static const u32 camellia_sp0222[256] = { - 0x00e0e0e0,0x00050505,0x00585858,0x00d9d9d9, - 0x00676767,0x004e4e4e,0x00818181,0x00cbcbcb, - 0x00c9c9c9,0x000b0b0b,0x00aeaeae,0x006a6a6a, - 0x00d5d5d5,0x00181818,0x005d5d5d,0x00828282, - 0x00464646,0x00dfdfdf,0x00d6d6d6,0x00272727, - 0x008a8a8a,0x00323232,0x004b4b4b,0x00424242, - 0x00dbdbdb,0x001c1c1c,0x009e9e9e,0x009c9c9c, - 0x003a3a3a,0x00cacaca,0x00252525,0x007b7b7b, - 0x000d0d0d,0x00717171,0x005f5f5f,0x001f1f1f, - 0x00f8f8f8,0x00d7d7d7,0x003e3e3e,0x009d9d9d, - 0x007c7c7c,0x00606060,0x00b9b9b9,0x00bebebe, - 0x00bcbcbc,0x008b8b8b,0x00161616,0x00343434, - 0x004d4d4d,0x00c3c3c3,0x00727272,0x00959595, - 0x00ababab,0x008e8e8e,0x00bababa,0x007a7a7a, - 0x00b3b3b3,0x00020202,0x00b4b4b4,0x00adadad, - 0x00a2a2a2,0x00acacac,0x00d8d8d8,0x009a9a9a, - 0x00171717,0x001a1a1a,0x00353535,0x00cccccc, - 0x00f7f7f7,0x00999999,0x00616161,0x005a5a5a, - 0x00e8e8e8,0x00242424,0x00565656,0x00404040, - 0x00e1e1e1,0x00636363,0x00090909,0x00333333, - 0x00bfbfbf,0x00989898,0x00979797,0x00858585, - 0x00686868,0x00fcfcfc,0x00ececec,0x000a0a0a, - 0x00dadada,0x006f6f6f,0x00535353,0x00626262, - 0x00a3a3a3,0x002e2e2e,0x00080808,0x00afafaf, - 0x00282828,0x00b0b0b0,0x00747474,0x00c2c2c2, - 0x00bdbdbd,0x00363636,0x00222222,0x00383838, - 0x00646464,0x001e1e1e,0x00393939,0x002c2c2c, - 0x00a6a6a6,0x00303030,0x00e5e5e5,0x00444444, - 0x00fdfdfd,0x00888888,0x009f9f9f,0x00656565, - 0x00878787,0x006b6b6b,0x00f4f4f4,0x00232323, - 0x00484848,0x00101010,0x00d1d1d1,0x00515151, - 0x00c0c0c0,0x00f9f9f9,0x00d2d2d2,0x00a0a0a0, - 0x00555555,0x00a1a1a1,0x00414141,0x00fafafa, - 0x00434343,0x00131313,0x00c4c4c4,0x002f2f2f, - 0x00a8a8a8,0x00b6b6b6,0x003c3c3c,0x002b2b2b, - 0x00c1c1c1,0x00ffffff,0x00c8c8c8,0x00a5a5a5, - 0x00202020,0x00898989,0x00000000,0x00909090, - 0x00474747,0x00efefef,0x00eaeaea,0x00b7b7b7, - 0x00151515,0x00060606,0x00cdcdcd,0x00b5b5b5, - 0x00121212,0x007e7e7e,0x00bbbbbb,0x00292929, - 0x000f0f0f,0x00b8b8b8,0x00070707,0x00040404, - 0x009b9b9b,0x00949494,0x00212121,0x00666666, - 0x00e6e6e6,0x00cecece,0x00ededed,0x00e7e7e7, - 0x003b3b3b,0x00fefefe,0x007f7f7f,0x00c5c5c5, - 0x00a4a4a4,0x00373737,0x00b1b1b1,0x004c4c4c, - 0x00919191,0x006e6e6e,0x008d8d8d,0x00767676, - 0x00030303,0x002d2d2d,0x00dedede,0x00969696, - 0x00262626,0x007d7d7d,0x00c6c6c6,0x005c5c5c, - 0x00d3d3d3,0x00f2f2f2,0x004f4f4f,0x00191919, - 0x003f3f3f,0x00dcdcdc,0x00797979,0x001d1d1d, - 0x00525252,0x00ebebeb,0x00f3f3f3,0x006d6d6d, - 0x005e5e5e,0x00fbfbfb,0x00696969,0x00b2b2b2, - 0x00f0f0f0,0x00313131,0x000c0c0c,0x00d4d4d4, - 0x00cfcfcf,0x008c8c8c,0x00e2e2e2,0x00757575, - 0x00a9a9a9,0x004a4a4a,0x00575757,0x00848484, - 0x00111111,0x00454545,0x001b1b1b,0x00f5f5f5, - 0x00e4e4e4,0x000e0e0e,0x00737373,0x00aaaaaa, - 0x00f1f1f1,0x00dddddd,0x00595959,0x00141414, - 0x006c6c6c,0x00929292,0x00545454,0x00d0d0d0, - 0x00787878,0x00707070,0x00e3e3e3,0x00494949, - 0x00808080,0x00505050,0x00a7a7a7,0x00f6f6f6, - 0x00777777,0x00939393,0x00868686,0x00838383, - 0x002a2a2a,0x00c7c7c7,0x005b5b5b,0x00e9e9e9, - 0x00eeeeee,0x008f8f8f,0x00010101,0x003d3d3d, + 0x00e0e0e0, 0x00050505, 0x00585858, 0x00d9d9d9, + 0x00676767, 0x004e4e4e, 0x00818181, 0x00cbcbcb, + 0x00c9c9c9, 0x000b0b0b, 0x00aeaeae, 0x006a6a6a, + 0x00d5d5d5, 0x00181818, 0x005d5d5d, 0x00828282, + 0x00464646, 0x00dfdfdf, 0x00d6d6d6, 0x00272727, + 0x008a8a8a, 0x00323232, 0x004b4b4b, 0x00424242, + 0x00dbdbdb, 0x001c1c1c, 0x009e9e9e, 0x009c9c9c, + 0x003a3a3a, 0x00cacaca, 0x00252525, 0x007b7b7b, + 0x000d0d0d, 0x00717171, 0x005f5f5f, 0x001f1f1f, + 0x00f8f8f8, 0x00d7d7d7, 0x003e3e3e, 0x009d9d9d, + 0x007c7c7c, 0x00606060, 0x00b9b9b9, 0x00bebebe, + 0x00bcbcbc, 0x008b8b8b, 0x00161616, 0x00343434, + 0x004d4d4d, 0x00c3c3c3, 0x00727272, 0x00959595, + 0x00ababab, 0x008e8e8e, 0x00bababa, 0x007a7a7a, + 0x00b3b3b3, 0x00020202, 0x00b4b4b4, 0x00adadad, + 0x00a2a2a2, 0x00acacac, 0x00d8d8d8, 0x009a9a9a, + 0x00171717, 0x001a1a1a, 0x00353535, 0x00cccccc, + 0x00f7f7f7, 0x00999999, 0x00616161, 0x005a5a5a, + 0x00e8e8e8, 0x00242424, 0x00565656, 0x00404040, + 0x00e1e1e1, 0x00636363, 0x00090909, 0x00333333, + 0x00bfbfbf, 0x00989898, 0x00979797, 0x00858585, + 0x00686868, 0x00fcfcfc, 0x00ececec, 0x000a0a0a, + 0x00dadada, 0x006f6f6f, 0x00535353, 0x00626262, + 0x00a3a3a3, 0x002e2e2e, 0x00080808, 0x00afafaf, + 0x00282828, 0x00b0b0b0, 0x00747474, 0x00c2c2c2, + 0x00bdbdbd, 0x00363636, 0x00222222, 0x00383838, + 0x00646464, 0x001e1e1e, 0x00393939, 0x002c2c2c, + 0x00a6a6a6, 0x00303030, 0x00e5e5e5, 0x00444444, + 0x00fdfdfd, 0x00888888, 0x009f9f9f, 0x00656565, + 0x00878787, 0x006b6b6b, 0x00f4f4f4, 0x00232323, + 0x00484848, 0x00101010, 0x00d1d1d1, 0x00515151, + 0x00c0c0c0, 0x00f9f9f9, 0x00d2d2d2, 0x00a0a0a0, + 0x00555555, 0x00a1a1a1, 0x00414141, 0x00fafafa, + 0x00434343, 0x00131313, 0x00c4c4c4, 0x002f2f2f, + 0x00a8a8a8, 0x00b6b6b6, 0x003c3c3c, 0x002b2b2b, + 0x00c1c1c1, 0x00ffffff, 0x00c8c8c8, 0x00a5a5a5, + 0x00202020, 0x00898989, 0x00000000, 0x00909090, + 0x00474747, 0x00efefef, 0x00eaeaea, 0x00b7b7b7, + 0x00151515, 0x00060606, 0x00cdcdcd, 0x00b5b5b5, + 0x00121212, 0x007e7e7e, 0x00bbbbbb, 0x00292929, + 0x000f0f0f, 0x00b8b8b8, 0x00070707, 0x00040404, + 0x009b9b9b, 0x00949494, 0x00212121, 0x00666666, + 0x00e6e6e6, 0x00cecece, 0x00ededed, 0x00e7e7e7, + 0x003b3b3b, 0x00fefefe, 0x007f7f7f, 0x00c5c5c5, + 0x00a4a4a4, 0x00373737, 0x00b1b1b1, 0x004c4c4c, + 0x00919191, 0x006e6e6e, 0x008d8d8d, 0x00767676, + 0x00030303, 0x002d2d2d, 0x00dedede, 0x00969696, + 0x00262626, 0x007d7d7d, 0x00c6c6c6, 0x005c5c5c, + 0x00d3d3d3, 0x00f2f2f2, 0x004f4f4f, 0x00191919, + 0x003f3f3f, 0x00dcdcdc, 0x00797979, 0x001d1d1d, + 0x00525252, 0x00ebebeb, 0x00f3f3f3, 0x006d6d6d, + 0x005e5e5e, 0x00fbfbfb, 0x00696969, 0x00b2b2b2, + 0x00f0f0f0, 0x00313131, 0x000c0c0c, 0x00d4d4d4, + 0x00cfcfcf, 0x008c8c8c, 0x00e2e2e2, 0x00757575, + 0x00a9a9a9, 0x004a4a4a, 0x00575757, 0x00848484, + 0x00111111, 0x00454545, 0x001b1b1b, 0x00f5f5f5, + 0x00e4e4e4, 0x000e0e0e, 0x00737373, 0x00aaaaaa, + 0x00f1f1f1, 0x00dddddd, 0x00595959, 0x00141414, + 0x006c6c6c, 0x00929292, 0x00545454, 0x00d0d0d0, + 0x00787878, 0x00707070, 0x00e3e3e3, 0x00494949, + 0x00808080, 0x00505050, 0x00a7a7a7, 0x00f6f6f6, + 0x00777777, 0x00939393, 0x00868686, 0x00838383, + 0x002a2a2a, 0x00c7c7c7, 0x005b5b5b, 0x00e9e9e9, + 0x00eeeeee, 0x008f8f8f, 0x00010101, 0x003d3d3d, }; static const u32 camellia_sp3033[256] = { - 0x38003838,0x41004141,0x16001616,0x76007676, - 0xd900d9d9,0x93009393,0x60006060,0xf200f2f2, - 0x72007272,0xc200c2c2,0xab00abab,0x9a009a9a, - 0x75007575,0x06000606,0x57005757,0xa000a0a0, - 0x91009191,0xf700f7f7,0xb500b5b5,0xc900c9c9, - 0xa200a2a2,0x8c008c8c,0xd200d2d2,0x90009090, - 0xf600f6f6,0x07000707,0xa700a7a7,0x27002727, - 0x8e008e8e,0xb200b2b2,0x49004949,0xde00dede, - 0x43004343,0x5c005c5c,0xd700d7d7,0xc700c7c7, - 0x3e003e3e,0xf500f5f5,0x8f008f8f,0x67006767, - 0x1f001f1f,0x18001818,0x6e006e6e,0xaf00afaf, - 0x2f002f2f,0xe200e2e2,0x85008585,0x0d000d0d, - 0x53005353,0xf000f0f0,0x9c009c9c,0x65006565, - 0xea00eaea,0xa300a3a3,0xae00aeae,0x9e009e9e, - 0xec00ecec,0x80008080,0x2d002d2d,0x6b006b6b, - 0xa800a8a8,0x2b002b2b,0x36003636,0xa600a6a6, - 0xc500c5c5,0x86008686,0x4d004d4d,0x33003333, - 0xfd00fdfd,0x66006666,0x58005858,0x96009696, - 0x3a003a3a,0x09000909,0x95009595,0x10001010, - 0x78007878,0xd800d8d8,0x42004242,0xcc00cccc, - 0xef00efef,0x26002626,0xe500e5e5,0x61006161, - 0x1a001a1a,0x3f003f3f,0x3b003b3b,0x82008282, - 0xb600b6b6,0xdb00dbdb,0xd400d4d4,0x98009898, - 0xe800e8e8,0x8b008b8b,0x02000202,0xeb00ebeb, - 0x0a000a0a,0x2c002c2c,0x1d001d1d,0xb000b0b0, - 0x6f006f6f,0x8d008d8d,0x88008888,0x0e000e0e, - 0x19001919,0x87008787,0x4e004e4e,0x0b000b0b, - 0xa900a9a9,0x0c000c0c,0x79007979,0x11001111, - 0x7f007f7f,0x22002222,0xe700e7e7,0x59005959, - 0xe100e1e1,0xda00dada,0x3d003d3d,0xc800c8c8, - 0x12001212,0x04000404,0x74007474,0x54005454, - 0x30003030,0x7e007e7e,0xb400b4b4,0x28002828, - 0x55005555,0x68006868,0x50005050,0xbe00bebe, - 0xd000d0d0,0xc400c4c4,0x31003131,0xcb00cbcb, - 0x2a002a2a,0xad00adad,0x0f000f0f,0xca00caca, - 0x70007070,0xff00ffff,0x32003232,0x69006969, - 0x08000808,0x62006262,0x00000000,0x24002424, - 0xd100d1d1,0xfb00fbfb,0xba00baba,0xed00eded, - 0x45004545,0x81008181,0x73007373,0x6d006d6d, - 0x84008484,0x9f009f9f,0xee00eeee,0x4a004a4a, - 0xc300c3c3,0x2e002e2e,0xc100c1c1,0x01000101, - 0xe600e6e6,0x25002525,0x48004848,0x99009999, - 0xb900b9b9,0xb300b3b3,0x7b007b7b,0xf900f9f9, - 0xce00cece,0xbf00bfbf,0xdf00dfdf,0x71007171, - 0x29002929,0xcd00cdcd,0x6c006c6c,0x13001313, - 0x64006464,0x9b009b9b,0x63006363,0x9d009d9d, - 0xc000c0c0,0x4b004b4b,0xb700b7b7,0xa500a5a5, - 0x89008989,0x5f005f5f,0xb100b1b1,0x17001717, - 0xf400f4f4,0xbc00bcbc,0xd300d3d3,0x46004646, - 0xcf00cfcf,0x37003737,0x5e005e5e,0x47004747, - 0x94009494,0xfa00fafa,0xfc00fcfc,0x5b005b5b, - 0x97009797,0xfe00fefe,0x5a005a5a,0xac00acac, - 0x3c003c3c,0x4c004c4c,0x03000303,0x35003535, - 0xf300f3f3,0x23002323,0xb800b8b8,0x5d005d5d, - 0x6a006a6a,0x92009292,0xd500d5d5,0x21002121, - 0x44004444,0x51005151,0xc600c6c6,0x7d007d7d, - 0x39003939,0x83008383,0xdc00dcdc,0xaa00aaaa, - 0x7c007c7c,0x77007777,0x56005656,0x05000505, - 0x1b001b1b,0xa400a4a4,0x15001515,0x34003434, - 0x1e001e1e,0x1c001c1c,0xf800f8f8,0x52005252, - 0x20002020,0x14001414,0xe900e9e9,0xbd00bdbd, - 0xdd00dddd,0xe400e4e4,0xa100a1a1,0xe000e0e0, - 0x8a008a8a,0xf100f1f1,0xd600d6d6,0x7a007a7a, - 0xbb00bbbb,0xe300e3e3,0x40004040,0x4f004f4f, + 0x38003838, 0x41004141, 0x16001616, 0x76007676, + 0xd900d9d9, 0x93009393, 0x60006060, 0xf200f2f2, + 0x72007272, 0xc200c2c2, 0xab00abab, 0x9a009a9a, + 0x75007575, 0x06000606, 0x57005757, 0xa000a0a0, + 0x91009191, 0xf700f7f7, 0xb500b5b5, 0xc900c9c9, + 0xa200a2a2, 0x8c008c8c, 0xd200d2d2, 0x90009090, + 0xf600f6f6, 0x07000707, 0xa700a7a7, 0x27002727, + 0x8e008e8e, 0xb200b2b2, 0x49004949, 0xde00dede, + 0x43004343, 0x5c005c5c, 0xd700d7d7, 0xc700c7c7, + 0x3e003e3e, 0xf500f5f5, 0x8f008f8f, 0x67006767, + 0x1f001f1f, 0x18001818, 0x6e006e6e, 0xaf00afaf, + 0x2f002f2f, 0xe200e2e2, 0x85008585, 0x0d000d0d, + 0x53005353, 0xf000f0f0, 0x9c009c9c, 0x65006565, + 0xea00eaea, 0xa300a3a3, 0xae00aeae, 0x9e009e9e, + 0xec00ecec, 0x80008080, 0x2d002d2d, 0x6b006b6b, + 0xa800a8a8, 0x2b002b2b, 0x36003636, 0xa600a6a6, + 0xc500c5c5, 0x86008686, 0x4d004d4d, 0x33003333, + 0xfd00fdfd, 0x66006666, 0x58005858, 0x96009696, + 0x3a003a3a, 0x09000909, 0x95009595, 0x10001010, + 0x78007878, 0xd800d8d8, 0x42004242, 0xcc00cccc, + 0xef00efef, 0x26002626, 0xe500e5e5, 0x61006161, + 0x1a001a1a, 0x3f003f3f, 0x3b003b3b, 0x82008282, + 0xb600b6b6, 0xdb00dbdb, 0xd400d4d4, 0x98009898, + 0xe800e8e8, 0x8b008b8b, 0x02000202, 0xeb00ebeb, + 0x0a000a0a, 0x2c002c2c, 0x1d001d1d, 0xb000b0b0, + 0x6f006f6f, 0x8d008d8d, 0x88008888, 0x0e000e0e, + 0x19001919, 0x87008787, 0x4e004e4e, 0x0b000b0b, + 0xa900a9a9, 0x0c000c0c, 0x79007979, 0x11001111, + 0x7f007f7f, 0x22002222, 0xe700e7e7, 0x59005959, + 0xe100e1e1, 0xda00dada, 0x3d003d3d, 0xc800c8c8, + 0x12001212, 0x04000404, 0x74007474, 0x54005454, + 0x30003030, 0x7e007e7e, 0xb400b4b4, 0x28002828, + 0x55005555, 0x68006868, 0x50005050, 0xbe00bebe, + 0xd000d0d0, 0xc400c4c4, 0x31003131, 0xcb00cbcb, + 0x2a002a2a, 0xad00adad, 0x0f000f0f, 0xca00caca, + 0x70007070, 0xff00ffff, 0x32003232, 0x69006969, + 0x08000808, 0x62006262, 0x00000000, 0x24002424, + 0xd100d1d1, 0xfb00fbfb, 0xba00baba, 0xed00eded, + 0x45004545, 0x81008181, 0x73007373, 0x6d006d6d, + 0x84008484, 0x9f009f9f, 0xee00eeee, 0x4a004a4a, + 0xc300c3c3, 0x2e002e2e, 0xc100c1c1, 0x01000101, + 0xe600e6e6, 0x25002525, 0x48004848, 0x99009999, + 0xb900b9b9, 0xb300b3b3, 0x7b007b7b, 0xf900f9f9, + 0xce00cece, 0xbf00bfbf, 0xdf00dfdf, 0x71007171, + 0x29002929, 0xcd00cdcd, 0x6c006c6c, 0x13001313, + 0x64006464, 0x9b009b9b, 0x63006363, 0x9d009d9d, + 0xc000c0c0, 0x4b004b4b, 0xb700b7b7, 0xa500a5a5, + 0x89008989, 0x5f005f5f, 0xb100b1b1, 0x17001717, + 0xf400f4f4, 0xbc00bcbc, 0xd300d3d3, 0x46004646, + 0xcf00cfcf, 0x37003737, 0x5e005e5e, 0x47004747, + 0x94009494, 0xfa00fafa, 0xfc00fcfc, 0x5b005b5b, + 0x97009797, 0xfe00fefe, 0x5a005a5a, 0xac00acac, + 0x3c003c3c, 0x4c004c4c, 0x03000303, 0x35003535, + 0xf300f3f3, 0x23002323, 0xb800b8b8, 0x5d005d5d, + 0x6a006a6a, 0x92009292, 0xd500d5d5, 0x21002121, + 0x44004444, 0x51005151, 0xc600c6c6, 0x7d007d7d, + 0x39003939, 0x83008383, 0xdc00dcdc, 0xaa00aaaa, + 0x7c007c7c, 0x77007777, 0x56005656, 0x05000505, + 0x1b001b1b, 0xa400a4a4, 0x15001515, 0x34003434, + 0x1e001e1e, 0x1c001c1c, 0xf800f8f8, 0x52005252, + 0x20002020, 0x14001414, 0xe900e9e9, 0xbd00bdbd, + 0xdd00dddd, 0xe400e4e4, 0xa100a1a1, 0xe000e0e0, + 0x8a008a8a, 0xf100f1f1, 0xd600d6d6, 0x7a007a7a, + 0xbb00bbbb, 0xe300e3e3, 0x40004040, 0x4f004f4f, }; static const u32 camellia_sp4404[256] = { - 0x70700070,0x2c2c002c,0xb3b300b3,0xc0c000c0, - 0xe4e400e4,0x57570057,0xeaea00ea,0xaeae00ae, - 0x23230023,0x6b6b006b,0x45450045,0xa5a500a5, - 0xeded00ed,0x4f4f004f,0x1d1d001d,0x92920092, - 0x86860086,0xafaf00af,0x7c7c007c,0x1f1f001f, - 0x3e3e003e,0xdcdc00dc,0x5e5e005e,0x0b0b000b, - 0xa6a600a6,0x39390039,0xd5d500d5,0x5d5d005d, - 0xd9d900d9,0x5a5a005a,0x51510051,0x6c6c006c, - 0x8b8b008b,0x9a9a009a,0xfbfb00fb,0xb0b000b0, - 0x74740074,0x2b2b002b,0xf0f000f0,0x84840084, - 0xdfdf00df,0xcbcb00cb,0x34340034,0x76760076, - 0x6d6d006d,0xa9a900a9,0xd1d100d1,0x04040004, - 0x14140014,0x3a3a003a,0xdede00de,0x11110011, - 0x32320032,0x9c9c009c,0x53530053,0xf2f200f2, - 0xfefe00fe,0xcfcf00cf,0xc3c300c3,0x7a7a007a, - 0x24240024,0xe8e800e8,0x60600060,0x69690069, - 0xaaaa00aa,0xa0a000a0,0xa1a100a1,0x62620062, - 0x54540054,0x1e1e001e,0xe0e000e0,0x64640064, - 0x10100010,0x00000000,0xa3a300a3,0x75750075, - 0x8a8a008a,0xe6e600e6,0x09090009,0xdddd00dd, - 0x87870087,0x83830083,0xcdcd00cd,0x90900090, - 0x73730073,0xf6f600f6,0x9d9d009d,0xbfbf00bf, - 0x52520052,0xd8d800d8,0xc8c800c8,0xc6c600c6, - 0x81810081,0x6f6f006f,0x13130013,0x63630063, - 0xe9e900e9,0xa7a700a7,0x9f9f009f,0xbcbc00bc, - 0x29290029,0xf9f900f9,0x2f2f002f,0xb4b400b4, - 0x78780078,0x06060006,0xe7e700e7,0x71710071, - 0xd4d400d4,0xabab00ab,0x88880088,0x8d8d008d, - 0x72720072,0xb9b900b9,0xf8f800f8,0xacac00ac, - 0x36360036,0x2a2a002a,0x3c3c003c,0xf1f100f1, - 0x40400040,0xd3d300d3,0xbbbb00bb,0x43430043, - 0x15150015,0xadad00ad,0x77770077,0x80800080, - 0x82820082,0xecec00ec,0x27270027,0xe5e500e5, - 0x85850085,0x35350035,0x0c0c000c,0x41410041, - 0xefef00ef,0x93930093,0x19190019,0x21210021, - 0x0e0e000e,0x4e4e004e,0x65650065,0xbdbd00bd, - 0xb8b800b8,0x8f8f008f,0xebeb00eb,0xcece00ce, - 0x30300030,0x5f5f005f,0xc5c500c5,0x1a1a001a, - 0xe1e100e1,0xcaca00ca,0x47470047,0x3d3d003d, - 0x01010001,0xd6d600d6,0x56560056,0x4d4d004d, - 0x0d0d000d,0x66660066,0xcccc00cc,0x2d2d002d, - 0x12120012,0x20200020,0xb1b100b1,0x99990099, - 0x4c4c004c,0xc2c200c2,0x7e7e007e,0x05050005, - 0xb7b700b7,0x31310031,0x17170017,0xd7d700d7, - 0x58580058,0x61610061,0x1b1b001b,0x1c1c001c, - 0x0f0f000f,0x16160016,0x18180018,0x22220022, - 0x44440044,0xb2b200b2,0xb5b500b5,0x91910091, - 0x08080008,0xa8a800a8,0xfcfc00fc,0x50500050, - 0xd0d000d0,0x7d7d007d,0x89890089,0x97970097, - 0x5b5b005b,0x95950095,0xffff00ff,0xd2d200d2, - 0xc4c400c4,0x48480048,0xf7f700f7,0xdbdb00db, - 0x03030003,0xdada00da,0x3f3f003f,0x94940094, - 0x5c5c005c,0x02020002,0x4a4a004a,0x33330033, - 0x67670067,0xf3f300f3,0x7f7f007f,0xe2e200e2, - 0x9b9b009b,0x26260026,0x37370037,0x3b3b003b, - 0x96960096,0x4b4b004b,0xbebe00be,0x2e2e002e, - 0x79790079,0x8c8c008c,0x6e6e006e,0x8e8e008e, - 0xf5f500f5,0xb6b600b6,0xfdfd00fd,0x59590059, - 0x98980098,0x6a6a006a,0x46460046,0xbaba00ba, - 0x25250025,0x42420042,0xa2a200a2,0xfafa00fa, - 0x07070007,0x55550055,0xeeee00ee,0x0a0a000a, - 0x49490049,0x68680068,0x38380038,0xa4a400a4, - 0x28280028,0x7b7b007b,0xc9c900c9,0xc1c100c1, - 0xe3e300e3,0xf4f400f4,0xc7c700c7,0x9e9e009e, + 0x70700070, 0x2c2c002c, 0xb3b300b3, 0xc0c000c0, + 0xe4e400e4, 0x57570057, 0xeaea00ea, 0xaeae00ae, + 0x23230023, 0x6b6b006b, 0x45450045, 0xa5a500a5, + 0xeded00ed, 0x4f4f004f, 0x1d1d001d, 0x92920092, + 0x86860086, 0xafaf00af, 0x7c7c007c, 0x1f1f001f, + 0x3e3e003e, 0xdcdc00dc, 0x5e5e005e, 0x0b0b000b, + 0xa6a600a6, 0x39390039, 0xd5d500d5, 0x5d5d005d, + 0xd9d900d9, 0x5a5a005a, 0x51510051, 0x6c6c006c, + 0x8b8b008b, 0x9a9a009a, 0xfbfb00fb, 0xb0b000b0, + 0x74740074, 0x2b2b002b, 0xf0f000f0, 0x84840084, + 0xdfdf00df, 0xcbcb00cb, 0x34340034, 0x76760076, + 0x6d6d006d, 0xa9a900a9, 0xd1d100d1, 0x04040004, + 0x14140014, 0x3a3a003a, 0xdede00de, 0x11110011, + 0x32320032, 0x9c9c009c, 0x53530053, 0xf2f200f2, + 0xfefe00fe, 0xcfcf00cf, 0xc3c300c3, 0x7a7a007a, + 0x24240024, 0xe8e800e8, 0x60600060, 0x69690069, + 0xaaaa00aa, 0xa0a000a0, 0xa1a100a1, 0x62620062, + 0x54540054, 0x1e1e001e, 0xe0e000e0, 0x64640064, + 0x10100010, 0x00000000, 0xa3a300a3, 0x75750075, + 0x8a8a008a, 0xe6e600e6, 0x09090009, 0xdddd00dd, + 0x87870087, 0x83830083, 0xcdcd00cd, 0x90900090, + 0x73730073, 0xf6f600f6, 0x9d9d009d, 0xbfbf00bf, + 0x52520052, 0xd8d800d8, 0xc8c800c8, 0xc6c600c6, + 0x81810081, 0x6f6f006f, 0x13130013, 0x63630063, + 0xe9e900e9, 0xa7a700a7, 0x9f9f009f, 0xbcbc00bc, + 0x29290029, 0xf9f900f9, 0x2f2f002f, 0xb4b400b4, + 0x78780078, 0x06060006, 0xe7e700e7, 0x71710071, + 0xd4d400d4, 0xabab00ab, 0x88880088, 0x8d8d008d, + 0x72720072, 0xb9b900b9, 0xf8f800f8, 0xacac00ac, + 0x36360036, 0x2a2a002a, 0x3c3c003c, 0xf1f100f1, + 0x40400040, 0xd3d300d3, 0xbbbb00bb, 0x43430043, + 0x15150015, 0xadad00ad, 0x77770077, 0x80800080, + 0x82820082, 0xecec00ec, 0x27270027, 0xe5e500e5, + 0x85850085, 0x35350035, 0x0c0c000c, 0x41410041, + 0xefef00ef, 0x93930093, 0x19190019, 0x21210021, + 0x0e0e000e, 0x4e4e004e, 0x65650065, 0xbdbd00bd, + 0xb8b800b8, 0x8f8f008f, 0xebeb00eb, 0xcece00ce, + 0x30300030, 0x5f5f005f, 0xc5c500c5, 0x1a1a001a, + 0xe1e100e1, 0xcaca00ca, 0x47470047, 0x3d3d003d, + 0x01010001, 0xd6d600d6, 0x56560056, 0x4d4d004d, + 0x0d0d000d, 0x66660066, 0xcccc00cc, 0x2d2d002d, + 0x12120012, 0x20200020, 0xb1b100b1, 0x99990099, + 0x4c4c004c, 0xc2c200c2, 0x7e7e007e, 0x05050005, + 0xb7b700b7, 0x31310031, 0x17170017, 0xd7d700d7, + 0x58580058, 0x61610061, 0x1b1b001b, 0x1c1c001c, + 0x0f0f000f, 0x16160016, 0x18180018, 0x22220022, + 0x44440044, 0xb2b200b2, 0xb5b500b5, 0x91910091, + 0x08080008, 0xa8a800a8, 0xfcfc00fc, 0x50500050, + 0xd0d000d0, 0x7d7d007d, 0x89890089, 0x97970097, + 0x5b5b005b, 0x95950095, 0xffff00ff, 0xd2d200d2, + 0xc4c400c4, 0x48480048, 0xf7f700f7, 0xdbdb00db, + 0x03030003, 0xdada00da, 0x3f3f003f, 0x94940094, + 0x5c5c005c, 0x02020002, 0x4a4a004a, 0x33330033, + 0x67670067, 0xf3f300f3, 0x7f7f007f, 0xe2e200e2, + 0x9b9b009b, 0x26260026, 0x37370037, 0x3b3b003b, + 0x96960096, 0x4b4b004b, 0xbebe00be, 0x2e2e002e, + 0x79790079, 0x8c8c008c, 0x6e6e006e, 0x8e8e008e, + 0xf5f500f5, 0xb6b600b6, 0xfdfd00fd, 0x59590059, + 0x98980098, 0x6a6a006a, 0x46460046, 0xbaba00ba, + 0x25250025, 0x42420042, 0xa2a200a2, 0xfafa00fa, + 0x07070007, 0x55550055, 0xeeee00ee, 0x0a0a000a, + 0x49490049, 0x68680068, 0x38380038, 0xa4a400a4, + 0x28280028, 0x7b7b007b, 0xc9c900c9, 0xc1c100c1, + 0xe3e300e3, 0xf4f400f4, 0xc7c700c7, 0x9e9e009e, }; @@ -344,7 +344,7 @@ static const u32 camellia_sp4404[256] = { lr = (lr << bits) + (rl >> (32 - bits)); \ rl = (rl << bits) + (rr >> (32 - bits)); \ rr = (rr << bits) + (w0 >> (32 - bits)); \ - } while(0) + } while (0) #define ROLDQo32(ll, lr, rl, rr, w0, w1, bits) \ do { \ @@ -354,7 +354,7 @@ static const u32 camellia_sp4404[256] = { lr = (rl << (bits - 32)) + (rr >> (64 - bits)); \ rl = (rr << (bits - 32)) + (w0 >> (64 - bits)); \ rr = (w0 << (bits - 32)) + (w1 >> (64 - bits)); \ - } while(0) + } while (0) #define CAMELLIA_F(xl, xr, kl, kr, yl, yr, il, ir, t0, t1) \ do { \ @@ -373,7 +373,7 @@ static const u32 camellia_sp4404[256] = { yl ^= yr; \ yr = ror32(yr, 8); \ yr ^= yl; \ - } while(0) + } while (0) #define SUBKEY_L(INDEX) (subkey[(INDEX)*2]) #define SUBKEY_R(INDEX) (subkey[(INDEX)*2 + 1]) @@ -835,7 +835,7 @@ static void camellia_setup256(const unsigned char *key, u32 *subkey) static void camellia_setup192(const unsigned char *key, u32 *subkey) { unsigned char kk[32]; - u32 krll, krlr, krrl,krrr; + u32 krll, krlr, krrl, krrr; memcpy(kk, key, 24); memcpy((unsigned char *)&krll, key+16, 4); @@ -865,7 +865,7 @@ static void camellia_setup192(const unsigned char *key, u32 *subkey) t1 |= lr; \ ll ^= t1; \ rr ^= rol32(t3, 1); \ - } while(0) + } while (0) #define CAMELLIA_ROUNDSM(xl, xr, kl, kr, yl, yr, il, ir) \ do { \ @@ -881,12 +881,12 @@ static void camellia_setup192(const unsigned char *key, u32 *subkey) ir ^= il ^ kr; \ yl ^= ir; \ yr ^= ror32(il, 8) ^ ir; \ - } while(0) + } while (0) /* max = 24: 128bit encrypt, max = 32: 256bit encrypt */ static void camellia_do_encrypt(const u32 *subkey, u32 *io, unsigned max) { - u32 il,ir,t0,t1; /* temporary variables */ + u32 il, ir, t0, t1; /* temporary variables */ /* pre whitening but absorb kw2 */ io[0] ^= SUBKEY_L(0); @@ -894,30 +894,30 @@ static void camellia_do_encrypt(const u32 *subkey, u32 *io, unsigned max) /* main iteration */ #define ROUNDS(i) do { \ - CAMELLIA_ROUNDSM(io[0],io[1], \ - SUBKEY_L(i + 2),SUBKEY_R(i + 2), \ - io[2],io[3],il,ir); \ - CAMELLIA_ROUNDSM(io[2],io[3], \ - SUBKEY_L(i + 3),SUBKEY_R(i + 3), \ - io[0],io[1],il,ir); \ - CAMELLIA_ROUNDSM(io[0],io[1], \ - SUBKEY_L(i + 4),SUBKEY_R(i + 4), \ - io[2],io[3],il,ir); \ - CAMELLIA_ROUNDSM(io[2],io[3], \ - SUBKEY_L(i + 5),SUBKEY_R(i + 5), \ - io[0],io[1],il,ir); \ - CAMELLIA_ROUNDSM(io[0],io[1], \ - SUBKEY_L(i + 6),SUBKEY_R(i + 6), \ - io[2],io[3],il,ir); \ - CAMELLIA_ROUNDSM(io[2],io[3], \ - SUBKEY_L(i + 7),SUBKEY_R(i + 7), \ - io[0],io[1],il,ir); \ + CAMELLIA_ROUNDSM(io[0], io[1], \ + SUBKEY_L(i + 2), SUBKEY_R(i + 2), \ + io[2], io[3], il, ir); \ + CAMELLIA_ROUNDSM(io[2], io[3], \ + SUBKEY_L(i + 3), SUBKEY_R(i + 3), \ + io[0], io[1], il, ir); \ + CAMELLIA_ROUNDSM(io[0], io[1], \ + SUBKEY_L(i + 4), SUBKEY_R(i + 4), \ + io[2], io[3], il, ir); \ + CAMELLIA_ROUNDSM(io[2], io[3], \ + SUBKEY_L(i + 5), SUBKEY_R(i + 5), \ + io[0], io[1], il, ir); \ + CAMELLIA_ROUNDSM(io[0], io[1], \ + SUBKEY_L(i + 6), SUBKEY_R(i + 6), \ + io[2], io[3], il, ir); \ + CAMELLIA_ROUNDSM(io[2], io[3], \ + SUBKEY_L(i + 7), SUBKEY_R(i + 7), \ + io[0], io[1], il, ir); \ } while (0) #define FLS(i) do { \ - CAMELLIA_FLS(io[0],io[1],io[2],io[3], \ - SUBKEY_L(i + 0),SUBKEY_R(i + 0), \ - SUBKEY_L(i + 1),SUBKEY_R(i + 1), \ - t0,t1,il,ir); \ + CAMELLIA_FLS(io[0], io[1], io[2], io[3], \ + SUBKEY_L(i + 0), SUBKEY_R(i + 0), \ + SUBKEY_L(i + 1), SUBKEY_R(i + 1), \ + t0, t1, il, ir); \ } while (0) ROUNDS(0); @@ -941,7 +941,7 @@ static void camellia_do_encrypt(const u32 *subkey, u32 *io, unsigned max) static void camellia_do_decrypt(const u32 *subkey, u32 *io, unsigned i) { - u32 il,ir,t0,t1; /* temporary variables */ + u32 il, ir, t0, t1; /* temporary variables */ /* pre whitening but absorb kw2 */ io[0] ^= SUBKEY_L(i); @@ -949,30 +949,30 @@ static void camellia_do_decrypt(const u32 *subkey, u32 *io, unsigned i) /* main iteration */ #define ROUNDS(i) do { \ - CAMELLIA_ROUNDSM(io[0],io[1], \ - SUBKEY_L(i + 7),SUBKEY_R(i + 7), \ - io[2],io[3],il,ir); \ - CAMELLIA_ROUNDSM(io[2],io[3], \ - SUBKEY_L(i + 6),SUBKEY_R(i + 6), \ - io[0],io[1],il,ir); \ - CAMELLIA_ROUNDSM(io[0],io[1], \ - SUBKEY_L(i + 5),SUBKEY_R(i + 5), \ - io[2],io[3],il,ir); \ - CAMELLIA_ROUNDSM(io[2],io[3], \ - SUBKEY_L(i + 4),SUBKEY_R(i + 4), \ - io[0],io[1],il,ir); \ - CAMELLIA_ROUNDSM(io[0],io[1], \ - SUBKEY_L(i + 3),SUBKEY_R(i + 3), \ - io[2],io[3],il,ir); \ - CAMELLIA_ROUNDSM(io[2],io[3], \ - SUBKEY_L(i + 2),SUBKEY_R(i + 2), \ - io[0],io[1],il,ir); \ + CAMELLIA_ROUNDSM(io[0], io[1], \ + SUBKEY_L(i + 7), SUBKEY_R(i + 7), \ + io[2], io[3], il, ir); \ + CAMELLIA_ROUNDSM(io[2], io[3], \ + SUBKEY_L(i + 6), SUBKEY_R(i + 6), \ + io[0], io[1], il, ir); \ + CAMELLIA_ROUNDSM(io[0], io[1], \ + SUBKEY_L(i + 5), SUBKEY_R(i + 5), \ + io[2], io[3], il, ir); \ + CAMELLIA_ROUNDSM(io[2], io[3], \ + SUBKEY_L(i + 4), SUBKEY_R(i + 4), \ + io[0], io[1], il, ir); \ + CAMELLIA_ROUNDSM(io[0], io[1], \ + SUBKEY_L(i + 3), SUBKEY_R(i + 3), \ + io[2], io[3], il, ir); \ + CAMELLIA_ROUNDSM(io[2], io[3], \ + SUBKEY_L(i + 2), SUBKEY_R(i + 2), \ + io[0], io[1], il, ir); \ } while (0) #define FLS(i) do { \ - CAMELLIA_FLS(io[0],io[1],io[2],io[3], \ - SUBKEY_L(i + 1),SUBKEY_R(i + 1), \ - SUBKEY_L(i + 0),SUBKEY_R(i + 0), \ - t0,t1,il,ir); \ + CAMELLIA_FLS(io[0], io[1], io[2], io[3], \ + SUBKEY_L(i + 1), SUBKEY_R(i + 1), \ + SUBKEY_L(i + 0), SUBKEY_R(i + 0), \ + t0, t1, il, ir); \ } while (0) if (i == 32) { diff --git a/crypto/cast5.c b/crypto/cast5.c index 8cbe28fa0e0c..a1d2294b50ad 100644 --- a/crypto/cast5.c +++ b/crypto/cast5.c @@ -569,12 +569,12 @@ static const u32 sb8[256] = { 0xeaee6801, 0x8db2a283, 0xea8bf59e }; -#define F1(D,m,r) ( (I = ((m) + (D))), (I=rol32(I,(r))), \ - (((s1[I >> 24] ^ s2[(I>>16)&0xff]) - s3[(I>>8)&0xff]) + s4[I&0xff]) ) -#define F2(D,m,r) ( (I = ((m) ^ (D))), (I=rol32(I,(r))), \ - (((s1[I >> 24] - s2[(I>>16)&0xff]) + s3[(I>>8)&0xff]) ^ s4[I&0xff]) ) -#define F3(D,m,r) ( (I = ((m) - (D))), (I=rol32(I,(r))), \ - (((s1[I >> 24] + s2[(I>>16)&0xff]) ^ s3[(I>>8)&0xff]) - s4[I&0xff]) ) +#define F1(D, m, r) ((I = ((m) + (D))), (I = rol32(I, (r))), \ + (((s1[I >> 24] ^ s2[(I>>16)&0xff]) - s3[(I>>8)&0xff]) + s4[I&0xff])) +#define F2(D, m, r) ((I = ((m) ^ (D))), (I = rol32(I, (r))), \ + (((s1[I >> 24] - s2[(I>>16)&0xff]) + s3[(I>>8)&0xff]) ^ s4[I&0xff])) +#define F3(D, m, r) ((I = ((m) - (D))), (I = rol32(I, (r))), \ + (((s1[I >> 24] + s2[(I>>16)&0xff]) ^ s3[(I>>8)&0xff]) - s4[I&0xff])) static void cast5_encrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf) @@ -694,7 +694,7 @@ static void cast5_decrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf) dst[1] = cpu_to_be32(l); } -static void key_schedule(u32 * x, u32 * z, u32 * k) +static void key_schedule(u32 *x, u32 *z, u32 *k) { #define xi(i) ((x[(i)/4] >> (8*(3-((i)%4)))) & 0xff) diff --git a/crypto/cast6.c b/crypto/cast6.c index 007d02beed67..e0c15a6c7c34 100644 --- a/crypto/cast6.c +++ b/crypto/cast6.c @@ -11,7 +11,7 @@ * under the terms of GNU General Public License as published by the Free * Software Foundation; either version 2 of the License, or (at your option) * any later version. - * + * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA @@ -35,12 +35,12 @@ struct cast6_ctx { u8 Kr[12][4]; }; -#define F1(D,r,m) ( (I = ((m) + (D))), (I=rol32(I,(r))), \ - (((s1[I >> 24] ^ s2[(I>>16)&0xff]) - s3[(I>>8)&0xff]) + s4[I&0xff]) ) -#define F2(D,r,m) ( (I = ((m) ^ (D))), (I=rol32(I,(r))), \ - (((s1[I >> 24] - s2[(I>>16)&0xff]) + s3[(I>>8)&0xff]) ^ s4[I&0xff]) ) -#define F3(D,r,m) ( (I = ((m) - (D))), (I=rol32(I,(r))), \ - (((s1[I >> 24] + s2[(I>>16)&0xff]) ^ s3[(I>>8)&0xff]) - s4[I&0xff]) ) +#define F1(D, r, m) ((I = ((m) + (D))), (I = rol32(I, (r))), \ + (((s1[I >> 24] ^ s2[(I>>16)&0xff]) - s3[(I>>8)&0xff]) + s4[I&0xff])) +#define F2(D, r, m) ((I = ((m) ^ (D))), (I = rol32(I, (r))), \ + (((s1[I >> 24] - s2[(I>>16)&0xff]) + s3[(I>>8)&0xff]) ^ s4[I&0xff])) +#define F3(D, r, m) ((I = ((m) - (D))), (I = rol32(I, (r))), \ + (((s1[I >> 24] + s2[(I>>16)&0xff]) ^ s3[(I>>8)&0xff]) - s4[I&0xff])) static const u32 s1[256] = { 0x30fb40d4, 0x9fa0ff0b, 0x6beccd2f, 0x3f258c7a, 0x1e213f2f, @@ -312,7 +312,7 @@ static const u32 s4[256] = { static const u32 Tm[24][8] = { { 0x5a827999, 0xc95c653a, 0x383650db, 0xa7103c7c, 0x15ea281d, - 0x84c413be, 0xf39dff5f, 0x6277eb00 } , + 0x84c413be, 0xf39dff5f, 0x6277eb00 } , { 0xd151d6a1, 0x402bc242, 0xaf05ade3, 0x1ddf9984, 0x8cb98525, 0xfb9370c6, 0x6a6d5c67, 0xd9474808 } , { 0x482133a9, 0xb6fb1f4a, 0x25d50aeb, 0x94aef68c, 0x0388e22d, @@ -369,7 +369,8 @@ static const u8 Tr[4][8] = { }; /* forward octave */ -static void W(u32 *key, unsigned int i) { +static void W(u32 *key, unsigned int i) +{ u32 I; key[6] ^= F1(key[7], Tr[i % 4][0], Tm[i][0]); key[5] ^= F2(key[6], Tr[i % 4][1], Tm[i][1]); @@ -377,7 +378,7 @@ static void W(u32 *key, unsigned int i) { key[3] ^= F1(key[4], Tr[i % 4][3], Tm[i][3]); key[2] ^= F2(key[3], Tr[i % 4][4], Tm[i][4]); key[1] ^= F3(key[2], Tr[i % 4][5], Tm[i][5]); - key[0] ^= F1(key[1], Tr[i % 4][6], Tm[i][6]); + key[0] ^= F1(key[1], Tr[i % 4][6], Tm[i][6]); key[7] ^= F2(key[0], Tr[i % 4][7], Tm[i][7]); } @@ -393,11 +394,11 @@ static int cast6_setkey(struct crypto_tfm *tfm, const u8 *in_key, if (key_len % 4 != 0) { *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; return -EINVAL; - } + } + + memset(p_key, 0, 32); + memcpy(p_key, in_key, key_len); - memset (p_key, 0, 32); - memcpy (p_key, in_key, key_len); - key[0] = be32_to_cpu(p_key[0]); /* A */ key[1] = be32_to_cpu(p_key[1]); /* B */ key[2] = be32_to_cpu(p_key[2]); /* C */ @@ -406,18 +407,16 @@ static int cast6_setkey(struct crypto_tfm *tfm, const u8 *in_key, key[5] = be32_to_cpu(p_key[5]); /* F */ key[6] = be32_to_cpu(p_key[6]); /* G */ key[7] = be32_to_cpu(p_key[7]); /* H */ - - for (i = 0; i < 12; i++) { - W (key, 2 * i); - W (key, 2 * i + 1); - + W(key, 2 * i); + W(key, 2 * i + 1); + c->Kr[i][0] = key[0] & 0x1f; c->Kr[i][1] = key[2] & 0x1f; c->Kr[i][2] = key[4] & 0x1f; c->Kr[i][3] = key[6] & 0x1f; - + c->Km[i][0] = key[7]; c->Km[i][1] = key[5]; c->Km[i][2] = key[3]; @@ -428,21 +427,23 @@ static int cast6_setkey(struct crypto_tfm *tfm, const u8 *in_key, } /*forward quad round*/ -static void Q (u32 * block, u8 * Kr, u32 * Km) { +static void Q(u32 *block, u8 *Kr, u32 *Km) +{ u32 I; block[2] ^= F1(block[3], Kr[0], Km[0]); block[1] ^= F2(block[2], Kr[1], Km[1]); block[0] ^= F3(block[1], Kr[2], Km[2]); - block[3] ^= F1(block[0], Kr[3], Km[3]); + block[3] ^= F1(block[0], Kr[3], Km[3]); } /*reverse quad round*/ -static void QBAR (u32 * block, u8 * Kr, u32 * Km) { +static void QBAR(u32 *block, u8 *Kr, u32 *Km) +{ u32 I; - block[3] ^= F1(block[0], Kr[3], Km[3]); - block[0] ^= F3(block[1], Kr[2], Km[2]); - block[1] ^= F2(block[2], Kr[1], Km[1]); - block[2] ^= F1(block[3], Kr[0], Km[0]); + block[3] ^= F1(block[0], Kr[3], Km[3]); + block[0] ^= F3(block[1], Kr[2], Km[2]); + block[1] ^= F2(block[2], Kr[1], Km[1]); + block[2] ^= F1(block[3], Kr[0], Km[0]); } static void cast6_encrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf) @@ -451,64 +452,65 @@ static void cast6_encrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf) const __be32 *src = (const __be32 *)inbuf; __be32 *dst = (__be32 *)outbuf; u32 block[4]; - u32 * Km; - u8 * Kr; + u32 *Km; + u8 *Kr; block[0] = be32_to_cpu(src[0]); block[1] = be32_to_cpu(src[1]); block[2] = be32_to_cpu(src[2]); block[3] = be32_to_cpu(src[3]); - Km = c->Km[0]; Kr = c->Kr[0]; Q (block, Kr, Km); - Km = c->Km[1]; Kr = c->Kr[1]; Q (block, Kr, Km); - Km = c->Km[2]; Kr = c->Kr[2]; Q (block, Kr, Km); - Km = c->Km[3]; Kr = c->Kr[3]; Q (block, Kr, Km); - Km = c->Km[4]; Kr = c->Kr[4]; Q (block, Kr, Km); - Km = c->Km[5]; Kr = c->Kr[5]; Q (block, Kr, Km); - Km = c->Km[6]; Kr = c->Kr[6]; QBAR (block, Kr, Km); - Km = c->Km[7]; Kr = c->Kr[7]; QBAR (block, Kr, Km); - Km = c->Km[8]; Kr = c->Kr[8]; QBAR (block, Kr, Km); - Km = c->Km[9]; Kr = c->Kr[9]; QBAR (block, Kr, Km); - Km = c->Km[10]; Kr = c->Kr[10]; QBAR (block, Kr, Km); - Km = c->Km[11]; Kr = c->Kr[11]; QBAR (block, Kr, Km); + Km = c->Km[0]; Kr = c->Kr[0]; Q(block, Kr, Km); + Km = c->Km[1]; Kr = c->Kr[1]; Q(block, Kr, Km); + Km = c->Km[2]; Kr = c->Kr[2]; Q(block, Kr, Km); + Km = c->Km[3]; Kr = c->Kr[3]; Q(block, Kr, Km); + Km = c->Km[4]; Kr = c->Kr[4]; Q(block, Kr, Km); + Km = c->Km[5]; Kr = c->Kr[5]; Q(block, Kr, Km); + Km = c->Km[6]; Kr = c->Kr[6]; QBAR(block, Kr, Km); + Km = c->Km[7]; Kr = c->Kr[7]; QBAR(block, Kr, Km); + Km = c->Km[8]; Kr = c->Kr[8]; QBAR(block, Kr, Km); + Km = c->Km[9]; Kr = c->Kr[9]; QBAR(block, Kr, Km); + Km = c->Km[10]; Kr = c->Kr[10]; QBAR(block, Kr, Km); + Km = c->Km[11]; Kr = c->Kr[11]; QBAR(block, Kr, Km); dst[0] = cpu_to_be32(block[0]); dst[1] = cpu_to_be32(block[1]); dst[2] = cpu_to_be32(block[2]); dst[3] = cpu_to_be32(block[3]); -} +} -static void cast6_decrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf) { - struct cast6_ctx * c = crypto_tfm_ctx(tfm); +static void cast6_decrypt(struct crypto_tfm *tfm, u8 *outbuf, const u8 *inbuf) +{ + struct cast6_ctx *c = crypto_tfm_ctx(tfm); const __be32 *src = (const __be32 *)inbuf; __be32 *dst = (__be32 *)outbuf; u32 block[4]; - u32 * Km; - u8 * Kr; + u32 *Km; + u8 *Kr; block[0] = be32_to_cpu(src[0]); block[1] = be32_to_cpu(src[1]); block[2] = be32_to_cpu(src[2]); block[3] = be32_to_cpu(src[3]); - Km = c->Km[11]; Kr = c->Kr[11]; Q (block, Kr, Km); - Km = c->Km[10]; Kr = c->Kr[10]; Q (block, Kr, Km); - Km = c->Km[9]; Kr = c->Kr[9]; Q (block, Kr, Km); - Km = c->Km[8]; Kr = c->Kr[8]; Q (block, Kr, Km); - Km = c->Km[7]; Kr = c->Kr[7]; Q (block, Kr, Km); - Km = c->Km[6]; Kr = c->Kr[6]; Q (block, Kr, Km); - Km = c->Km[5]; Kr = c->Kr[5]; QBAR (block, Kr, Km); - Km = c->Km[4]; Kr = c->Kr[4]; QBAR (block, Kr, Km); - Km = c->Km[3]; Kr = c->Kr[3]; QBAR (block, Kr, Km); - Km = c->Km[2]; Kr = c->Kr[2]; QBAR (block, Kr, Km); - Km = c->Km[1]; Kr = c->Kr[1]; QBAR (block, Kr, Km); - Km = c->Km[0]; Kr = c->Kr[0]; QBAR (block, Kr, Km); - + Km = c->Km[11]; Kr = c->Kr[11]; Q(block, Kr, Km); + Km = c->Km[10]; Kr = c->Kr[10]; Q(block, Kr, Km); + Km = c->Km[9]; Kr = c->Kr[9]; Q(block, Kr, Km); + Km = c->Km[8]; Kr = c->Kr[8]; Q(block, Kr, Km); + Km = c->Km[7]; Kr = c->Kr[7]; Q(block, Kr, Km); + Km = c->Km[6]; Kr = c->Kr[6]; Q(block, Kr, Km); + Km = c->Km[5]; Kr = c->Kr[5]; QBAR(block, Kr, Km); + Km = c->Km[4]; Kr = c->Kr[4]; QBAR(block, Kr, Km); + Km = c->Km[3]; Kr = c->Kr[3]; QBAR(block, Kr, Km); + Km = c->Km[2]; Kr = c->Kr[2]; QBAR(block, Kr, Km); + Km = c->Km[1]; Kr = c->Kr[1]; QBAR(block, Kr, Km); + Km = c->Km[0]; Kr = c->Kr[0]; QBAR(block, Kr, Km); + dst[0] = cpu_to_be32(block[0]); dst[1] = cpu_to_be32(block[1]); dst[2] = cpu_to_be32(block[2]); dst[3] = cpu_to_be32(block[3]); -} +} static struct crypto_alg alg = { .cra_name = "cast6", diff --git a/crypto/cipher.c b/crypto/cipher.c index 9a1a7316eeac..39541e0e537d 100644 --- a/crypto/cipher.c +++ b/crypto/cipher.c @@ -8,7 +8,7 @@ * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) + * Software Foundation; either version 2 of the License, or (at your option) * any later version. * */ diff --git a/crypto/compress.c b/crypto/compress.c index 1ee357085d3a..c33f0763a956 100644 --- a/crypto/compress.c +++ b/crypto/compress.c @@ -7,7 +7,7 @@ * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) + * Software Foundation; either version 2 of the License, or (at your option) * any later version. * */ @@ -39,7 +39,7 @@ int crypto_init_compress_ops(struct crypto_tfm *tfm) ops->cot_compress = crypto_compress; ops->cot_decompress = crypto_decompress; - + return 0; } diff --git a/crypto/crc32c.c b/crypto/crc32c.c index 973bc2cfab2e..de9e55c29794 100644 --- a/crypto/crc32c.c +++ b/crypto/crc32c.c @@ -1,4 +1,4 @@ -/* +/* * Cryptographic API. * * CRC32C chksum @@ -30,7 +30,7 @@ * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) + * Software Foundation; either version 2 of the License, or (at your option) * any later version. * */ @@ -142,7 +142,7 @@ static u32 crc32c(u32 crc, const u8 *data, unsigned int length) } /* - * Steps through buffer one byte at at time, calculates reflected + * Steps through buffer one byte at at time, calculates reflected * crc using table. */ diff --git a/crypto/cryptd.c b/crypto/cryptd.c index ae5fa99d5d36..e46d21ae26bc 100644 --- a/crypto/cryptd.c +++ b/crypto/cryptd.c @@ -3,6 +3,13 @@ * * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> * + * Added AEAD support to cryptd. + * Authors: Tadeusz Struk (tadeusz.struk@intel.com) + * Adrian Hoban <adrian.hoban@intel.com> + * Gabriele Paoloni <gabriele.paoloni@intel.com> + * Aidan O'Mahony (aidan.o.mahony@intel.com) + * Copyright (c) 2010, Intel Corporation. + * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation; either version 2 of the License, or (at your option) @@ -12,6 +19,7 @@ #include <crypto/algapi.h> #include <crypto/internal/hash.h> +#include <crypto/internal/aead.h> #include <crypto/cryptd.h> #include <crypto/crypto_wq.h> #include <linux/err.h> @@ -31,7 +39,7 @@ struct cryptd_cpu_queue { }; struct cryptd_queue { - struct cryptd_cpu_queue *cpu_queue; + struct cryptd_cpu_queue __percpu *cpu_queue; }; struct cryptd_instance_ctx { @@ -39,6 +47,16 @@ struct cryptd_instance_ctx { struct cryptd_queue *queue; }; +struct hashd_instance_ctx { + struct crypto_shash_spawn spawn; + struct cryptd_queue *queue; +}; + +struct aead_instance_ctx { + struct crypto_aead_spawn aead_spawn; + struct cryptd_queue *queue; +}; + struct cryptd_blkcipher_ctx { struct crypto_blkcipher *child; }; @@ -48,11 +66,20 @@ struct cryptd_blkcipher_request_ctx { }; struct cryptd_hash_ctx { - struct crypto_hash *child; + struct crypto_shash *child; }; struct cryptd_hash_request_ctx { crypto_completion_t complete; + struct shash_desc desc; +}; + +struct cryptd_aead_ctx { + struct crypto_aead *child; +}; + +struct cryptd_aead_request_ctx { + crypto_completion_t complete; }; static void cryptd_queue_worker(struct work_struct *work); @@ -93,7 +120,7 @@ static int cryptd_enqueue_request(struct cryptd_queue *queue, struct cryptd_cpu_queue *cpu_queue; cpu = get_cpu(); - cpu_queue = per_cpu_ptr(queue->cpu_queue, cpu); + cpu_queue = this_cpu_ptr(queue->cpu_queue); err = crypto_enqueue_request(&cpu_queue->queue, request); queue_work_on(cpu, kcrypto_wq, &cpu_queue->work); put_cpu(); @@ -249,32 +276,24 @@ static void cryptd_blkcipher_exit_tfm(struct crypto_tfm *tfm) crypto_free_blkcipher(ctx->child); } -static struct crypto_instance *cryptd_alloc_instance(struct crypto_alg *alg, - struct cryptd_queue *queue) +static void *cryptd_alloc_instance(struct crypto_alg *alg, unsigned int head, + unsigned int tail) { + char *p; struct crypto_instance *inst; - struct cryptd_instance_ctx *ctx; int err; - inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); - if (!inst) { - inst = ERR_PTR(-ENOMEM); - goto out; - } + p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL); + if (!p) + return ERR_PTR(-ENOMEM); + + inst = (void *)(p + head); err = -ENAMETOOLONG; if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "cryptd(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto out_free_inst; - ctx = crypto_instance_ctx(inst); - err = crypto_init_spawn(&ctx->spawn, alg, inst, - CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC); - if (err) - goto out_free_inst; - - ctx->queue = queue; - memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); inst->alg.cra_priority = alg->cra_priority + 50; @@ -282,29 +301,41 @@ static struct crypto_instance *cryptd_alloc_instance(struct crypto_alg *alg, inst->alg.cra_alignmask = alg->cra_alignmask; out: - return inst; + return p; out_free_inst: - kfree(inst); - inst = ERR_PTR(err); + kfree(p); + p = ERR_PTR(err); goto out; } -static struct crypto_instance *cryptd_alloc_blkcipher( - struct rtattr **tb, struct cryptd_queue *queue) +static int cryptd_create_blkcipher(struct crypto_template *tmpl, + struct rtattr **tb, + struct cryptd_queue *queue) { + struct cryptd_instance_ctx *ctx; struct crypto_instance *inst; struct crypto_alg *alg; + int err; alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_BLKCIPHER, CRYPTO_ALG_TYPE_MASK); if (IS_ERR(alg)) - return ERR_CAST(alg); + return PTR_ERR(alg); - inst = cryptd_alloc_instance(alg, queue); + inst = cryptd_alloc_instance(alg, 0, sizeof(*ctx)); + err = PTR_ERR(inst); if (IS_ERR(inst)) goto out_put_alg; + ctx = crypto_instance_ctx(inst); + ctx->queue = queue; + + err = crypto_init_spawn(&ctx->spawn, alg, inst, + CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC); + if (err) + goto out_free_inst; + inst->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC; inst->alg.cra_type = &crypto_ablkcipher_type; @@ -323,26 +354,34 @@ static struct crypto_instance *cryptd_alloc_blkcipher( inst->alg.cra_ablkcipher.encrypt = cryptd_blkcipher_encrypt_enqueue; inst->alg.cra_ablkcipher.decrypt = cryptd_blkcipher_decrypt_enqueue; + err = crypto_register_instance(tmpl, inst); + if (err) { + crypto_drop_spawn(&ctx->spawn); +out_free_inst: + kfree(inst); + } + out_put_alg: crypto_mod_put(alg); - return inst; + return err; } static int cryptd_hash_init_tfm(struct crypto_tfm *tfm) { struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); - struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst); - struct crypto_spawn *spawn = &ictx->spawn; + struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst); + struct crypto_shash_spawn *spawn = &ictx->spawn; struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); - struct crypto_hash *cipher; + struct crypto_shash *hash; - cipher = crypto_spawn_hash(spawn); - if (IS_ERR(cipher)) - return PTR_ERR(cipher); + hash = crypto_spawn_shash(spawn); + if (IS_ERR(hash)) + return PTR_ERR(hash); - ctx->child = cipher; - tfm->crt_ahash.reqsize = - sizeof(struct cryptd_hash_request_ctx); + ctx->child = hash; + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct cryptd_hash_request_ctx) + + crypto_shash_descsize(hash)); return 0; } @@ -350,22 +389,22 @@ static void cryptd_hash_exit_tfm(struct crypto_tfm *tfm) { struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); - crypto_free_hash(ctx->child); + crypto_free_shash(ctx->child); } static int cryptd_hash_setkey(struct crypto_ahash *parent, const u8 *key, unsigned int keylen) { struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(parent); - struct crypto_hash *child = ctx->child; + struct crypto_shash *child = ctx->child; int err; - crypto_hash_clear_flags(child, CRYPTO_TFM_REQ_MASK); - crypto_hash_set_flags(child, crypto_ahash_get_flags(parent) & - CRYPTO_TFM_REQ_MASK); - err = crypto_hash_setkey(child, key, keylen); - crypto_ahash_set_flags(parent, crypto_hash_get_flags(child) & - CRYPTO_TFM_RES_MASK); + crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK); + crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) & + CRYPTO_TFM_REQ_MASK); + err = crypto_shash_setkey(child, key, keylen); + crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) & + CRYPTO_TFM_RES_MASK); return err; } @@ -385,21 +424,19 @@ static int cryptd_hash_enqueue(struct ahash_request *req, static void cryptd_hash_init(struct crypto_async_request *req_async, int err) { - struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); - struct crypto_hash *child = ctx->child; - struct ahash_request *req = ahash_request_cast(req_async); - struct cryptd_hash_request_ctx *rctx; - struct hash_desc desc; - - rctx = ahash_request_ctx(req); + struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); + struct crypto_shash *child = ctx->child; + struct ahash_request *req = ahash_request_cast(req_async); + struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); + struct shash_desc *desc = &rctx->desc; if (unlikely(err == -EINPROGRESS)) goto out; - desc.tfm = child; - desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; + desc->tfm = child; + desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; - err = crypto_hash_crt(child)->init(&desc); + err = crypto_shash_init(desc); req->base.complete = rctx->complete; @@ -416,23 +453,15 @@ static int cryptd_hash_init_enqueue(struct ahash_request *req) static void cryptd_hash_update(struct crypto_async_request *req_async, int err) { - struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); - struct crypto_hash *child = ctx->child; - struct ahash_request *req = ahash_request_cast(req_async); + struct ahash_request *req = ahash_request_cast(req_async); struct cryptd_hash_request_ctx *rctx; - struct hash_desc desc; rctx = ahash_request_ctx(req); if (unlikely(err == -EINPROGRESS)) goto out; - desc.tfm = child; - desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; - - err = crypto_hash_crt(child)->update(&desc, - req->src, - req->nbytes); + err = shash_ahash_update(req, &rctx->desc); req->base.complete = rctx->complete; @@ -449,21 +478,13 @@ static int cryptd_hash_update_enqueue(struct ahash_request *req) static void cryptd_hash_final(struct crypto_async_request *req_async, int err) { - struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); - struct crypto_hash *child = ctx->child; - struct ahash_request *req = ahash_request_cast(req_async); - struct cryptd_hash_request_ctx *rctx; - struct hash_desc desc; - - rctx = ahash_request_ctx(req); + struct ahash_request *req = ahash_request_cast(req_async); + struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); if (unlikely(err == -EINPROGRESS)) goto out; - desc.tfm = child; - desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; - - err = crypto_hash_crt(child)->final(&desc, req->result); + err = crypto_shash_final(&rctx->desc, req->result); req->base.complete = rctx->complete; @@ -478,26 +499,44 @@ static int cryptd_hash_final_enqueue(struct ahash_request *req) return cryptd_hash_enqueue(req, cryptd_hash_final); } -static void cryptd_hash_digest(struct crypto_async_request *req_async, int err) +static void cryptd_hash_finup(struct crypto_async_request *req_async, int err) { - struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); - struct crypto_hash *child = ctx->child; - struct ahash_request *req = ahash_request_cast(req_async); - struct cryptd_hash_request_ctx *rctx; - struct hash_desc desc; + struct ahash_request *req = ahash_request_cast(req_async); + struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); - rctx = ahash_request_ctx(req); + if (unlikely(err == -EINPROGRESS)) + goto out; + + err = shash_ahash_finup(req, &rctx->desc); + + req->base.complete = rctx->complete; + +out: + local_bh_disable(); + rctx->complete(&req->base, err); + local_bh_enable(); +} + +static int cryptd_hash_finup_enqueue(struct ahash_request *req) +{ + return cryptd_hash_enqueue(req, cryptd_hash_finup); +} + +static void cryptd_hash_digest(struct crypto_async_request *req_async, int err) +{ + struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); + struct crypto_shash *child = ctx->child; + struct ahash_request *req = ahash_request_cast(req_async); + struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); + struct shash_desc *desc = &rctx->desc; if (unlikely(err == -EINPROGRESS)) goto out; - desc.tfm = child; - desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; + desc->tfm = child; + desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; - err = crypto_hash_crt(child)->digest(&desc, - req->src, - req->nbytes, - req->result); + err = shash_ahash_digest(req, desc); req->base.complete = rctx->complete; @@ -512,72 +551,261 @@ static int cryptd_hash_digest_enqueue(struct ahash_request *req) return cryptd_hash_enqueue(req, cryptd_hash_digest); } -static struct crypto_instance *cryptd_alloc_hash( - struct rtattr **tb, struct cryptd_queue *queue) +static int cryptd_hash_export(struct ahash_request *req, void *out) { - struct crypto_instance *inst; + struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); + + return crypto_shash_export(&rctx->desc, out); +} + +static int cryptd_hash_import(struct ahash_request *req, const void *in) +{ + struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); + + return crypto_shash_import(&rctx->desc, in); +} + +static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, + struct cryptd_queue *queue) +{ + struct hashd_instance_ctx *ctx; + struct ahash_instance *inst; + struct shash_alg *salg; struct crypto_alg *alg; + int err; - alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_HASH, - CRYPTO_ALG_TYPE_HASH_MASK); - if (IS_ERR(alg)) - return ERR_PTR(PTR_ERR(alg)); + salg = shash_attr_alg(tb[1], 0, 0); + if (IS_ERR(salg)) + return PTR_ERR(salg); - inst = cryptd_alloc_instance(alg, queue); + alg = &salg->base; + inst = cryptd_alloc_instance(alg, ahash_instance_headroom(), + sizeof(*ctx)); + err = PTR_ERR(inst); if (IS_ERR(inst)) goto out_put_alg; - inst->alg.cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC; - inst->alg.cra_type = &crypto_ahash_type; + ctx = ahash_instance_ctx(inst); + ctx->queue = queue; + + err = crypto_init_shash_spawn(&ctx->spawn, salg, + ahash_crypto_instance(inst)); + if (err) + goto out_free_inst; + + inst->alg.halg.base.cra_flags = CRYPTO_ALG_ASYNC; + + inst->alg.halg.digestsize = salg->digestsize; + inst->alg.halg.base.cra_ctxsize = sizeof(struct cryptd_hash_ctx); + + inst->alg.halg.base.cra_init = cryptd_hash_init_tfm; + inst->alg.halg.base.cra_exit = cryptd_hash_exit_tfm; + + inst->alg.init = cryptd_hash_init_enqueue; + inst->alg.update = cryptd_hash_update_enqueue; + inst->alg.final = cryptd_hash_final_enqueue; + inst->alg.finup = cryptd_hash_finup_enqueue; + inst->alg.export = cryptd_hash_export; + inst->alg.import = cryptd_hash_import; + inst->alg.setkey = cryptd_hash_setkey; + inst->alg.digest = cryptd_hash_digest_enqueue; + + err = ahash_register_instance(tmpl, inst); + if (err) { + crypto_drop_shash(&ctx->spawn); +out_free_inst: + kfree(inst); + } + +out_put_alg: + crypto_mod_put(alg); + return err; +} + +static void cryptd_aead_crypt(struct aead_request *req, + struct crypto_aead *child, + int err, + int (*crypt)(struct aead_request *req)) +{ + struct cryptd_aead_request_ctx *rctx; + rctx = aead_request_ctx(req); + + if (unlikely(err == -EINPROGRESS)) + goto out; + aead_request_set_tfm(req, child); + err = crypt( req ); + req->base.complete = rctx->complete; +out: + local_bh_disable(); + rctx->complete(&req->base, err); + local_bh_enable(); +} + +static void cryptd_aead_encrypt(struct crypto_async_request *areq, int err) +{ + struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm); + struct crypto_aead *child = ctx->child; + struct aead_request *req; + + req = container_of(areq, struct aead_request, base); + cryptd_aead_crypt(req, child, err, crypto_aead_crt(child)->encrypt); +} + +static void cryptd_aead_decrypt(struct crypto_async_request *areq, int err) +{ + struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(areq->tfm); + struct crypto_aead *child = ctx->child; + struct aead_request *req; + + req = container_of(areq, struct aead_request, base); + cryptd_aead_crypt(req, child, err, crypto_aead_crt(child)->decrypt); +} + +static int cryptd_aead_enqueue(struct aead_request *req, + crypto_completion_t complete) +{ + struct cryptd_aead_request_ctx *rctx = aead_request_ctx(req); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct cryptd_queue *queue = cryptd_get_queue(crypto_aead_tfm(tfm)); + + rctx->complete = req->base.complete; + req->base.complete = complete; + return cryptd_enqueue_request(queue, &req->base); +} + +static int cryptd_aead_encrypt_enqueue(struct aead_request *req) +{ + return cryptd_aead_enqueue(req, cryptd_aead_encrypt ); +} + +static int cryptd_aead_decrypt_enqueue(struct aead_request *req) +{ + return cryptd_aead_enqueue(req, cryptd_aead_decrypt ); +} + +static int cryptd_aead_init_tfm(struct crypto_tfm *tfm) +{ + struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); + struct aead_instance_ctx *ictx = crypto_instance_ctx(inst); + struct crypto_aead_spawn *spawn = &ictx->aead_spawn; + struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_aead *cipher; + + cipher = crypto_spawn_aead(spawn); + if (IS_ERR(cipher)) + return PTR_ERR(cipher); + + crypto_aead_set_flags(cipher, CRYPTO_TFM_REQ_MAY_SLEEP); + ctx->child = cipher; + tfm->crt_aead.reqsize = sizeof(struct cryptd_aead_request_ctx); + return 0; +} + +static void cryptd_aead_exit_tfm(struct crypto_tfm *tfm) +{ + struct cryptd_aead_ctx *ctx = crypto_tfm_ctx(tfm); + crypto_free_aead(ctx->child); +} + +static int cryptd_create_aead(struct crypto_template *tmpl, + struct rtattr **tb, + struct cryptd_queue *queue) +{ + struct aead_instance_ctx *ctx; + struct crypto_instance *inst; + struct crypto_alg *alg; + int err; + + alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_AEAD, + CRYPTO_ALG_TYPE_MASK); + if (IS_ERR(alg)) + return PTR_ERR(alg); - inst->alg.cra_ahash.digestsize = alg->cra_hash.digestsize; - inst->alg.cra_ctxsize = sizeof(struct cryptd_hash_ctx); + inst = cryptd_alloc_instance(alg, 0, sizeof(*ctx)); + err = PTR_ERR(inst); + if (IS_ERR(inst)) + goto out_put_alg; - inst->alg.cra_init = cryptd_hash_init_tfm; - inst->alg.cra_exit = cryptd_hash_exit_tfm; + ctx = crypto_instance_ctx(inst); + ctx->queue = queue; - inst->alg.cra_ahash.init = cryptd_hash_init_enqueue; - inst->alg.cra_ahash.update = cryptd_hash_update_enqueue; - inst->alg.cra_ahash.final = cryptd_hash_final_enqueue; - inst->alg.cra_ahash.setkey = cryptd_hash_setkey; - inst->alg.cra_ahash.digest = cryptd_hash_digest_enqueue; + err = crypto_init_spawn(&ctx->aead_spawn.base, alg, inst, + CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC); + if (err) + goto out_free_inst; + inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC; + inst->alg.cra_type = alg->cra_type; + inst->alg.cra_ctxsize = sizeof(struct cryptd_aead_ctx); + inst->alg.cra_init = cryptd_aead_init_tfm; + inst->alg.cra_exit = cryptd_aead_exit_tfm; + inst->alg.cra_aead.setkey = alg->cra_aead.setkey; + inst->alg.cra_aead.setauthsize = alg->cra_aead.setauthsize; + inst->alg.cra_aead.geniv = alg->cra_aead.geniv; + inst->alg.cra_aead.ivsize = alg->cra_aead.ivsize; + inst->alg.cra_aead.maxauthsize = alg->cra_aead.maxauthsize; + inst->alg.cra_aead.encrypt = cryptd_aead_encrypt_enqueue; + inst->alg.cra_aead.decrypt = cryptd_aead_decrypt_enqueue; + inst->alg.cra_aead.givencrypt = alg->cra_aead.givencrypt; + inst->alg.cra_aead.givdecrypt = alg->cra_aead.givdecrypt; + + err = crypto_register_instance(tmpl, inst); + if (err) { + crypto_drop_spawn(&ctx->aead_spawn.base); +out_free_inst: + kfree(inst); + } out_put_alg: crypto_mod_put(alg); - return inst; + return err; } static struct cryptd_queue queue; -static struct crypto_instance *cryptd_alloc(struct rtattr **tb) +static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb) { struct crypto_attr_type *algt; algt = crypto_get_attr_type(tb); if (IS_ERR(algt)) - return ERR_CAST(algt); + return PTR_ERR(algt); switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { case CRYPTO_ALG_TYPE_BLKCIPHER: - return cryptd_alloc_blkcipher(tb, &queue); + return cryptd_create_blkcipher(tmpl, tb, &queue); case CRYPTO_ALG_TYPE_DIGEST: - return cryptd_alloc_hash(tb, &queue); + return cryptd_create_hash(tmpl, tb, &queue); + case CRYPTO_ALG_TYPE_AEAD: + return cryptd_create_aead(tmpl, tb, &queue); } - return ERR_PTR(-EINVAL); + return -EINVAL; } static void cryptd_free(struct crypto_instance *inst) { struct cryptd_instance_ctx *ctx = crypto_instance_ctx(inst); + struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst); + struct aead_instance_ctx *aead_ctx = crypto_instance_ctx(inst); - crypto_drop_spawn(&ctx->spawn); - kfree(inst); + switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) { + case CRYPTO_ALG_TYPE_AHASH: + crypto_drop_shash(&hctx->spawn); + kfree(ahash_instance(inst)); + return; + case CRYPTO_ALG_TYPE_AEAD: + crypto_drop_spawn(&aead_ctx->aead_spawn.base); + kfree(inst); + return; + default: + crypto_drop_spawn(&ctx->spawn); + kfree(inst); + } } static struct crypto_template cryptd_tmpl = { .name = "cryptd", - .alloc = cryptd_alloc, + .create = cryptd_create, .free = cryptd_free, .module = THIS_MODULE, }; @@ -620,6 +848,82 @@ void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm) } EXPORT_SYMBOL_GPL(cryptd_free_ablkcipher); +struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name, + u32 type, u32 mask) +{ + char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; + struct crypto_ahash *tfm; + + if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, + "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) + return ERR_PTR(-EINVAL); + tfm = crypto_alloc_ahash(cryptd_alg_name, type, mask); + if (IS_ERR(tfm)) + return ERR_CAST(tfm); + if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { + crypto_free_ahash(tfm); + return ERR_PTR(-EINVAL); + } + + return __cryptd_ahash_cast(tfm); +} +EXPORT_SYMBOL_GPL(cryptd_alloc_ahash); + +struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm) +{ + struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); + + return ctx->child; +} +EXPORT_SYMBOL_GPL(cryptd_ahash_child); + +struct shash_desc *cryptd_shash_desc(struct ahash_request *req) +{ + struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); + return &rctx->desc; +} +EXPORT_SYMBOL_GPL(cryptd_shash_desc); + +void cryptd_free_ahash(struct cryptd_ahash *tfm) +{ + crypto_free_ahash(&tfm->base); +} +EXPORT_SYMBOL_GPL(cryptd_free_ahash); + +struct cryptd_aead *cryptd_alloc_aead(const char *alg_name, + u32 type, u32 mask) +{ + char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; + struct crypto_aead *tfm; + + if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, + "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) + return ERR_PTR(-EINVAL); + tfm = crypto_alloc_aead(cryptd_alg_name, type, mask); + if (IS_ERR(tfm)) + return ERR_CAST(tfm); + if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { + crypto_free_aead(tfm); + return ERR_PTR(-EINVAL); + } + return __cryptd_aead_cast(tfm); +} +EXPORT_SYMBOL_GPL(cryptd_alloc_aead); + +struct crypto_aead *cryptd_aead_child(struct cryptd_aead *tfm) +{ + struct cryptd_aead_ctx *ctx; + ctx = crypto_aead_ctx(&tfm->base); + return ctx->child; +} +EXPORT_SYMBOL_GPL(cryptd_aead_child); + +void cryptd_free_aead(struct cryptd_aead *tfm) +{ + crypto_free_aead(&tfm->base); +} +EXPORT_SYMBOL_GPL(cryptd_free_aead); + static int __init cryptd_init(void) { int err; diff --git a/crypto/crypto_null.c b/crypto/crypto_null.c index cb71c9122bc0..07a8a96d46fc 100644 --- a/crypto/crypto_null.c +++ b/crypto/crypto_null.c @@ -1,11 +1,11 @@ -/* +/* * Cryptographic API. * * Null algorithms, aka Much Ado About Nothing. * * These are needed for IPsec, and may be useful in general for * testing & debugging. - * + * * The null cipher is compliant with RFC2410. * * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> @@ -163,7 +163,7 @@ MODULE_ALIAS("cipher_null"); static int __init crypto_null_mod_init(void) { int ret = 0; - + ret = crypto_register_alg(&cipher_null); if (ret < 0) goto out; @@ -180,7 +180,7 @@ static int __init crypto_null_mod_init(void) if (ret < 0) goto out_unregister_digest; -out: +out: return ret; out_unregister_digest: diff --git a/crypto/ctr.c b/crypto/ctr.c index 2d7425f0e7b8..4ca7222cfeb6 100644 --- a/crypto/ctr.c +++ b/crypto/ctr.c @@ -185,7 +185,7 @@ static struct crypto_instance *crypto_ctr_alloc(struct rtattr **tb) alg = crypto_attr_alg(tb[1], CRYPTO_ALG_TYPE_CIPHER, CRYPTO_ALG_TYPE_MASK); if (IS_ERR(alg)) - return ERR_PTR(PTR_ERR(alg)); + return ERR_CAST(alg); /* Block size must be >= 4 bytes. */ err = -EINVAL; @@ -219,6 +219,8 @@ static struct crypto_instance *crypto_ctr_alloc(struct rtattr **tb) inst->alg.cra_blkcipher.encrypt = crypto_ctr_crypt; inst->alg.cra_blkcipher.decrypt = crypto_ctr_crypt; + inst->alg.cra_blkcipher.geniv = "chainiv"; + out: crypto_mod_put(alg); return inst; diff --git a/crypto/deflate.c b/crypto/deflate.c index 9128da44e953..463dc859aa05 100644 --- a/crypto/deflate.c +++ b/crypto/deflate.c @@ -1,14 +1,14 @@ -/* +/* * Cryptographic API. * * Deflate algorithm (RFC 1951), implemented here primarily for use * by IPCOMP (RFC 3173 & RFC 2394). * * Copyright (c) 2003 James Morris <jmorris@intercode.com.au> - * + * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) + * Software Foundation; either version 2 of the License, or (at your option) * any later version. * * FIXME: deflate transforms will require up to a total of about 436k of kernel @@ -49,7 +49,7 @@ static int deflate_comp_init(struct deflate_ctx *ctx) struct z_stream_s *stream = &ctx->comp_stream; stream->workspace = vmalloc(zlib_deflate_workspacesize()); - if (!stream->workspace ) { + if (!stream->workspace) { ret = -ENOMEM; goto out; } @@ -61,7 +61,7 @@ static int deflate_comp_init(struct deflate_ctx *ctx) ret = -EINVAL; goto out_free; } -out: +out: return ret; out_free: vfree(stream->workspace); @@ -74,7 +74,7 @@ static int deflate_decomp_init(struct deflate_ctx *ctx) struct z_stream_s *stream = &ctx->decomp_stream; stream->workspace = kzalloc(zlib_inflate_workspacesize(), GFP_KERNEL); - if (!stream->workspace ) { + if (!stream->workspace) { ret = -ENOMEM; goto out; } @@ -106,7 +106,7 @@ static int deflate_init(struct crypto_tfm *tfm) { struct deflate_ctx *ctx = crypto_tfm_ctx(tfm); int ret; - + ret = deflate_comp_init(ctx); if (ret) goto out; @@ -153,11 +153,11 @@ static int deflate_compress(struct crypto_tfm *tfm, const u8 *src, out: return ret; } - + static int deflate_decompress(struct crypto_tfm *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen) { - + int ret = 0; struct deflate_ctx *dctx = crypto_tfm_ctx(tfm); struct z_stream_s *stream = &dctx->decomp_stream; @@ -182,7 +182,7 @@ static int deflate_decompress(struct crypto_tfm *tfm, const u8 *src, if (ret == Z_OK && !stream->avail_in && stream->avail_out) { u8 zerostuff = 0; stream->next_in = &zerostuff; - stream->avail_in = 1; + stream->avail_in = 1; ret = zlib_inflate(stream, Z_FINISH); } if (ret != Z_STREAM_END) { diff --git a/crypto/des_generic.c b/crypto/des_generic.c index 5bd3ee345a64..873818d48e86 100644 --- a/crypto/des_generic.c +++ b/crypto/des_generic.c @@ -614,7 +614,7 @@ static const u32 S8[64] = { #define T3(x) pt[2 * (x) + 2] #define T4(x) pt[2 * (x) + 3] -#define PC2(a, b, c, d) (T4(d) | T3(c) | T2(b) | T1(a)) +#define DES_PC2(a, b, c, d) (T4(d) | T3(c) | T2(b) | T1(a)) /* * Encryption key expansion @@ -639,22 +639,22 @@ unsigned long des_ekey(u32 *pe, const u8 *k) b = k[6]; b &= 0x0e; b <<= 4; b |= k[2] & 0x1e; b = pc1[b]; a = k[7]; a &= 0x0e; a <<= 4; a |= k[3] & 0x1e; a = pc1[a]; - pe[15 * 2 + 0] = PC2(a, b, c, d); d = rs[d]; - pe[14 * 2 + 0] = PC2(d, a, b, c); c = rs[c]; b = rs[b]; - pe[13 * 2 + 0] = PC2(b, c, d, a); a = rs[a]; d = rs[d]; - pe[12 * 2 + 0] = PC2(d, a, b, c); c = rs[c]; b = rs[b]; - pe[11 * 2 + 0] = PC2(b, c, d, a); a = rs[a]; d = rs[d]; - pe[10 * 2 + 0] = PC2(d, a, b, c); c = rs[c]; b = rs[b]; - pe[ 9 * 2 + 0] = PC2(b, c, d, a); a = rs[a]; d = rs[d]; - pe[ 8 * 2 + 0] = PC2(d, a, b, c); c = rs[c]; - pe[ 7 * 2 + 0] = PC2(c, d, a, b); b = rs[b]; a = rs[a]; - pe[ 6 * 2 + 0] = PC2(a, b, c, d); d = rs[d]; c = rs[c]; - pe[ 5 * 2 + 0] = PC2(c, d, a, b); b = rs[b]; a = rs[a]; - pe[ 4 * 2 + 0] = PC2(a, b, c, d); d = rs[d]; c = rs[c]; - pe[ 3 * 2 + 0] = PC2(c, d, a, b); b = rs[b]; a = rs[a]; - pe[ 2 * 2 + 0] = PC2(a, b, c, d); d = rs[d]; c = rs[c]; - pe[ 1 * 2 + 0] = PC2(c, d, a, b); b = rs[b]; - pe[ 0 * 2 + 0] = PC2(b, c, d, a); + pe[15 * 2 + 0] = DES_PC2(a, b, c, d); d = rs[d]; + pe[14 * 2 + 0] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[13 * 2 + 0] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[12 * 2 + 0] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[11 * 2 + 0] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[10 * 2 + 0] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[ 9 * 2 + 0] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[ 8 * 2 + 0] = DES_PC2(d, a, b, c); c = rs[c]; + pe[ 7 * 2 + 0] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[ 6 * 2 + 0] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[ 5 * 2 + 0] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[ 4 * 2 + 0] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[ 3 * 2 + 0] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[ 2 * 2 + 0] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[ 1 * 2 + 0] = DES_PC2(c, d, a, b); b = rs[b]; + pe[ 0 * 2 + 0] = DES_PC2(b, c, d, a); /* Check if first half is weak */ w = (a ^ c) | (b ^ d) | (rs[a] ^ c) | (b ^ rs[d]); @@ -670,22 +670,22 @@ unsigned long des_ekey(u32 *pe, const u8 *k) /* Check if second half is weak */ w |= (a ^ c) | (b ^ d) | (rs[a] ^ c) | (b ^ rs[d]); - pe[15 * 2 + 1] = PC2(a, b, c, d); d = rs[d]; - pe[14 * 2 + 1] = PC2(d, a, b, c); c = rs[c]; b = rs[b]; - pe[13 * 2 + 1] = PC2(b, c, d, a); a = rs[a]; d = rs[d]; - pe[12 * 2 + 1] = PC2(d, a, b, c); c = rs[c]; b = rs[b]; - pe[11 * 2 + 1] = PC2(b, c, d, a); a = rs[a]; d = rs[d]; - pe[10 * 2 + 1] = PC2(d, a, b, c); c = rs[c]; b = rs[b]; - pe[ 9 * 2 + 1] = PC2(b, c, d, a); a = rs[a]; d = rs[d]; - pe[ 8 * 2 + 1] = PC2(d, a, b, c); c = rs[c]; - pe[ 7 * 2 + 1] = PC2(c, d, a, b); b = rs[b]; a = rs[a]; - pe[ 6 * 2 + 1] = PC2(a, b, c, d); d = rs[d]; c = rs[c]; - pe[ 5 * 2 + 1] = PC2(c, d, a, b); b = rs[b]; a = rs[a]; - pe[ 4 * 2 + 1] = PC2(a, b, c, d); d = rs[d]; c = rs[c]; - pe[ 3 * 2 + 1] = PC2(c, d, a, b); b = rs[b]; a = rs[a]; - pe[ 2 * 2 + 1] = PC2(a, b, c, d); d = rs[d]; c = rs[c]; - pe[ 1 * 2 + 1] = PC2(c, d, a, b); b = rs[b]; - pe[ 0 * 2 + 1] = PC2(b, c, d, a); + pe[15 * 2 + 1] = DES_PC2(a, b, c, d); d = rs[d]; + pe[14 * 2 + 1] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[13 * 2 + 1] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[12 * 2 + 1] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[11 * 2 + 1] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[10 * 2 + 1] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[ 9 * 2 + 1] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[ 8 * 2 + 1] = DES_PC2(d, a, b, c); c = rs[c]; + pe[ 7 * 2 + 1] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[ 6 * 2 + 1] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[ 5 * 2 + 1] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[ 4 * 2 + 1] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[ 3 * 2 + 1] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[ 2 * 2 + 1] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[ 1 * 2 + 1] = DES_PC2(c, d, a, b); b = rs[b]; + pe[ 0 * 2 + 1] = DES_PC2(b, c, d, a); /* Fixup: 2413 5768 -> 1357 2468 */ for (d = 0; d < 16; ++d) { @@ -722,22 +722,22 @@ static void dkey(u32 *pe, const u8 *k) b = k[6]; b &= 0x0e; b <<= 4; b |= k[2] & 0x1e; b = pc1[b]; a = k[7]; a &= 0x0e; a <<= 4; a |= k[3] & 0x1e; a = pc1[a]; - pe[ 0 * 2] = PC2(a, b, c, d); d = rs[d]; - pe[ 1 * 2] = PC2(d, a, b, c); c = rs[c]; b = rs[b]; - pe[ 2 * 2] = PC2(b, c, d, a); a = rs[a]; d = rs[d]; - pe[ 3 * 2] = PC2(d, a, b, c); c = rs[c]; b = rs[b]; - pe[ 4 * 2] = PC2(b, c, d, a); a = rs[a]; d = rs[d]; - pe[ 5 * 2] = PC2(d, a, b, c); c = rs[c]; b = rs[b]; - pe[ 6 * 2] = PC2(b, c, d, a); a = rs[a]; d = rs[d]; - pe[ 7 * 2] = PC2(d, a, b, c); c = rs[c]; - pe[ 8 * 2] = PC2(c, d, a, b); b = rs[b]; a = rs[a]; - pe[ 9 * 2] = PC2(a, b, c, d); d = rs[d]; c = rs[c]; - pe[10 * 2] = PC2(c, d, a, b); b = rs[b]; a = rs[a]; - pe[11 * 2] = PC2(a, b, c, d); d = rs[d]; c = rs[c]; - pe[12 * 2] = PC2(c, d, a, b); b = rs[b]; a = rs[a]; - pe[13 * 2] = PC2(a, b, c, d); d = rs[d]; c = rs[c]; - pe[14 * 2] = PC2(c, d, a, b); b = rs[b]; - pe[15 * 2] = PC2(b, c, d, a); + pe[ 0 * 2] = DES_PC2(a, b, c, d); d = rs[d]; + pe[ 1 * 2] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[ 2 * 2] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[ 3 * 2] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[ 4 * 2] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[ 5 * 2] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[ 6 * 2] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[ 7 * 2] = DES_PC2(d, a, b, c); c = rs[c]; + pe[ 8 * 2] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[ 9 * 2] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[10 * 2] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[11 * 2] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[12 * 2] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[13 * 2] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[14 * 2] = DES_PC2(c, d, a, b); b = rs[b]; + pe[15 * 2] = DES_PC2(b, c, d, a); /* Skip to next table set */ pt += 512; @@ -747,22 +747,22 @@ static void dkey(u32 *pe, const u8 *k) b = k[2]; b &= 0xe0; b >>= 4; b |= k[6] & 0xf0; b = pc1[b + 1]; a = k[3]; a &= 0xe0; a >>= 4; a |= k[7] & 0xf0; a = pc1[a + 1]; - pe[ 0 * 2 + 1] = PC2(a, b, c, d); d = rs[d]; - pe[ 1 * 2 + 1] = PC2(d, a, b, c); c = rs[c]; b = rs[b]; - pe[ 2 * 2 + 1] = PC2(b, c, d, a); a = rs[a]; d = rs[d]; - pe[ 3 * 2 + 1] = PC2(d, a, b, c); c = rs[c]; b = rs[b]; - pe[ 4 * 2 + 1] = PC2(b, c, d, a); a = rs[a]; d = rs[d]; - pe[ 5 * 2 + 1] = PC2(d, a, b, c); c = rs[c]; b = rs[b]; - pe[ 6 * 2 + 1] = PC2(b, c, d, a); a = rs[a]; d = rs[d]; - pe[ 7 * 2 + 1] = PC2(d, a, b, c); c = rs[c]; - pe[ 8 * 2 + 1] = PC2(c, d, a, b); b = rs[b]; a = rs[a]; - pe[ 9 * 2 + 1] = PC2(a, b, c, d); d = rs[d]; c = rs[c]; - pe[10 * 2 + 1] = PC2(c, d, a, b); b = rs[b]; a = rs[a]; - pe[11 * 2 + 1] = PC2(a, b, c, d); d = rs[d]; c = rs[c]; - pe[12 * 2 + 1] = PC2(c, d, a, b); b = rs[b]; a = rs[a]; - pe[13 * 2 + 1] = PC2(a, b, c, d); d = rs[d]; c = rs[c]; - pe[14 * 2 + 1] = PC2(c, d, a, b); b = rs[b]; - pe[15 * 2 + 1] = PC2(b, c, d, a); + pe[ 0 * 2 + 1] = DES_PC2(a, b, c, d); d = rs[d]; + pe[ 1 * 2 + 1] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[ 2 * 2 + 1] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[ 3 * 2 + 1] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[ 4 * 2 + 1] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[ 5 * 2 + 1] = DES_PC2(d, a, b, c); c = rs[c]; b = rs[b]; + pe[ 6 * 2 + 1] = DES_PC2(b, c, d, a); a = rs[a]; d = rs[d]; + pe[ 7 * 2 + 1] = DES_PC2(d, a, b, c); c = rs[c]; + pe[ 8 * 2 + 1] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[ 9 * 2 + 1] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[10 * 2 + 1] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[11 * 2 + 1] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[12 * 2 + 1] = DES_PC2(c, d, a, b); b = rs[b]; a = rs[a]; + pe[13 * 2 + 1] = DES_PC2(a, b, c, d); d = rs[d]; c = rs[c]; + pe[14 * 2 + 1] = DES_PC2(c, d, a, b); b = rs[b]; + pe[15 * 2 + 1] = DES_PC2(b, c, d, a); /* Fixup: 2413 5768 -> 1357 2468 */ for (d = 0; d < 16; ++d) { @@ -869,8 +869,7 @@ static int des3_ede_setkey(struct crypto_tfm *tfm, const u8 *key, if (unlikely(!((K[0] ^ K[2]) | (K[1] ^ K[3])) || !((K[2] ^ K[4]) | (K[3] ^ K[5]))) && - (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) - { + (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { *flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } diff --git a/crypto/digest.c b/crypto/digest.c deleted file mode 100644 index 5d3f1303da98..000000000000 --- a/crypto/digest.c +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Cryptographic API. - * - * Digest operations. - * - * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) - * any later version. - * - */ - -#include <crypto/internal/hash.h> -#include <crypto/scatterwalk.h> -#include <linux/mm.h> -#include <linux/errno.h> -#include <linux/hardirq.h> -#include <linux/highmem.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/scatterlist.h> - -#include "internal.h" - -static int init(struct hash_desc *desc) -{ - struct crypto_tfm *tfm = crypto_hash_tfm(desc->tfm); - - tfm->__crt_alg->cra_digest.dia_init(tfm); - return 0; -} - -static int update2(struct hash_desc *desc, - struct scatterlist *sg, unsigned int nbytes) -{ - struct crypto_tfm *tfm = crypto_hash_tfm(desc->tfm); - unsigned int alignmask = crypto_tfm_alg_alignmask(tfm); - - if (!nbytes) - return 0; - - for (;;) { - struct page *pg = sg_page(sg); - unsigned int offset = sg->offset; - unsigned int l = sg->length; - - if (unlikely(l > nbytes)) - l = nbytes; - nbytes -= l; - - do { - unsigned int bytes_from_page = min(l, ((unsigned int) - (PAGE_SIZE)) - - offset); - char *src = crypto_kmap(pg, 0); - char *p = src + offset; - - if (unlikely(offset & alignmask)) { - unsigned int bytes = - alignmask + 1 - (offset & alignmask); - bytes = min(bytes, bytes_from_page); - tfm->__crt_alg->cra_digest.dia_update(tfm, p, - bytes); - p += bytes; - bytes_from_page -= bytes; - l -= bytes; - } - tfm->__crt_alg->cra_digest.dia_update(tfm, p, - bytes_from_page); - crypto_kunmap(src, 0); - crypto_yield(desc->flags); - offset = 0; - pg++; - l -= bytes_from_page; - } while (l > 0); - - if (!nbytes) - break; - sg = scatterwalk_sg_next(sg); - } - - return 0; -} - -static int update(struct hash_desc *desc, - struct scatterlist *sg, unsigned int nbytes) -{ - if (WARN_ON_ONCE(in_irq())) - return -EDEADLK; - return update2(desc, sg, nbytes); -} - -static int final(struct hash_desc *desc, u8 *out) -{ - struct crypto_tfm *tfm = crypto_hash_tfm(desc->tfm); - unsigned long alignmask = crypto_tfm_alg_alignmask(tfm); - struct digest_alg *digest = &tfm->__crt_alg->cra_digest; - - if (unlikely((unsigned long)out & alignmask)) { - unsigned long align = alignmask + 1; - unsigned long addr = (unsigned long)crypto_tfm_ctx(tfm); - u8 *dst = (u8 *)ALIGN(addr, align) + - ALIGN(tfm->__crt_alg->cra_ctxsize, align); - - digest->dia_final(tfm, dst); - memcpy(out, dst, digest->dia_digestsize); - } else - digest->dia_final(tfm, out); - - return 0; -} - -static int nosetkey(struct crypto_hash *tfm, const u8 *key, unsigned int keylen) -{ - crypto_hash_clear_flags(tfm, CRYPTO_TFM_RES_MASK); - return -ENOSYS; -} - -static int setkey(struct crypto_hash *hash, const u8 *key, unsigned int keylen) -{ - struct crypto_tfm *tfm = crypto_hash_tfm(hash); - - crypto_hash_clear_flags(hash, CRYPTO_TFM_RES_MASK); - return tfm->__crt_alg->cra_digest.dia_setkey(tfm, key, keylen); -} - -static int digest(struct hash_desc *desc, - struct scatterlist *sg, unsigned int nbytes, u8 *out) -{ - if (WARN_ON_ONCE(in_irq())) - return -EDEADLK; - - init(desc); - update2(desc, sg, nbytes); - return final(desc, out); -} - -int crypto_init_digest_ops(struct crypto_tfm *tfm) -{ - struct hash_tfm *ops = &tfm->crt_hash; - struct digest_alg *dalg = &tfm->__crt_alg->cra_digest; - - if (dalg->dia_digestsize > PAGE_SIZE / 8) - return -EINVAL; - - ops->init = init; - ops->update = update; - ops->final = final; - ops->digest = digest; - ops->setkey = dalg->dia_setkey ? setkey : nosetkey; - ops->digestsize = dalg->dia_digestsize; - - return 0; -} - -void crypto_exit_digest_ops(struct crypto_tfm *tfm) -{ -} - -static int digest_async_nosetkey(struct crypto_ahash *tfm_async, const u8 *key, - unsigned int keylen) -{ - crypto_ahash_clear_flags(tfm_async, CRYPTO_TFM_RES_MASK); - return -ENOSYS; -} - -static int digest_async_setkey(struct crypto_ahash *tfm_async, const u8 *key, - unsigned int keylen) -{ - struct crypto_tfm *tfm = crypto_ahash_tfm(tfm_async); - struct digest_alg *dalg = &tfm->__crt_alg->cra_digest; - - crypto_ahash_clear_flags(tfm_async, CRYPTO_TFM_RES_MASK); - return dalg->dia_setkey(tfm, key, keylen); -} - -static int digest_async_init(struct ahash_request *req) -{ - struct crypto_tfm *tfm = req->base.tfm; - struct digest_alg *dalg = &tfm->__crt_alg->cra_digest; - - dalg->dia_init(tfm); - return 0; -} - -static int digest_async_update(struct ahash_request *req) -{ - struct crypto_tfm *tfm = req->base.tfm; - struct hash_desc desc = { - .tfm = __crypto_hash_cast(tfm), - .flags = req->base.flags, - }; - - update(&desc, req->src, req->nbytes); - return 0; -} - -static int digest_async_final(struct ahash_request *req) -{ - struct crypto_tfm *tfm = req->base.tfm; - struct hash_desc desc = { - .tfm = __crypto_hash_cast(tfm), - .flags = req->base.flags, - }; - - final(&desc, req->result); - return 0; -} - -static int digest_async_digest(struct ahash_request *req) -{ - struct crypto_tfm *tfm = req->base.tfm; - struct hash_desc desc = { - .tfm = __crypto_hash_cast(tfm), - .flags = req->base.flags, - }; - - return digest(&desc, req->src, req->nbytes, req->result); -} - -int crypto_init_digest_ops_async(struct crypto_tfm *tfm) -{ - struct ahash_tfm *crt = &tfm->crt_ahash; - struct digest_alg *dalg = &tfm->__crt_alg->cra_digest; - - if (dalg->dia_digestsize > PAGE_SIZE / 8) - return -EINVAL; - - crt->init = digest_async_init; - crt->update = digest_async_update; - crt->final = digest_async_final; - crt->digest = digest_async_digest; - crt->setkey = dalg->dia_setkey ? digest_async_setkey : - digest_async_nosetkey; - crt->digestsize = dalg->dia_digestsize; - - return 0; -} diff --git a/crypto/ecb.c b/crypto/ecb.c index a46838e98a71..935cfef4aa84 100644 --- a/crypto/ecb.c +++ b/crypto/ecb.c @@ -55,7 +55,7 @@ static int crypto_ecb_crypt(struct blkcipher_desc *desc, do { fn(crypto_cipher_tfm(tfm), wdst, wsrc); - + wsrc += bsize; wdst += bsize; } while ((nbytes -= bsize) >= bsize); diff --git a/crypto/fcrypt.c b/crypto/fcrypt.c index b82d61f4e26c..c33107e340b6 100644 --- a/crypto/fcrypt.c +++ b/crypto/fcrypt.c @@ -60,13 +60,13 @@ do { \ u32 t = lo & ((1 << n) - 1); \ lo = (lo >> n) | ((hi & ((1 << n) - 1)) << (32 - n)); \ hi = (hi >> n) | (t << (24-n)); \ -} while(0) +} while (0) /* Rotate right one 64 bit number as a 56 bit number */ #define ror56_64(k, n) \ do { \ k = (k >> n) | ((k & ((1 << n) - 1)) << (56 - n)); \ -} while(0) +} while (0) /* * Sboxes for Feistel network derived from @@ -228,7 +228,7 @@ do { \ union lc4 { __be32 l; u8 c[4]; } u; \ u.l = sched ^ R; \ L ^= sbox0[u.c[0]] ^ sbox1[u.c[1]] ^ sbox2[u.c[2]] ^ sbox3[u.c[3]]; \ -} while(0) +} while (0) /* * encryptor diff --git a/crypto/gcm.c b/crypto/gcm.c index e70afd0c73dd..2f5fbba6576c 100644 --- a/crypto/gcm.c +++ b/crypto/gcm.c @@ -11,7 +11,10 @@ #include <crypto/gf128mul.h> #include <crypto/internal/aead.h> #include <crypto/internal/skcipher.h> +#include <crypto/internal/hash.h> #include <crypto/scatterwalk.h> +#include <crypto/hash.h> +#include "internal.h" #include <linux/completion.h> #include <linux/err.h> #include <linux/init.h> @@ -21,11 +24,12 @@ struct gcm_instance_ctx { struct crypto_skcipher_spawn ctr; + struct crypto_ahash_spawn ghash; }; struct crypto_gcm_ctx { struct crypto_ablkcipher *ctr; - struct gf128mul_4k *gf128; + struct crypto_ahash *ghash; }; struct crypto_rfc4106_ctx { @@ -33,11 +37,23 @@ struct crypto_rfc4106_ctx { u8 nonce[4]; }; +struct crypto_rfc4543_ctx { + struct crypto_aead *child; + u8 nonce[4]; +}; + +struct crypto_rfc4543_req_ctx { + u8 auth_tag[16]; + struct scatterlist cipher[1]; + struct scatterlist payload[2]; + struct scatterlist assoc[2]; + struct aead_request subreq; +}; + struct crypto_gcm_ghash_ctx { - u32 bytes; - u32 flags; - struct gf128mul_4k *gf128; - u8 buffer[16]; + unsigned int cryptlen; + struct scatterlist *src; + void (*complete)(struct aead_request *req, int err); }; struct crypto_gcm_req_priv_ctx { @@ -45,8 +61,11 @@ struct crypto_gcm_req_priv_ctx { u8 iauth_tag[16]; struct scatterlist src[2]; struct scatterlist dst[2]; - struct crypto_gcm_ghash_ctx ghash; - struct ablkcipher_request abreq; + struct crypto_gcm_ghash_ctx ghash_ctx; + union { + struct ahash_request ahreq; + struct ablkcipher_request abreq; + } u; }; struct crypto_gcm_setkey_result { @@ -54,6 +73,8 @@ struct crypto_gcm_setkey_result { struct completion completion; }; +static void *gcm_zeroes; + static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx( struct aead_request *req) { @@ -62,113 +83,6 @@ static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx( return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1); } -static void crypto_gcm_ghash_init(struct crypto_gcm_ghash_ctx *ctx, u32 flags, - struct gf128mul_4k *gf128) -{ - ctx->bytes = 0; - ctx->flags = flags; - ctx->gf128 = gf128; - memset(ctx->buffer, 0, 16); -} - -static void crypto_gcm_ghash_update(struct crypto_gcm_ghash_ctx *ctx, - const u8 *src, unsigned int srclen) -{ - u8 *dst = ctx->buffer; - - if (ctx->bytes) { - int n = min(srclen, ctx->bytes); - u8 *pos = dst + (16 - ctx->bytes); - - ctx->bytes -= n; - srclen -= n; - - while (n--) - *pos++ ^= *src++; - - if (!ctx->bytes) - gf128mul_4k_lle((be128 *)dst, ctx->gf128); - } - - while (srclen >= 16) { - crypto_xor(dst, src, 16); - gf128mul_4k_lle((be128 *)dst, ctx->gf128); - src += 16; - srclen -= 16; - } - - if (srclen) { - ctx->bytes = 16 - srclen; - while (srclen--) - *dst++ ^= *src++; - } -} - -static void crypto_gcm_ghash_update_sg(struct crypto_gcm_ghash_ctx *ctx, - struct scatterlist *sg, int len) -{ - struct scatter_walk walk; - u8 *src; - int n; - - if (!len) - return; - - scatterwalk_start(&walk, sg); - - while (len) { - n = scatterwalk_clamp(&walk, len); - - if (!n) { - scatterwalk_start(&walk, scatterwalk_sg_next(walk.sg)); - n = scatterwalk_clamp(&walk, len); - } - - src = scatterwalk_map(&walk, 0); - - crypto_gcm_ghash_update(ctx, src, n); - len -= n; - - scatterwalk_unmap(src, 0); - scatterwalk_advance(&walk, n); - scatterwalk_done(&walk, 0, len); - if (len) - crypto_yield(ctx->flags); - } -} - -static void crypto_gcm_ghash_flush(struct crypto_gcm_ghash_ctx *ctx) -{ - u8 *dst = ctx->buffer; - - if (ctx->bytes) { - u8 *tmp = dst + (16 - ctx->bytes); - - while (ctx->bytes--) - *tmp++ ^= 0; - - gf128mul_4k_lle((be128 *)dst, ctx->gf128); - } - - ctx->bytes = 0; -} - -static void crypto_gcm_ghash_final_xor(struct crypto_gcm_ghash_ctx *ctx, - unsigned int authlen, - unsigned int cryptlen, u8 *dst) -{ - u8 *buf = ctx->buffer; - u128 lengths; - - lengths.a = cpu_to_be64(authlen * 8); - lengths.b = cpu_to_be64(cryptlen * 8); - - crypto_gcm_ghash_flush(ctx); - crypto_xor(buf, (u8 *)&lengths, 16); - gf128mul_4k_lle((be128 *)buf, ctx->gf128); - crypto_xor(dst, buf, 16); -} - static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err) { struct crypto_gcm_setkey_result *result = req->data; @@ -184,6 +98,7 @@ static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); + struct crypto_ahash *ghash = ctx->ghash; struct crypto_ablkcipher *ctr = ctx->ctr; struct { be128 hash; @@ -233,13 +148,12 @@ static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, if (err) goto out; - if (ctx->gf128 != NULL) - gf128mul_free_4k(ctx->gf128); - - ctx->gf128 = gf128mul_init_4k_lle(&data->hash); - - if (ctx->gf128 == NULL) - err = -ENOMEM; + crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK); + crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) & + CRYPTO_TFM_REQ_MASK); + err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128)); + crypto_aead_set_flags(aead, crypto_ahash_get_flags(ghash) & + CRYPTO_TFM_RES_MASK); out: kfree(data); @@ -272,8 +186,6 @@ static void crypto_gcm_init_crypt(struct ablkcipher_request *ablk_req, struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - u32 flags = req->base.tfm->crt_flags; - struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash; struct scatterlist *dst; __be32 counter = cpu_to_be32(1); @@ -296,35 +208,329 @@ static void crypto_gcm_init_crypt(struct ablkcipher_request *ablk_req, ablkcipher_request_set_crypt(ablk_req, pctx->src, dst, cryptlen + sizeof(pctx->auth_tag), req->iv); +} - crypto_gcm_ghash_init(ghash, flags, ctx->gf128); +static inline unsigned int gcm_remain(unsigned int len) +{ + len &= 0xfU; + return len ? 16 - len : 0; +} + +static void gcm_hash_len_done(struct crypto_async_request *areq, int err); +static void gcm_hash_final_done(struct crypto_async_request *areq, int err); + +static int gcm_hash_update(struct aead_request *req, + struct crypto_gcm_req_priv_ctx *pctx, + crypto_completion_t complete, + struct scatterlist *src, + unsigned int len) +{ + struct ahash_request *ahreq = &pctx->u.ahreq; - crypto_gcm_ghash_update_sg(ghash, req->assoc, req->assoclen); - crypto_gcm_ghash_flush(ghash); + ahash_request_set_callback(ahreq, aead_request_flags(req), + complete, req); + ahash_request_set_crypt(ahreq, src, NULL, len); + + return crypto_ahash_update(ahreq); } -static int crypto_gcm_hash(struct aead_request *req) +static int gcm_hash_remain(struct aead_request *req, + struct crypto_gcm_req_priv_ctx *pctx, + unsigned int remain, + crypto_completion_t complete) +{ + struct ahash_request *ahreq = &pctx->u.ahreq; + + ahash_request_set_callback(ahreq, aead_request_flags(req), + complete, req); + sg_init_one(pctx->src, gcm_zeroes, remain); + ahash_request_set_crypt(ahreq, pctx->src, NULL, remain); + + return crypto_ahash_update(ahreq); +} + +static int gcm_hash_len(struct aead_request *req, + struct crypto_gcm_req_priv_ctx *pctx) +{ + struct ahash_request *ahreq = &pctx->u.ahreq; + struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; + u128 lengths; + + lengths.a = cpu_to_be64(req->assoclen * 8); + lengths.b = cpu_to_be64(gctx->cryptlen * 8); + memcpy(pctx->iauth_tag, &lengths, 16); + sg_init_one(pctx->src, pctx->iauth_tag, 16); + ahash_request_set_callback(ahreq, aead_request_flags(req), + gcm_hash_len_done, req); + ahash_request_set_crypt(ahreq, pctx->src, + NULL, sizeof(lengths)); + + return crypto_ahash_update(ahreq); +} + +static int gcm_hash_final(struct aead_request *req, + struct crypto_gcm_req_priv_ctx *pctx) +{ + struct ahash_request *ahreq = &pctx->u.ahreq; + + ahash_request_set_callback(ahreq, aead_request_flags(req), + gcm_hash_final_done, req); + ahash_request_set_crypt(ahreq, NULL, pctx->iauth_tag, 0); + + return crypto_ahash_final(ahreq); +} + +static void __gcm_hash_final_done(struct aead_request *req, int err) { - struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - u8 *auth_tag = pctx->auth_tag; - struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash; + struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; + + if (!err) + crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16); + + gctx->complete(req, err); +} - crypto_gcm_ghash_update_sg(ghash, req->dst, req->cryptlen); - crypto_gcm_ghash_final_xor(ghash, req->assoclen, req->cryptlen, - auth_tag); +static void gcm_hash_final_done(struct crypto_async_request *areq, int err) +{ + struct aead_request *req = areq->data; + + __gcm_hash_final_done(req, err); +} + +static void __gcm_hash_len_done(struct aead_request *req, int err) +{ + struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); + + if (!err) { + err = gcm_hash_final(req, pctx); + if (err == -EINPROGRESS || err == -EBUSY) + return; + } + + __gcm_hash_final_done(req, err); +} + +static void gcm_hash_len_done(struct crypto_async_request *areq, int err) +{ + struct aead_request *req = areq->data; + + __gcm_hash_len_done(req, err); +} + +static void __gcm_hash_crypt_remain_done(struct aead_request *req, int err) +{ + struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); + + if (!err) { + err = gcm_hash_len(req, pctx); + if (err == -EINPROGRESS || err == -EBUSY) + return; + } + + __gcm_hash_len_done(req, err); +} + +static void gcm_hash_crypt_remain_done(struct crypto_async_request *areq, + int err) +{ + struct aead_request *req = areq->data; + + __gcm_hash_crypt_remain_done(req, err); +} + +static void __gcm_hash_crypt_done(struct aead_request *req, int err) +{ + struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); + struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; + unsigned int remain; + + if (!err) { + remain = gcm_remain(gctx->cryptlen); + BUG_ON(!remain); + err = gcm_hash_remain(req, pctx, remain, + gcm_hash_crypt_remain_done); + if (err == -EINPROGRESS || err == -EBUSY) + return; + } + + __gcm_hash_crypt_remain_done(req, err); +} + +static void gcm_hash_crypt_done(struct crypto_async_request *areq, int err) +{ + struct aead_request *req = areq->data; + + __gcm_hash_crypt_done(req, err); +} + +static void __gcm_hash_assoc_remain_done(struct aead_request *req, int err) +{ + struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); + struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; + crypto_completion_t complete; + unsigned int remain = 0; + + if (!err && gctx->cryptlen) { + remain = gcm_remain(gctx->cryptlen); + complete = remain ? gcm_hash_crypt_done : + gcm_hash_crypt_remain_done; + err = gcm_hash_update(req, pctx, complete, + gctx->src, gctx->cryptlen); + if (err == -EINPROGRESS || err == -EBUSY) + return; + } + + if (remain) + __gcm_hash_crypt_done(req, err); + else + __gcm_hash_crypt_remain_done(req, err); +} + +static void gcm_hash_assoc_remain_done(struct crypto_async_request *areq, + int err) +{ + struct aead_request *req = areq->data; + + __gcm_hash_assoc_remain_done(req, err); +} + +static void __gcm_hash_assoc_done(struct aead_request *req, int err) +{ + struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); + unsigned int remain; + + if (!err) { + remain = gcm_remain(req->assoclen); + BUG_ON(!remain); + err = gcm_hash_remain(req, pctx, remain, + gcm_hash_assoc_remain_done); + if (err == -EINPROGRESS || err == -EBUSY) + return; + } + + __gcm_hash_assoc_remain_done(req, err); +} + +static void gcm_hash_assoc_done(struct crypto_async_request *areq, int err) +{ + struct aead_request *req = areq->data; + + __gcm_hash_assoc_done(req, err); +} + +static void __gcm_hash_init_done(struct aead_request *req, int err) +{ + struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); + crypto_completion_t complete; + unsigned int remain = 0; + + if (!err && req->assoclen) { + remain = gcm_remain(req->assoclen); + complete = remain ? gcm_hash_assoc_done : + gcm_hash_assoc_remain_done; + err = gcm_hash_update(req, pctx, complete, + req->assoc, req->assoclen); + if (err == -EINPROGRESS || err == -EBUSY) + return; + } + + if (remain) + __gcm_hash_assoc_done(req, err); + else + __gcm_hash_assoc_remain_done(req, err); +} + +static void gcm_hash_init_done(struct crypto_async_request *areq, int err) +{ + struct aead_request *req = areq->data; + + __gcm_hash_init_done(req, err); +} + +static int gcm_hash(struct aead_request *req, + struct crypto_gcm_req_priv_ctx *pctx) +{ + struct ahash_request *ahreq = &pctx->u.ahreq; + struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; + struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + unsigned int remain; + crypto_completion_t complete; + int err; + + ahash_request_set_tfm(ahreq, ctx->ghash); + + ahash_request_set_callback(ahreq, aead_request_flags(req), + gcm_hash_init_done, req); + err = crypto_ahash_init(ahreq); + if (err) + return err; + remain = gcm_remain(req->assoclen); + complete = remain ? gcm_hash_assoc_done : gcm_hash_assoc_remain_done; + err = gcm_hash_update(req, pctx, complete, req->assoc, req->assoclen); + if (err) + return err; + if (remain) { + err = gcm_hash_remain(req, pctx, remain, + gcm_hash_assoc_remain_done); + if (err) + return err; + } + remain = gcm_remain(gctx->cryptlen); + complete = remain ? gcm_hash_crypt_done : gcm_hash_crypt_remain_done; + err = gcm_hash_update(req, pctx, complete, gctx->src, gctx->cryptlen); + if (err) + return err; + if (remain) { + err = gcm_hash_remain(req, pctx, remain, + gcm_hash_crypt_remain_done); + if (err) + return err; + } + err = gcm_hash_len(req, pctx); + if (err) + return err; + err = gcm_hash_final(req, pctx); + if (err) + return err; + + return 0; +} + +static void gcm_enc_copy_hash(struct aead_request *req, + struct crypto_gcm_req_priv_ctx *pctx) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + u8 *auth_tag = pctx->auth_tag; scatterwalk_map_and_copy(auth_tag, req->dst, req->cryptlen, crypto_aead_authsize(aead), 1); - return 0; } -static void crypto_gcm_encrypt_done(struct crypto_async_request *areq, int err) +static void gcm_enc_hash_done(struct aead_request *req, int err) { - struct aead_request *req = areq->data; + struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); if (!err) - err = crypto_gcm_hash(req); + gcm_enc_copy_hash(req, pctx); + + aead_request_complete(req, err); +} + +static void gcm_encrypt_done(struct crypto_async_request *areq, int err) +{ + struct aead_request *req = areq->data; + struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); + + if (!err) { + err = gcm_hash(req, pctx); + if (err == -EINPROGRESS || err == -EBUSY) + return; + else if (!err) { + crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16); + gcm_enc_copy_hash(req, pctx); + } + } aead_request_complete(req, err); } @@ -332,43 +538,73 @@ static void crypto_gcm_encrypt_done(struct crypto_async_request *areq, int err) static int crypto_gcm_encrypt(struct aead_request *req) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct ablkcipher_request *abreq = &pctx->abreq; + struct ablkcipher_request *abreq = &pctx->u.abreq; + struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; int err; crypto_gcm_init_crypt(abreq, req, req->cryptlen); ablkcipher_request_set_callback(abreq, aead_request_flags(req), - crypto_gcm_encrypt_done, req); + gcm_encrypt_done, req); + + gctx->src = req->dst; + gctx->cryptlen = req->cryptlen; + gctx->complete = gcm_enc_hash_done; err = crypto_ablkcipher_encrypt(abreq); if (err) return err; - return crypto_gcm_hash(req); + err = gcm_hash(req, pctx); + if (err) + return err; + + crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16); + gcm_enc_copy_hash(req, pctx); + + return 0; } -static int crypto_gcm_verify(struct aead_request *req) +static int crypto_gcm_verify(struct aead_request *req, + struct crypto_gcm_req_priv_ctx *pctx) { struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash; u8 *auth_tag = pctx->auth_tag; u8 *iauth_tag = pctx->iauth_tag; unsigned int authsize = crypto_aead_authsize(aead); unsigned int cryptlen = req->cryptlen - authsize; - crypto_gcm_ghash_final_xor(ghash, req->assoclen, cryptlen, auth_tag); - - authsize = crypto_aead_authsize(aead); + crypto_xor(auth_tag, iauth_tag, 16); scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0); return memcmp(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0; } -static void crypto_gcm_decrypt_done(struct crypto_async_request *areq, int err) +static void gcm_decrypt_done(struct crypto_async_request *areq, int err) { struct aead_request *req = areq->data; + struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); if (!err) - err = crypto_gcm_verify(req); + err = crypto_gcm_verify(req, pctx); + + aead_request_complete(req, err); +} + +static void gcm_dec_hash_done(struct aead_request *req, int err) +{ + struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); + struct ablkcipher_request *abreq = &pctx->u.abreq; + struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; + + if (!err) { + ablkcipher_request_set_callback(abreq, aead_request_flags(req), + gcm_decrypt_done, req); + crypto_gcm_init_crypt(abreq, req, gctx->cryptlen); + err = crypto_ablkcipher_decrypt(abreq); + if (err == -EINPROGRESS || err == -EBUSY) + return; + else if (!err) + err = crypto_gcm_verify(req, pctx); + } aead_request_complete(req, err); } @@ -377,27 +613,32 @@ static int crypto_gcm_decrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct ablkcipher_request *abreq = &pctx->abreq; - struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash; - unsigned int cryptlen = req->cryptlen; + struct ablkcipher_request *abreq = &pctx->u.abreq; + struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; unsigned int authsize = crypto_aead_authsize(aead); + unsigned int cryptlen = req->cryptlen; int err; if (cryptlen < authsize) return -EINVAL; cryptlen -= authsize; - crypto_gcm_init_crypt(abreq, req, cryptlen); - ablkcipher_request_set_callback(abreq, aead_request_flags(req), - crypto_gcm_decrypt_done, req); + gctx->src = req->src; + gctx->cryptlen = cryptlen; + gctx->complete = gcm_dec_hash_done; - crypto_gcm_ghash_update_sg(ghash, req->src, cryptlen); + err = gcm_hash(req, pctx); + if (err) + return err; + ablkcipher_request_set_callback(abreq, aead_request_flags(req), + gcm_decrypt_done, req); + crypto_gcm_init_crypt(abreq, req, cryptlen); err = crypto_ablkcipher_decrypt(abreq); if (err) return err; - return crypto_gcm_verify(req); + return crypto_gcm_verify(req, pctx); } static int crypto_gcm_init_tfm(struct crypto_tfm *tfm) @@ -406,43 +647,56 @@ static int crypto_gcm_init_tfm(struct crypto_tfm *tfm) struct gcm_instance_ctx *ictx = crypto_instance_ctx(inst); struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm); struct crypto_ablkcipher *ctr; + struct crypto_ahash *ghash; unsigned long align; int err; + ghash = crypto_spawn_ahash(&ictx->ghash); + if (IS_ERR(ghash)) + return PTR_ERR(ghash); + ctr = crypto_spawn_skcipher(&ictx->ctr); err = PTR_ERR(ctr); if (IS_ERR(ctr)) - return err; + goto err_free_hash; ctx->ctr = ctr; - ctx->gf128 = NULL; + ctx->ghash = ghash; align = crypto_tfm_alg_alignmask(tfm); align &= ~(crypto_tfm_ctx_alignment() - 1); tfm->crt_aead.reqsize = align + - sizeof(struct crypto_gcm_req_priv_ctx) + - crypto_ablkcipher_reqsize(ctr); + offsetof(struct crypto_gcm_req_priv_ctx, u) + + max(sizeof(struct ablkcipher_request) + + crypto_ablkcipher_reqsize(ctr), + sizeof(struct ahash_request) + + crypto_ahash_reqsize(ghash)); return 0; + +err_free_hash: + crypto_free_ahash(ghash); + return err; } static void crypto_gcm_exit_tfm(struct crypto_tfm *tfm) { struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm); - if (ctx->gf128 != NULL) - gf128mul_free_4k(ctx->gf128); - + crypto_free_ahash(ctx->ghash); crypto_free_ablkcipher(ctx->ctr); } static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb, const char *full_name, - const char *ctr_name) + const char *ctr_name, + const char *ghash_name) { struct crypto_attr_type *algt; struct crypto_instance *inst; struct crypto_alg *ctr; + struct crypto_alg *ghash_alg; + struct ahash_alg *ghash_ahash_alg; struct gcm_instance_ctx *ctx; int err; @@ -454,17 +708,31 @@ static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb, if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) return ERR_PTR(-EINVAL); + ghash_alg = crypto_find_alg(ghash_name, &crypto_ahash_type, + CRYPTO_ALG_TYPE_HASH, + CRYPTO_ALG_TYPE_AHASH_MASK); + err = PTR_ERR(ghash_alg); + if (IS_ERR(ghash_alg)) + return ERR_PTR(err); + + err = -ENOMEM; inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); if (!inst) - return ERR_PTR(-ENOMEM); + goto out_put_ghash; ctx = crypto_instance_ctx(inst); + ghash_ahash_alg = container_of(ghash_alg, struct ahash_alg, halg.base); + err = crypto_init_ahash_spawn(&ctx->ghash, &ghash_ahash_alg->halg, + inst); + if (err) + goto err_free_inst; + crypto_set_skcipher_spawn(&ctx->ctr, inst); err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0, crypto_requires_sync(algt->type, algt->mask)); if (err) - goto err_free_inst; + goto err_drop_ghash; ctr = crypto_skcipher_spawn_alg(&ctx->ctr); @@ -479,7 +747,8 @@ static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb, err = -ENAMETOOLONG; if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, - "gcm_base(%s)", ctr->cra_driver_name) >= + "gcm_base(%s,%s)", ctr->cra_driver_name, + ghash_alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto out_put_ctr; @@ -502,12 +771,16 @@ static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb, inst->alg.cra_aead.decrypt = crypto_gcm_decrypt; out: + crypto_mod_put(ghash_alg); return inst; out_put_ctr: crypto_drop_skcipher(&ctx->ctr); +err_drop_ghash: + crypto_drop_ahash(&ctx->ghash); err_free_inst: kfree(inst); +out_put_ghash: inst = ERR_PTR(err); goto out; } @@ -532,7 +805,7 @@ static struct crypto_instance *crypto_gcm_alloc(struct rtattr **tb) CRYPTO_MAX_ALG_NAME) return ERR_PTR(-ENAMETOOLONG); - return crypto_gcm_alloc_common(tb, full_name, ctr_name); + return crypto_gcm_alloc_common(tb, full_name, ctr_name, "ghash"); } static void crypto_gcm_free(struct crypto_instance *inst) @@ -540,6 +813,7 @@ static void crypto_gcm_free(struct crypto_instance *inst) struct gcm_instance_ctx *ctx = crypto_instance_ctx(inst); crypto_drop_skcipher(&ctx->ctr); + crypto_drop_ahash(&ctx->ghash); kfree(inst); } @@ -554,6 +828,7 @@ static struct crypto_instance *crypto_gcm_base_alloc(struct rtattr **tb) { int err; const char *ctr_name; + const char *ghash_name; char full_name[CRYPTO_MAX_ALG_NAME]; ctr_name = crypto_attr_alg_name(tb[1]); @@ -561,11 +836,16 @@ static struct crypto_instance *crypto_gcm_base_alloc(struct rtattr **tb) if (IS_ERR(ctr_name)) return ERR_PTR(err); - if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s)", - ctr_name) >= CRYPTO_MAX_ALG_NAME) + ghash_name = crypto_attr_alg_name(tb[2]); + err = PTR_ERR(ghash_name); + if (IS_ERR(ghash_name)) + return ERR_PTR(err); + + if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s,%s)", + ctr_name, ghash_name) >= CRYPTO_MAX_ALG_NAME) return ERR_PTR(-ENAMETOOLONG); - return crypto_gcm_alloc_common(tb, full_name, ctr_name); + return crypto_gcm_alloc_common(tb, full_name, ctr_name, ghash_name); } static struct crypto_template crypto_gcm_base_tmpl = { @@ -780,10 +1060,280 @@ static struct crypto_template crypto_rfc4106_tmpl = { .module = THIS_MODULE, }; +static inline struct crypto_rfc4543_req_ctx *crypto_rfc4543_reqctx( + struct aead_request *req) +{ + unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req)); + + return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1); +} + +static int crypto_rfc4543_setkey(struct crypto_aead *parent, const u8 *key, + unsigned int keylen) +{ + struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent); + struct crypto_aead *child = ctx->child; + int err; + + if (keylen < 4) + return -EINVAL; + + keylen -= 4; + memcpy(ctx->nonce, key + keylen, 4); + + crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(child, crypto_aead_get_flags(parent) & + CRYPTO_TFM_REQ_MASK); + err = crypto_aead_setkey(child, key, keylen); + crypto_aead_set_flags(parent, crypto_aead_get_flags(child) & + CRYPTO_TFM_RES_MASK); + + return err; +} + +static int crypto_rfc4543_setauthsize(struct crypto_aead *parent, + unsigned int authsize) +{ + struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(parent); + + if (authsize != 16) + return -EINVAL; + + return crypto_aead_setauthsize(ctx->child, authsize); +} + +/* this is the same as crypto_authenc_chain */ +static void crypto_rfc4543_chain(struct scatterlist *head, + struct scatterlist *sg, int chain) +{ + if (chain) { + head->length += sg->length; + sg = scatterwalk_sg_next(sg); + } + + if (sg) + scatterwalk_sg_chain(head, 2, sg); + else + sg_mark_end(head); +} + +static struct aead_request *crypto_rfc4543_crypt(struct aead_request *req, + int enc) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct crypto_rfc4543_ctx *ctx = crypto_aead_ctx(aead); + struct crypto_rfc4543_req_ctx *rctx = crypto_rfc4543_reqctx(req); + struct aead_request *subreq = &rctx->subreq; + struct scatterlist *dst = req->dst; + struct scatterlist *cipher = rctx->cipher; + struct scatterlist *payload = rctx->payload; + struct scatterlist *assoc = rctx->assoc; + unsigned int authsize = crypto_aead_authsize(aead); + unsigned int assoclen = req->assoclen; + struct page *dstp; + u8 *vdst; + u8 *iv = PTR_ALIGN((u8 *)(rctx + 1) + crypto_aead_reqsize(ctx->child), + crypto_aead_alignmask(ctx->child) + 1); + + memcpy(iv, ctx->nonce, 4); + memcpy(iv + 4, req->iv, 8); + + /* construct cipher/plaintext */ + if (enc) + memset(rctx->auth_tag, 0, authsize); + else + scatterwalk_map_and_copy(rctx->auth_tag, dst, + req->cryptlen - authsize, + authsize, 0); + + sg_init_one(cipher, rctx->auth_tag, authsize); + + /* construct the aad */ + dstp = sg_page(dst); + vdst = PageHighMem(dstp) ? NULL : page_address(dstp) + dst->offset; + + sg_init_table(payload, 2); + sg_set_buf(payload, req->iv, 8); + crypto_rfc4543_chain(payload, dst, vdst == req->iv + 8); + assoclen += 8 + req->cryptlen - (enc ? 0 : authsize); + + sg_init_table(assoc, 2); + sg_set_page(assoc, sg_page(req->assoc), req->assoc->length, + req->assoc->offset); + crypto_rfc4543_chain(assoc, payload, 0); + + aead_request_set_tfm(subreq, ctx->child); + aead_request_set_callback(subreq, req->base.flags, req->base.complete, + req->base.data); + aead_request_set_crypt(subreq, cipher, cipher, enc ? 0 : authsize, iv); + aead_request_set_assoc(subreq, assoc, assoclen); + + return subreq; +} + +static int crypto_rfc4543_encrypt(struct aead_request *req) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct crypto_rfc4543_req_ctx *rctx = crypto_rfc4543_reqctx(req); + struct aead_request *subreq; + int err; + + subreq = crypto_rfc4543_crypt(req, 1); + err = crypto_aead_encrypt(subreq); + if (err) + return err; + + scatterwalk_map_and_copy(rctx->auth_tag, req->dst, req->cryptlen, + crypto_aead_authsize(aead), 1); + + return 0; +} + +static int crypto_rfc4543_decrypt(struct aead_request *req) +{ + req = crypto_rfc4543_crypt(req, 0); + + return crypto_aead_decrypt(req); +} + +static int crypto_rfc4543_init_tfm(struct crypto_tfm *tfm) +{ + struct crypto_instance *inst = (void *)tfm->__crt_alg; + struct crypto_aead_spawn *spawn = crypto_instance_ctx(inst); + struct crypto_rfc4543_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_aead *aead; + unsigned long align; + + aead = crypto_spawn_aead(spawn); + if (IS_ERR(aead)) + return PTR_ERR(aead); + + ctx->child = aead; + + align = crypto_aead_alignmask(aead); + align &= ~(crypto_tfm_ctx_alignment() - 1); + tfm->crt_aead.reqsize = sizeof(struct crypto_rfc4543_req_ctx) + + ALIGN(crypto_aead_reqsize(aead), + crypto_tfm_ctx_alignment()) + + align + 16; + + return 0; +} + +static void crypto_rfc4543_exit_tfm(struct crypto_tfm *tfm) +{ + struct crypto_rfc4543_ctx *ctx = crypto_tfm_ctx(tfm); + + crypto_free_aead(ctx->child); +} + +static struct crypto_instance *crypto_rfc4543_alloc(struct rtattr **tb) +{ + struct crypto_attr_type *algt; + struct crypto_instance *inst; + struct crypto_aead_spawn *spawn; + struct crypto_alg *alg; + const char *ccm_name; + int err; + + algt = crypto_get_attr_type(tb); + err = PTR_ERR(algt); + if (IS_ERR(algt)) + return ERR_PTR(err); + + if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) + return ERR_PTR(-EINVAL); + + ccm_name = crypto_attr_alg_name(tb[1]); + err = PTR_ERR(ccm_name); + if (IS_ERR(ccm_name)) + return ERR_PTR(err); + + inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); + if (!inst) + return ERR_PTR(-ENOMEM); + + spawn = crypto_instance_ctx(inst); + crypto_set_aead_spawn(spawn, inst); + err = crypto_grab_aead(spawn, ccm_name, 0, + crypto_requires_sync(algt->type, algt->mask)); + if (err) + goto out_free_inst; + + alg = crypto_aead_spawn_alg(spawn); + + err = -EINVAL; + + /* We only support 16-byte blocks. */ + if (alg->cra_aead.ivsize != 16) + goto out_drop_alg; + + /* Not a stream cipher? */ + if (alg->cra_blocksize != 1) + goto out_drop_alg; + + err = -ENAMETOOLONG; + if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, + "rfc4543(%s)", alg->cra_name) >= CRYPTO_MAX_ALG_NAME || + snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, + "rfc4543(%s)", alg->cra_driver_name) >= + CRYPTO_MAX_ALG_NAME) + goto out_drop_alg; + + inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD; + inst->alg.cra_flags |= alg->cra_flags & CRYPTO_ALG_ASYNC; + inst->alg.cra_priority = alg->cra_priority; + inst->alg.cra_blocksize = 1; + inst->alg.cra_alignmask = alg->cra_alignmask; + inst->alg.cra_type = &crypto_nivaead_type; + + inst->alg.cra_aead.ivsize = 8; + inst->alg.cra_aead.maxauthsize = 16; + + inst->alg.cra_ctxsize = sizeof(struct crypto_rfc4543_ctx); + + inst->alg.cra_init = crypto_rfc4543_init_tfm; + inst->alg.cra_exit = crypto_rfc4543_exit_tfm; + + inst->alg.cra_aead.setkey = crypto_rfc4543_setkey; + inst->alg.cra_aead.setauthsize = crypto_rfc4543_setauthsize; + inst->alg.cra_aead.encrypt = crypto_rfc4543_encrypt; + inst->alg.cra_aead.decrypt = crypto_rfc4543_decrypt; + + inst->alg.cra_aead.geniv = "seqiv"; + +out: + return inst; + +out_drop_alg: + crypto_drop_aead(spawn); +out_free_inst: + kfree(inst); + inst = ERR_PTR(err); + goto out; +} + +static void crypto_rfc4543_free(struct crypto_instance *inst) +{ + crypto_drop_spawn(crypto_instance_ctx(inst)); + kfree(inst); +} + +static struct crypto_template crypto_rfc4543_tmpl = { + .name = "rfc4543", + .alloc = crypto_rfc4543_alloc, + .free = crypto_rfc4543_free, + .module = THIS_MODULE, +}; + static int __init crypto_gcm_module_init(void) { int err; + gcm_zeroes = kzalloc(16, GFP_KERNEL); + if (!gcm_zeroes) + return -ENOMEM; + err = crypto_register_template(&crypto_gcm_base_tmpl); if (err) goto out; @@ -796,18 +1346,27 @@ static int __init crypto_gcm_module_init(void) if (err) goto out_undo_gcm; -out: - return err; + err = crypto_register_template(&crypto_rfc4543_tmpl); + if (err) + goto out_undo_rfc4106; + return 0; + +out_undo_rfc4106: + crypto_unregister_template(&crypto_rfc4106_tmpl); out_undo_gcm: crypto_unregister_template(&crypto_gcm_tmpl); out_undo_base: crypto_unregister_template(&crypto_gcm_base_tmpl); - goto out; +out: + kfree(gcm_zeroes); + return err; } static void __exit crypto_gcm_module_exit(void) { + kfree(gcm_zeroes); + crypto_unregister_template(&crypto_rfc4543_tmpl); crypto_unregister_template(&crypto_rfc4106_tmpl); crypto_unregister_template(&crypto_gcm_tmpl); crypto_unregister_template(&crypto_gcm_base_tmpl); @@ -821,3 +1380,4 @@ MODULE_DESCRIPTION("Galois/Counter Mode"); MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>"); MODULE_ALIAS("gcm_base"); MODULE_ALIAS("rfc4106"); +MODULE_ALIAS("rfc4543"); diff --git a/crypto/ghash-generic.c b/crypto/ghash-generic.c new file mode 100644 index 000000000000..be4425616931 --- /dev/null +++ b/crypto/ghash-generic.c @@ -0,0 +1,170 @@ +/* + * GHASH: digest algorithm for GCM (Galois/Counter Mode). + * + * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi> + * Copyright (c) 2009 Intel Corp. + * Author: Huang Ying <ying.huang@intel.com> + * + * The algorithm implementation is copied from gcm.c. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + */ + +#include <crypto/algapi.h> +#include <crypto/gf128mul.h> +#include <crypto/internal/hash.h> +#include <linux/crypto.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> + +#define GHASH_BLOCK_SIZE 16 +#define GHASH_DIGEST_SIZE 16 + +struct ghash_ctx { + struct gf128mul_4k *gf128; +}; + +struct ghash_desc_ctx { + u8 buffer[GHASH_BLOCK_SIZE]; + u32 bytes; +}; + +static int ghash_init(struct shash_desc *desc) +{ + struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); + + memset(dctx, 0, sizeof(*dctx)); + + return 0; +} + +static int ghash_setkey(struct crypto_shash *tfm, + const u8 *key, unsigned int keylen) +{ + struct ghash_ctx *ctx = crypto_shash_ctx(tfm); + + if (keylen != GHASH_BLOCK_SIZE) { + crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + if (ctx->gf128) + gf128mul_free_4k(ctx->gf128); + ctx->gf128 = gf128mul_init_4k_lle((be128 *)key); + if (!ctx->gf128) + return -ENOMEM; + + return 0; +} + +static int ghash_update(struct shash_desc *desc, + const u8 *src, unsigned int srclen) +{ + struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); + struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); + u8 *dst = dctx->buffer; + + if (dctx->bytes) { + int n = min(srclen, dctx->bytes); + u8 *pos = dst + (GHASH_BLOCK_SIZE - dctx->bytes); + + dctx->bytes -= n; + srclen -= n; + + while (n--) + *pos++ ^= *src++; + + if (!dctx->bytes) + gf128mul_4k_lle((be128 *)dst, ctx->gf128); + } + + while (srclen >= GHASH_BLOCK_SIZE) { + crypto_xor(dst, src, GHASH_BLOCK_SIZE); + gf128mul_4k_lle((be128 *)dst, ctx->gf128); + src += GHASH_BLOCK_SIZE; + srclen -= GHASH_BLOCK_SIZE; + } + + if (srclen) { + dctx->bytes = GHASH_BLOCK_SIZE - srclen; + while (srclen--) + *dst++ ^= *src++; + } + + return 0; +} + +static void ghash_flush(struct ghash_ctx *ctx, struct ghash_desc_ctx *dctx) +{ + u8 *dst = dctx->buffer; + + if (dctx->bytes) { + u8 *tmp = dst + (GHASH_BLOCK_SIZE - dctx->bytes); + + while (dctx->bytes--) + *tmp++ ^= 0; + + gf128mul_4k_lle((be128 *)dst, ctx->gf128); + } + + dctx->bytes = 0; +} + +static int ghash_final(struct shash_desc *desc, u8 *dst) +{ + struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); + struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); + u8 *buf = dctx->buffer; + + ghash_flush(ctx, dctx); + memcpy(dst, buf, GHASH_BLOCK_SIZE); + + return 0; +} + +static void ghash_exit_tfm(struct crypto_tfm *tfm) +{ + struct ghash_ctx *ctx = crypto_tfm_ctx(tfm); + if (ctx->gf128) + gf128mul_free_4k(ctx->gf128); +} + +static struct shash_alg ghash_alg = { + .digestsize = GHASH_DIGEST_SIZE, + .init = ghash_init, + .update = ghash_update, + .final = ghash_final, + .setkey = ghash_setkey, + .descsize = sizeof(struct ghash_desc_ctx), + .base = { + .cra_name = "ghash", + .cra_driver_name = "ghash-generic", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = GHASH_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ghash_ctx), + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(ghash_alg.base.cra_list), + .cra_exit = ghash_exit_tfm, + }, +}; + +static int __init ghash_mod_init(void) +{ + return crypto_register_shash(&ghash_alg); +} + +static void __exit ghash_mod_exit(void) +{ + crypto_unregister_shash(&ghash_alg); +} + +module_init(ghash_mod_init); +module_exit(ghash_mod_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("GHASH Message Digest Algorithm"); +MODULE_ALIAS("ghash"); diff --git a/crypto/hash.c b/crypto/hash.c deleted file mode 100644 index cb86b19fd105..000000000000 --- a/crypto/hash.c +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Cryptographic Hash operations. - * - * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) - * any later version. - */ - -#include <crypto/internal/hash.h> -#include <linux/errno.h> -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/slab.h> -#include <linux/seq_file.h> - -#include "internal.h" - -static unsigned int crypto_hash_ctxsize(struct crypto_alg *alg, u32 type, - u32 mask) -{ - return alg->cra_ctxsize; -} - -static int hash_setkey_unaligned(struct crypto_hash *crt, const u8 *key, - unsigned int keylen) -{ - struct crypto_tfm *tfm = crypto_hash_tfm(crt); - struct hash_alg *alg = &tfm->__crt_alg->cra_hash; - unsigned long alignmask = crypto_hash_alignmask(crt); - int ret; - u8 *buffer, *alignbuffer; - unsigned long absize; - - absize = keylen + alignmask; - buffer = kmalloc(absize, GFP_ATOMIC); - if (!buffer) - return -ENOMEM; - - alignbuffer = (u8 *)ALIGN((unsigned long)buffer, alignmask + 1); - memcpy(alignbuffer, key, keylen); - ret = alg->setkey(crt, alignbuffer, keylen); - memset(alignbuffer, 0, keylen); - kfree(buffer); - return ret; -} - -static int hash_setkey(struct crypto_hash *crt, const u8 *key, - unsigned int keylen) -{ - struct crypto_tfm *tfm = crypto_hash_tfm(crt); - struct hash_alg *alg = &tfm->__crt_alg->cra_hash; - unsigned long alignmask = crypto_hash_alignmask(crt); - - if ((unsigned long)key & alignmask) - return hash_setkey_unaligned(crt, key, keylen); - - return alg->setkey(crt, key, keylen); -} - -static int hash_async_setkey(struct crypto_ahash *tfm_async, const u8 *key, - unsigned int keylen) -{ - struct crypto_tfm *tfm = crypto_ahash_tfm(tfm_async); - struct crypto_hash *tfm_hash = __crypto_hash_cast(tfm); - struct hash_alg *alg = &tfm->__crt_alg->cra_hash; - - return alg->setkey(tfm_hash, key, keylen); -} - -static int hash_async_init(struct ahash_request *req) -{ - struct crypto_tfm *tfm = req->base.tfm; - struct hash_alg *alg = &tfm->__crt_alg->cra_hash; - struct hash_desc desc = { - .tfm = __crypto_hash_cast(tfm), - .flags = req->base.flags, - }; - - return alg->init(&desc); -} - -static int hash_async_update(struct ahash_request *req) -{ - struct crypto_tfm *tfm = req->base.tfm; - struct hash_alg *alg = &tfm->__crt_alg->cra_hash; - struct hash_desc desc = { - .tfm = __crypto_hash_cast(tfm), - .flags = req->base.flags, - }; - - return alg->update(&desc, req->src, req->nbytes); -} - -static int hash_async_final(struct ahash_request *req) -{ - struct crypto_tfm *tfm = req->base.tfm; - struct hash_alg *alg = &tfm->__crt_alg->cra_hash; - struct hash_desc desc = { - .tfm = __crypto_hash_cast(tfm), - .flags = req->base.flags, - }; - - return alg->final(&desc, req->result); -} - -static int hash_async_digest(struct ahash_request *req) -{ - struct crypto_tfm *tfm = req->base.tfm; - struct hash_alg *alg = &tfm->__crt_alg->cra_hash; - struct hash_desc desc = { - .tfm = __crypto_hash_cast(tfm), - .flags = req->base.flags, - }; - - return alg->digest(&desc, req->src, req->nbytes, req->result); -} - -static int crypto_init_hash_ops_async(struct crypto_tfm *tfm) -{ - struct ahash_tfm *crt = &tfm->crt_ahash; - struct hash_alg *alg = &tfm->__crt_alg->cra_hash; - - crt->init = hash_async_init; - crt->update = hash_async_update; - crt->final = hash_async_final; - crt->digest = hash_async_digest; - crt->setkey = hash_async_setkey; - crt->digestsize = alg->digestsize; - - return 0; -} - -static int crypto_init_hash_ops_sync(struct crypto_tfm *tfm) -{ - struct hash_tfm *crt = &tfm->crt_hash; - struct hash_alg *alg = &tfm->__crt_alg->cra_hash; - - crt->init = alg->init; - crt->update = alg->update; - crt->final = alg->final; - crt->digest = alg->digest; - crt->setkey = hash_setkey; - crt->digestsize = alg->digestsize; - - return 0; -} - -static int crypto_init_hash_ops(struct crypto_tfm *tfm, u32 type, u32 mask) -{ - struct hash_alg *alg = &tfm->__crt_alg->cra_hash; - - if (alg->digestsize > PAGE_SIZE / 8) - return -EINVAL; - - if ((mask & CRYPTO_ALG_TYPE_HASH_MASK) != CRYPTO_ALG_TYPE_HASH_MASK) - return crypto_init_hash_ops_async(tfm); - else - return crypto_init_hash_ops_sync(tfm); -} - -static void crypto_hash_show(struct seq_file *m, struct crypto_alg *alg) - __attribute__ ((unused)); -static void crypto_hash_show(struct seq_file *m, struct crypto_alg *alg) -{ - seq_printf(m, "type : hash\n"); - seq_printf(m, "blocksize : %u\n", alg->cra_blocksize); - seq_printf(m, "digestsize : %u\n", alg->cra_hash.digestsize); -} - -const struct crypto_type crypto_hash_type = { - .ctxsize = crypto_hash_ctxsize, - .init = crypto_init_hash_ops, -#ifdef CONFIG_PROC_FS - .show = crypto_hash_show, -#endif -}; -EXPORT_SYMBOL_GPL(crypto_hash_type); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Generic cryptographic hash type"); diff --git a/crypto/hmac.c b/crypto/hmac.c index 0ad39c374963..8d9544cf8169 100644 --- a/crypto/hmac.c +++ b/crypto/hmac.c @@ -23,11 +23,10 @@ #include <linux/kernel.h> #include <linux/module.h> #include <linux/scatterlist.h> -#include <linux/slab.h> #include <linux/string.h> struct hmac_ctx { - struct crypto_hash *child; + struct crypto_shash *hash; }; static inline void *align_ptr(void *p, unsigned int align) @@ -35,65 +34,45 @@ static inline void *align_ptr(void *p, unsigned int align) return (void *)ALIGN((unsigned long)p, align); } -static inline struct hmac_ctx *hmac_ctx(struct crypto_hash *tfm) +static inline struct hmac_ctx *hmac_ctx(struct crypto_shash *tfm) { - return align_ptr(crypto_hash_ctx_aligned(tfm) + - crypto_hash_blocksize(tfm) * 2 + - crypto_hash_digestsize(tfm), sizeof(void *)); + return align_ptr(crypto_shash_ctx_aligned(tfm) + + crypto_shash_statesize(tfm) * 2, + crypto_tfm_ctx_alignment()); } -static int hmac_setkey(struct crypto_hash *parent, +static int hmac_setkey(struct crypto_shash *parent, const u8 *inkey, unsigned int keylen) { - int bs = crypto_hash_blocksize(parent); - int ds = crypto_hash_digestsize(parent); - char *ipad = crypto_hash_ctx_aligned(parent); - char *opad = ipad + bs; - char *digest = opad + bs; - struct hmac_ctx *ctx = align_ptr(digest + ds, sizeof(void *)); - struct crypto_hash *tfm = ctx->child; + int bs = crypto_shash_blocksize(parent); + int ds = crypto_shash_digestsize(parent); + int ss = crypto_shash_statesize(parent); + char *ipad = crypto_shash_ctx_aligned(parent); + char *opad = ipad + ss; + struct hmac_ctx *ctx = align_ptr(opad + ss, + crypto_tfm_ctx_alignment()); + struct crypto_shash *hash = ctx->hash; + struct { + struct shash_desc shash; + char ctx[crypto_shash_descsize(hash)]; + } desc; unsigned int i; + desc.shash.tfm = hash; + desc.shash.flags = crypto_shash_get_flags(parent) & + CRYPTO_TFM_REQ_MAY_SLEEP; + if (keylen > bs) { - struct hash_desc desc; - struct scatterlist tmp; - int tmplen; int err; - desc.tfm = tfm; - desc.flags = crypto_hash_get_flags(parent); - desc.flags &= CRYPTO_TFM_REQ_MAY_SLEEP; - - err = crypto_hash_init(&desc); + err = crypto_shash_digest(&desc.shash, inkey, keylen, ipad); if (err) return err; - tmplen = bs * 2 + ds; - sg_init_one(&tmp, ipad, tmplen); - - for (; keylen > tmplen; inkey += tmplen, keylen -= tmplen) { - memcpy(ipad, inkey, tmplen); - err = crypto_hash_update(&desc, &tmp, tmplen); - if (err) - return err; - } - - if (keylen) { - memcpy(ipad, inkey, keylen); - err = crypto_hash_update(&desc, &tmp, keylen); - if (err) - return err; - } - - err = crypto_hash_final(&desc, digest); - if (err) - return err; - - inkey = digest; keylen = ds; - } + } else + memcpy(ipad, inkey, keylen); - memcpy(ipad, inkey, keylen); memset(ipad + keylen, 0, bs - keylen); memcpy(opad, ipad, bs); @@ -102,184 +81,178 @@ static int hmac_setkey(struct crypto_hash *parent, opad[i] ^= 0x5c; } - return 0; + return crypto_shash_init(&desc.shash) ?: + crypto_shash_update(&desc.shash, ipad, bs) ?: + crypto_shash_export(&desc.shash, ipad) ?: + crypto_shash_init(&desc.shash) ?: + crypto_shash_update(&desc.shash, opad, bs) ?: + crypto_shash_export(&desc.shash, opad); } -static int hmac_init(struct hash_desc *pdesc) +static int hmac_export(struct shash_desc *pdesc, void *out) { - struct crypto_hash *parent = pdesc->tfm; - int bs = crypto_hash_blocksize(parent); - int ds = crypto_hash_digestsize(parent); - char *ipad = crypto_hash_ctx_aligned(parent); - struct hmac_ctx *ctx = align_ptr(ipad + bs * 2 + ds, sizeof(void *)); - struct hash_desc desc; - struct scatterlist tmp; - int err; + struct shash_desc *desc = shash_desc_ctx(pdesc); - desc.tfm = ctx->child; - desc.flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; - sg_init_one(&tmp, ipad, bs); + desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; - err = crypto_hash_init(&desc); - if (unlikely(err)) - return err; - - return crypto_hash_update(&desc, &tmp, bs); + return crypto_shash_export(desc, out); } -static int hmac_update(struct hash_desc *pdesc, - struct scatterlist *sg, unsigned int nbytes) +static int hmac_import(struct shash_desc *pdesc, const void *in) { + struct shash_desc *desc = shash_desc_ctx(pdesc); struct hmac_ctx *ctx = hmac_ctx(pdesc->tfm); - struct hash_desc desc; - desc.tfm = ctx->child; - desc.flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; + desc->tfm = ctx->hash; + desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; - return crypto_hash_update(&desc, sg, nbytes); + return crypto_shash_import(desc, in); } -static int hmac_final(struct hash_desc *pdesc, u8 *out) +static int hmac_init(struct shash_desc *pdesc) { - struct crypto_hash *parent = pdesc->tfm; - int bs = crypto_hash_blocksize(parent); - int ds = crypto_hash_digestsize(parent); - char *opad = crypto_hash_ctx_aligned(parent) + bs; - char *digest = opad + bs; - struct hmac_ctx *ctx = align_ptr(digest + ds, sizeof(void *)); - struct hash_desc desc; - struct scatterlist tmp; - int err; + return hmac_import(pdesc, crypto_shash_ctx_aligned(pdesc->tfm)); +} - desc.tfm = ctx->child; - desc.flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; - sg_init_one(&tmp, opad, bs + ds); +static int hmac_update(struct shash_desc *pdesc, + const u8 *data, unsigned int nbytes) +{ + struct shash_desc *desc = shash_desc_ctx(pdesc); - err = crypto_hash_final(&desc, digest); - if (unlikely(err)) - return err; + desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; - return crypto_hash_digest(&desc, &tmp, bs + ds, out); + return crypto_shash_update(desc, data, nbytes); } -static int hmac_digest(struct hash_desc *pdesc, struct scatterlist *sg, - unsigned int nbytes, u8 *out) +static int hmac_final(struct shash_desc *pdesc, u8 *out) { - struct crypto_hash *parent = pdesc->tfm; - int bs = crypto_hash_blocksize(parent); - int ds = crypto_hash_digestsize(parent); - char *ipad = crypto_hash_ctx_aligned(parent); - char *opad = ipad + bs; - char *digest = opad + bs; - struct hmac_ctx *ctx = align_ptr(digest + ds, sizeof(void *)); - struct hash_desc desc; - struct scatterlist sg1[2]; - struct scatterlist sg2[1]; - int err; + struct crypto_shash *parent = pdesc->tfm; + int ds = crypto_shash_digestsize(parent); + int ss = crypto_shash_statesize(parent); + char *opad = crypto_shash_ctx_aligned(parent) + ss; + struct shash_desc *desc = shash_desc_ctx(pdesc); - desc.tfm = ctx->child; - desc.flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; + desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; - sg_init_table(sg1, 2); - sg_set_buf(sg1, ipad, bs); - scatterwalk_sg_chain(sg1, 2, sg); + return crypto_shash_final(desc, out) ?: + crypto_shash_import(desc, opad) ?: + crypto_shash_finup(desc, out, ds, out); +} - sg_init_table(sg2, 1); - sg_set_buf(sg2, opad, bs + ds); +static int hmac_finup(struct shash_desc *pdesc, const u8 *data, + unsigned int nbytes, u8 *out) +{ - err = crypto_hash_digest(&desc, sg1, nbytes + bs, digest); - if (unlikely(err)) - return err; + struct crypto_shash *parent = pdesc->tfm; + int ds = crypto_shash_digestsize(parent); + int ss = crypto_shash_statesize(parent); + char *opad = crypto_shash_ctx_aligned(parent) + ss; + struct shash_desc *desc = shash_desc_ctx(pdesc); - return crypto_hash_digest(&desc, sg2, bs + ds, out); + desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; + + return crypto_shash_finup(desc, data, nbytes, out) ?: + crypto_shash_import(desc, opad) ?: + crypto_shash_finup(desc, out, ds, out); } static int hmac_init_tfm(struct crypto_tfm *tfm) { - struct crypto_hash *hash; + struct crypto_shash *parent = __crypto_shash_cast(tfm); + struct crypto_shash *hash; struct crypto_instance *inst = (void *)tfm->__crt_alg; - struct crypto_spawn *spawn = crypto_instance_ctx(inst); - struct hmac_ctx *ctx = hmac_ctx(__crypto_hash_cast(tfm)); + struct crypto_shash_spawn *spawn = crypto_instance_ctx(inst); + struct hmac_ctx *ctx = hmac_ctx(parent); - hash = crypto_spawn_hash(spawn); + hash = crypto_spawn_shash(spawn); if (IS_ERR(hash)) return PTR_ERR(hash); - ctx->child = hash; + parent->descsize = sizeof(struct shash_desc) + + crypto_shash_descsize(hash); + + ctx->hash = hash; return 0; } static void hmac_exit_tfm(struct crypto_tfm *tfm) { - struct hmac_ctx *ctx = hmac_ctx(__crypto_hash_cast(tfm)); - crypto_free_hash(ctx->child); + struct hmac_ctx *ctx = hmac_ctx(__crypto_shash_cast(tfm)); + crypto_free_shash(ctx->hash); } -static void hmac_free(struct crypto_instance *inst) +static int hmac_create(struct crypto_template *tmpl, struct rtattr **tb) { - crypto_drop_spawn(crypto_instance_ctx(inst)); - kfree(inst); -} - -static struct crypto_instance *hmac_alloc(struct rtattr **tb) -{ - struct crypto_instance *inst; + struct shash_instance *inst; struct crypto_alg *alg; + struct shash_alg *salg; int err; int ds; + int ss; - err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_HASH); + err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH); if (err) - return ERR_PTR(err); - - alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_HASH, - CRYPTO_ALG_TYPE_HASH_MASK); - if (IS_ERR(alg)) - return ERR_CAST(alg); - - inst = ERR_PTR(-EINVAL); - ds = alg->cra_type == &crypto_hash_type ? - alg->cra_hash.digestsize : - alg->cra_type ? - __crypto_shash_alg(alg)->digestsize : - alg->cra_digest.dia_digestsize; - if (ds > alg->cra_blocksize) + return err; + + salg = shash_attr_alg(tb[1], 0, 0); + if (IS_ERR(salg)) + return PTR_ERR(salg); + + err = -EINVAL; + ds = salg->digestsize; + ss = salg->statesize; + alg = &salg->base; + if (ds > alg->cra_blocksize || + ss < alg->cra_blocksize) goto out_put_alg; - inst = crypto_alloc_instance("hmac", alg); + inst = shash_alloc_instance("hmac", alg); + err = PTR_ERR(inst); if (IS_ERR(inst)) goto out_put_alg; - inst->alg.cra_flags = CRYPTO_ALG_TYPE_HASH; - inst->alg.cra_priority = alg->cra_priority; - inst->alg.cra_blocksize = alg->cra_blocksize; - inst->alg.cra_alignmask = alg->cra_alignmask; - inst->alg.cra_type = &crypto_hash_type; - - inst->alg.cra_hash.digestsize = ds; - - inst->alg.cra_ctxsize = sizeof(struct hmac_ctx) + - ALIGN(inst->alg.cra_blocksize * 2 + ds, - sizeof(void *)); - - inst->alg.cra_init = hmac_init_tfm; - inst->alg.cra_exit = hmac_exit_tfm; - - inst->alg.cra_hash.init = hmac_init; - inst->alg.cra_hash.update = hmac_update; - inst->alg.cra_hash.final = hmac_final; - inst->alg.cra_hash.digest = hmac_digest; - inst->alg.cra_hash.setkey = hmac_setkey; + err = crypto_init_shash_spawn(shash_instance_ctx(inst), salg, + shash_crypto_instance(inst)); + if (err) + goto out_free_inst; + + inst->alg.base.cra_priority = alg->cra_priority; + inst->alg.base.cra_blocksize = alg->cra_blocksize; + inst->alg.base.cra_alignmask = alg->cra_alignmask; + + ss = ALIGN(ss, alg->cra_alignmask + 1); + inst->alg.digestsize = ds; + inst->alg.statesize = ss; + + inst->alg.base.cra_ctxsize = sizeof(struct hmac_ctx) + + ALIGN(ss * 2, crypto_tfm_ctx_alignment()); + + inst->alg.base.cra_init = hmac_init_tfm; + inst->alg.base.cra_exit = hmac_exit_tfm; + + inst->alg.init = hmac_init; + inst->alg.update = hmac_update; + inst->alg.final = hmac_final; + inst->alg.finup = hmac_finup; + inst->alg.export = hmac_export; + inst->alg.import = hmac_import; + inst->alg.setkey = hmac_setkey; + + err = shash_register_instance(tmpl, inst); + if (err) { +out_free_inst: + shash_free_instance(shash_crypto_instance(inst)); + } out_put_alg: crypto_mod_put(alg); - return inst; + return err; } static struct crypto_template hmac_tmpl = { .name = "hmac", - .alloc = hmac_alloc, - .free = hmac_free, + .create = hmac_create, + .free = shash_free_instance, .module = THIS_MODULE, }; diff --git a/crypto/internal.h b/crypto/internal.h index 113579a82dff..d4384b08ab29 100644 --- a/crypto/internal.h +++ b/crypto/internal.h @@ -6,7 +6,7 @@ * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) + * Software Foundation; either version 2 of the License, or (at your option) * any later version. * */ @@ -25,12 +25,7 @@ #include <linux/notifier.h> #include <linux/rwsem.h> #include <linux/slab.h> - -#ifdef CONFIG_CRYPTO_FIPS -extern int fips_enabled; -#else -#define fips_enabled 0 -#endif +#include <linux/fips.h> /* Crypto notification events. */ enum { @@ -65,18 +60,6 @@ static inline void crypto_exit_proc(void) { } #endif -static inline unsigned int crypto_digest_ctxsize(struct crypto_alg *alg) -{ - unsigned int len = alg->cra_ctxsize; - - if (alg->cra_alignmask) { - len = ALIGN(len, (unsigned long)alg->cra_alignmask + 1); - len += alg->cra_digest.dia_digestsize; - } - - return len; -} - static inline unsigned int crypto_cipher_ctxsize(struct crypto_alg *alg) { return alg->cra_ctxsize; @@ -91,12 +74,9 @@ struct crypto_alg *crypto_mod_get(struct crypto_alg *alg); struct crypto_alg *crypto_alg_lookup(const char *name, u32 type, u32 mask); struct crypto_alg *crypto_alg_mod_lookup(const char *name, u32 type, u32 mask); -int crypto_init_digest_ops(struct crypto_tfm *tfm); -int crypto_init_digest_ops_async(struct crypto_tfm *tfm); int crypto_init_cipher_ops(struct crypto_tfm *tfm); int crypto_init_compress_ops(struct crypto_tfm *tfm); -void crypto_exit_digest_ops(struct crypto_tfm *tfm); void crypto_exit_cipher_ops(struct crypto_tfm *tfm); void crypto_exit_compress_ops(struct crypto_tfm *tfm); @@ -111,12 +91,12 @@ struct crypto_tfm *__crypto_alloc_tfm(struct crypto_alg *alg, u32 type, u32 mask); void *crypto_create_tfm(struct crypto_alg *alg, const struct crypto_type *frontend); +struct crypto_alg *crypto_find_alg(const char *alg_name, + const struct crypto_type *frontend, + u32 type, u32 mask); void *crypto_alloc_tfm(const char *alg_name, const struct crypto_type *frontend, u32 type, u32 mask); -int crypto_register_instance(struct crypto_template *tmpl, - struct crypto_instance *inst); - int crypto_register_notifier(struct notifier_block *nb); int crypto_unregister_notifier(struct notifier_block *nb); int crypto_probing_notify(unsigned long val, void *v); diff --git a/crypto/md5.c b/crypto/md5.c index 83eb52961750..30efc7dad891 100644 --- a/crypto/md5.c +++ b/crypto/md5.c @@ -16,17 +16,13 @@ * */ #include <crypto/internal/hash.h> +#include <crypto/md5.h> #include <linux/init.h> #include <linux/module.h> #include <linux/string.h> #include <linux/types.h> #include <asm/byteorder.h> -#define MD5_DIGEST_SIZE 16 -#define MD5_HMAC_BLOCK_SIZE 64 -#define MD5_BLOCK_WORDS 16 -#define MD5_HASH_WORDS 4 - #define F1(x, y, z) (z ^ (x & (y ^ z))) #define F2(x, y, z) F1(z, x, y) #define F3(x, y, z) (x ^ y ^ z) @@ -35,12 +31,6 @@ #define MD5STEP(f, w, x, y, z, in, s) \ (w += f(x, y, z) + in, w = (w<<s | w>>(32-s)) + x) -struct md5_ctx { - u32 hash[MD5_HASH_WORDS]; - u32 block[MD5_BLOCK_WORDS]; - u64 byte_count; -}; - static void md5_transform(u32 *hash, u32 const *in) { u32 a, b, c, d; @@ -141,7 +131,7 @@ static inline void cpu_to_le32_array(u32 *buf, unsigned int words) } } -static inline void md5_transform_helper(struct md5_ctx *ctx) +static inline void md5_transform_helper(struct md5_state *ctx) { le32_to_cpu_array(ctx->block, sizeof(ctx->block) / sizeof(u32)); md5_transform(ctx->hash, ctx->block); @@ -149,7 +139,7 @@ static inline void md5_transform_helper(struct md5_ctx *ctx) static int md5_init(struct shash_desc *desc) { - struct md5_ctx *mctx = shash_desc_ctx(desc); + struct md5_state *mctx = shash_desc_ctx(desc); mctx->hash[0] = 0x67452301; mctx->hash[1] = 0xefcdab89; @@ -162,7 +152,7 @@ static int md5_init(struct shash_desc *desc) static int md5_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct md5_ctx *mctx = shash_desc_ctx(desc); + struct md5_state *mctx = shash_desc_ctx(desc); const u32 avail = sizeof(mctx->block) - (mctx->byte_count & 0x3f); mctx->byte_count += len; @@ -194,7 +184,7 @@ static int md5_update(struct shash_desc *desc, const u8 *data, unsigned int len) static int md5_final(struct shash_desc *desc, u8 *out) { - struct md5_ctx *mctx = shash_desc_ctx(desc); + struct md5_state *mctx = shash_desc_ctx(desc); const unsigned int offset = mctx->byte_count & 0x3f; char *p = (char *)mctx->block + offset; int padding = 56 - (offset + 1); @@ -220,12 +210,31 @@ static int md5_final(struct shash_desc *desc, u8 *out) return 0; } +static int md5_export(struct shash_desc *desc, void *out) +{ + struct md5_state *ctx = shash_desc_ctx(desc); + + memcpy(out, ctx, sizeof(*ctx)); + return 0; +} + +static int md5_import(struct shash_desc *desc, const void *in) +{ + struct md5_state *ctx = shash_desc_ctx(desc); + + memcpy(ctx, in, sizeof(*ctx)); + return 0; +} + static struct shash_alg alg = { .digestsize = MD5_DIGEST_SIZE, .init = md5_init, .update = md5_update, .final = md5_final, - .descsize = sizeof(struct md5_ctx), + .export = md5_export, + .import = md5_import, + .descsize = sizeof(struct md5_state), + .statesize = sizeof(struct md5_state), .base = { .cra_name = "md5", .cra_flags = CRYPTO_ALG_TYPE_SHASH, diff --git a/crypto/pcompress.c b/crypto/pcompress.c index bcadc03726b7..f7c4a7d7412e 100644 --- a/crypto/pcompress.c +++ b/crypto/pcompress.c @@ -36,14 +36,12 @@ static int crypto_pcomp_init(struct crypto_tfm *tfm, u32 type, u32 mask) return 0; } -static unsigned int crypto_pcomp_extsize(struct crypto_alg *alg, - const struct crypto_type *frontend) +static unsigned int crypto_pcomp_extsize(struct crypto_alg *alg) { return alg->cra_ctxsize; } -static int crypto_pcomp_init_tfm(struct crypto_tfm *tfm, - const struct crypto_type *frontend) +static int crypto_pcomp_init_tfm(struct crypto_tfm *tfm) { return 0; } diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c new file mode 100644 index 000000000000..75586f1f86e7 --- /dev/null +++ b/crypto/pcrypt.c @@ -0,0 +1,566 @@ +/* + * pcrypt - Parallel crypto wrapper. + * + * Copyright (C) 2009 secunet Security Networks AG + * Copyright (C) 2009 Steffen Klassert <steffen.klassert@secunet.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + */ + +#include <crypto/algapi.h> +#include <crypto/internal/aead.h> +#include <linux/err.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/notifier.h> +#include <linux/kobject.h> +#include <linux/cpu.h> +#include <crypto/pcrypt.h> + +struct padata_pcrypt { + struct padata_instance *pinst; + struct workqueue_struct *wq; + + /* + * Cpumask for callback CPUs. It should be + * equal to serial cpumask of corresponding padata instance, + * so it is updated when padata notifies us about serial + * cpumask change. + * + * cb_cpumask is protected by RCU. This fact prevents us from + * using cpumask_var_t directly because the actual type of + * cpumsak_var_t depends on kernel configuration(particularly on + * CONFIG_CPUMASK_OFFSTACK macro). Depending on the configuration + * cpumask_var_t may be either a pointer to the struct cpumask + * or a variable allocated on the stack. Thus we can not safely use + * cpumask_var_t with RCU operations such as rcu_assign_pointer or + * rcu_dereference. So cpumask_var_t is wrapped with struct + * pcrypt_cpumask which makes possible to use it with RCU. + */ + struct pcrypt_cpumask { + cpumask_var_t mask; + } *cb_cpumask; + struct notifier_block nblock; +}; + +static struct padata_pcrypt pencrypt; +static struct padata_pcrypt pdecrypt; +static struct kset *pcrypt_kset; + +struct pcrypt_instance_ctx { + struct crypto_spawn spawn; + unsigned int tfm_count; +}; + +struct pcrypt_aead_ctx { + struct crypto_aead *child; + unsigned int cb_cpu; +}; + +static int pcrypt_do_parallel(struct padata_priv *padata, unsigned int *cb_cpu, + struct padata_pcrypt *pcrypt) +{ + unsigned int cpu_index, cpu, i; + struct pcrypt_cpumask *cpumask; + + cpu = *cb_cpu; + + rcu_read_lock_bh(); + cpumask = rcu_dereference(pcrypt->cb_cpumask); + if (cpumask_test_cpu(cpu, cpumask->mask)) + goto out; + + if (!cpumask_weight(cpumask->mask)) + goto out; + + cpu_index = cpu % cpumask_weight(cpumask->mask); + + cpu = cpumask_first(cpumask->mask); + for (i = 0; i < cpu_index; i++) + cpu = cpumask_next(cpu, cpumask->mask); + + *cb_cpu = cpu; + +out: + rcu_read_unlock_bh(); + return padata_do_parallel(pcrypt->pinst, padata, cpu); +} + +static int pcrypt_aead_setkey(struct crypto_aead *parent, + const u8 *key, unsigned int keylen) +{ + struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent); + + return crypto_aead_setkey(ctx->child, key, keylen); +} + +static int pcrypt_aead_setauthsize(struct crypto_aead *parent, + unsigned int authsize) +{ + struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(parent); + + return crypto_aead_setauthsize(ctx->child, authsize); +} + +static void pcrypt_aead_serial(struct padata_priv *padata) +{ + struct pcrypt_request *preq = pcrypt_padata_request(padata); + struct aead_request *req = pcrypt_request_ctx(preq); + + aead_request_complete(req->base.data, padata->info); +} + +static void pcrypt_aead_giv_serial(struct padata_priv *padata) +{ + struct pcrypt_request *preq = pcrypt_padata_request(padata); + struct aead_givcrypt_request *req = pcrypt_request_ctx(preq); + + aead_request_complete(req->areq.base.data, padata->info); +} + +static void pcrypt_aead_done(struct crypto_async_request *areq, int err) +{ + struct aead_request *req = areq->data; + struct pcrypt_request *preq = aead_request_ctx(req); + struct padata_priv *padata = pcrypt_request_padata(preq); + + padata->info = err; + req->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + + padata_do_serial(padata); +} + +static void pcrypt_aead_enc(struct padata_priv *padata) +{ + struct pcrypt_request *preq = pcrypt_padata_request(padata); + struct aead_request *req = pcrypt_request_ctx(preq); + + padata->info = crypto_aead_encrypt(req); + + if (padata->info == -EINPROGRESS) + return; + + padata_do_serial(padata); +} + +static int pcrypt_aead_encrypt(struct aead_request *req) +{ + int err; + struct pcrypt_request *preq = aead_request_ctx(req); + struct aead_request *creq = pcrypt_request_ctx(preq); + struct padata_priv *padata = pcrypt_request_padata(preq); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead); + u32 flags = aead_request_flags(req); + + memset(padata, 0, sizeof(struct padata_priv)); + + padata->parallel = pcrypt_aead_enc; + padata->serial = pcrypt_aead_serial; + + aead_request_set_tfm(creq, ctx->child); + aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP, + pcrypt_aead_done, req); + aead_request_set_crypt(creq, req->src, req->dst, + req->cryptlen, req->iv); + aead_request_set_assoc(creq, req->assoc, req->assoclen); + + err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt); + if (!err) + return -EINPROGRESS; + + return err; +} + +static void pcrypt_aead_dec(struct padata_priv *padata) +{ + struct pcrypt_request *preq = pcrypt_padata_request(padata); + struct aead_request *req = pcrypt_request_ctx(preq); + + padata->info = crypto_aead_decrypt(req); + + if (padata->info == -EINPROGRESS) + return; + + padata_do_serial(padata); +} + +static int pcrypt_aead_decrypt(struct aead_request *req) +{ + int err; + struct pcrypt_request *preq = aead_request_ctx(req); + struct aead_request *creq = pcrypt_request_ctx(preq); + struct padata_priv *padata = pcrypt_request_padata(preq); + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead); + u32 flags = aead_request_flags(req); + + memset(padata, 0, sizeof(struct padata_priv)); + + padata->parallel = pcrypt_aead_dec; + padata->serial = pcrypt_aead_serial; + + aead_request_set_tfm(creq, ctx->child); + aead_request_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP, + pcrypt_aead_done, req); + aead_request_set_crypt(creq, req->src, req->dst, + req->cryptlen, req->iv); + aead_request_set_assoc(creq, req->assoc, req->assoclen); + + err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pdecrypt); + if (!err) + return -EINPROGRESS; + + return err; +} + +static void pcrypt_aead_givenc(struct padata_priv *padata) +{ + struct pcrypt_request *preq = pcrypt_padata_request(padata); + struct aead_givcrypt_request *req = pcrypt_request_ctx(preq); + + padata->info = crypto_aead_givencrypt(req); + + if (padata->info == -EINPROGRESS) + return; + + padata_do_serial(padata); +} + +static int pcrypt_aead_givencrypt(struct aead_givcrypt_request *req) +{ + int err; + struct aead_request *areq = &req->areq; + struct pcrypt_request *preq = aead_request_ctx(areq); + struct aead_givcrypt_request *creq = pcrypt_request_ctx(preq); + struct padata_priv *padata = pcrypt_request_padata(preq); + struct crypto_aead *aead = aead_givcrypt_reqtfm(req); + struct pcrypt_aead_ctx *ctx = crypto_aead_ctx(aead); + u32 flags = aead_request_flags(areq); + + memset(padata, 0, sizeof(struct padata_priv)); + + padata->parallel = pcrypt_aead_givenc; + padata->serial = pcrypt_aead_giv_serial; + + aead_givcrypt_set_tfm(creq, ctx->child); + aead_givcrypt_set_callback(creq, flags & ~CRYPTO_TFM_REQ_MAY_SLEEP, + pcrypt_aead_done, areq); + aead_givcrypt_set_crypt(creq, areq->src, areq->dst, + areq->cryptlen, areq->iv); + aead_givcrypt_set_assoc(creq, areq->assoc, areq->assoclen); + aead_givcrypt_set_giv(creq, req->giv, req->seq); + + err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt); + if (!err) + return -EINPROGRESS; + + return err; +} + +static int pcrypt_aead_init_tfm(struct crypto_tfm *tfm) +{ + int cpu, cpu_index; + struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); + struct pcrypt_instance_ctx *ictx = crypto_instance_ctx(inst); + struct pcrypt_aead_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_aead *cipher; + + ictx->tfm_count++; + + cpu_index = ictx->tfm_count % cpumask_weight(cpu_active_mask); + + ctx->cb_cpu = cpumask_first(cpu_active_mask); + for (cpu = 0; cpu < cpu_index; cpu++) + ctx->cb_cpu = cpumask_next(ctx->cb_cpu, cpu_active_mask); + + cipher = crypto_spawn_aead(crypto_instance_ctx(inst)); + + if (IS_ERR(cipher)) + return PTR_ERR(cipher); + + ctx->child = cipher; + tfm->crt_aead.reqsize = sizeof(struct pcrypt_request) + + sizeof(struct aead_givcrypt_request) + + crypto_aead_reqsize(cipher); + + return 0; +} + +static void pcrypt_aead_exit_tfm(struct crypto_tfm *tfm) +{ + struct pcrypt_aead_ctx *ctx = crypto_tfm_ctx(tfm); + + crypto_free_aead(ctx->child); +} + +static struct crypto_instance *pcrypt_alloc_instance(struct crypto_alg *alg) +{ + struct crypto_instance *inst; + struct pcrypt_instance_ctx *ctx; + int err; + + inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); + if (!inst) { + inst = ERR_PTR(-ENOMEM); + goto out; + } + + err = -ENAMETOOLONG; + if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, + "pcrypt(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) + goto out_free_inst; + + memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); + + ctx = crypto_instance_ctx(inst); + err = crypto_init_spawn(&ctx->spawn, alg, inst, + CRYPTO_ALG_TYPE_MASK); + if (err) + goto out_free_inst; + + inst->alg.cra_priority = alg->cra_priority + 100; + inst->alg.cra_blocksize = alg->cra_blocksize; + inst->alg.cra_alignmask = alg->cra_alignmask; + +out: + return inst; + +out_free_inst: + kfree(inst); + inst = ERR_PTR(err); + goto out; +} + +static struct crypto_instance *pcrypt_alloc_aead(struct rtattr **tb, + u32 type, u32 mask) +{ + struct crypto_instance *inst; + struct crypto_alg *alg; + + alg = crypto_get_attr_alg(tb, type, (mask & CRYPTO_ALG_TYPE_MASK)); + if (IS_ERR(alg)) + return ERR_CAST(alg); + + inst = pcrypt_alloc_instance(alg); + if (IS_ERR(inst)) + goto out_put_alg; + + inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC; + inst->alg.cra_type = &crypto_aead_type; + + inst->alg.cra_aead.ivsize = alg->cra_aead.ivsize; + inst->alg.cra_aead.geniv = alg->cra_aead.geniv; + inst->alg.cra_aead.maxauthsize = alg->cra_aead.maxauthsize; + + inst->alg.cra_ctxsize = sizeof(struct pcrypt_aead_ctx); + + inst->alg.cra_init = pcrypt_aead_init_tfm; + inst->alg.cra_exit = pcrypt_aead_exit_tfm; + + inst->alg.cra_aead.setkey = pcrypt_aead_setkey; + inst->alg.cra_aead.setauthsize = pcrypt_aead_setauthsize; + inst->alg.cra_aead.encrypt = pcrypt_aead_encrypt; + inst->alg.cra_aead.decrypt = pcrypt_aead_decrypt; + inst->alg.cra_aead.givencrypt = pcrypt_aead_givencrypt; + +out_put_alg: + crypto_mod_put(alg); + return inst; +} + +static struct crypto_instance *pcrypt_alloc(struct rtattr **tb) +{ + struct crypto_attr_type *algt; + + algt = crypto_get_attr_type(tb); + if (IS_ERR(algt)) + return ERR_CAST(algt); + + switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { + case CRYPTO_ALG_TYPE_AEAD: + return pcrypt_alloc_aead(tb, algt->type, algt->mask); + } + + return ERR_PTR(-EINVAL); +} + +static void pcrypt_free(struct crypto_instance *inst) +{ + struct pcrypt_instance_ctx *ctx = crypto_instance_ctx(inst); + + crypto_drop_spawn(&ctx->spawn); + kfree(inst); +} + +static int pcrypt_cpumask_change_notify(struct notifier_block *self, + unsigned long val, void *data) +{ + struct padata_pcrypt *pcrypt; + struct pcrypt_cpumask *new_mask, *old_mask; + struct padata_cpumask *cpumask = (struct padata_cpumask *)data; + + if (!(val & PADATA_CPU_SERIAL)) + return 0; + + pcrypt = container_of(self, struct padata_pcrypt, nblock); + new_mask = kmalloc(sizeof(*new_mask), GFP_KERNEL); + if (!new_mask) + return -ENOMEM; + if (!alloc_cpumask_var(&new_mask->mask, GFP_KERNEL)) { + kfree(new_mask); + return -ENOMEM; + } + + old_mask = pcrypt->cb_cpumask; + + cpumask_copy(new_mask->mask, cpumask->cbcpu); + rcu_assign_pointer(pcrypt->cb_cpumask, new_mask); + synchronize_rcu_bh(); + + free_cpumask_var(old_mask->mask); + kfree(old_mask); + return 0; +} + +static int pcrypt_sysfs_add(struct padata_instance *pinst, const char *name) +{ + int ret; + + pinst->kobj.kset = pcrypt_kset; + ret = kobject_add(&pinst->kobj, NULL, name); + if (!ret) + kobject_uevent(&pinst->kobj, KOBJ_ADD); + + return ret; +} + +static int pcrypt_init_padata(struct padata_pcrypt *pcrypt, + const char *name) +{ + int ret = -ENOMEM; + struct pcrypt_cpumask *mask; + + get_online_cpus(); + + pcrypt->wq = create_workqueue(name); + if (!pcrypt->wq) + goto err; + + pcrypt->pinst = padata_alloc_possible(pcrypt->wq); + if (!pcrypt->pinst) + goto err_destroy_workqueue; + + mask = kmalloc(sizeof(*mask), GFP_KERNEL); + if (!mask) + goto err_free_padata; + if (!alloc_cpumask_var(&mask->mask, GFP_KERNEL)) { + kfree(mask); + goto err_free_padata; + } + + cpumask_and(mask->mask, cpu_possible_mask, cpu_active_mask); + rcu_assign_pointer(pcrypt->cb_cpumask, mask); + + pcrypt->nblock.notifier_call = pcrypt_cpumask_change_notify; + ret = padata_register_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); + if (ret) + goto err_free_cpumask; + + ret = pcrypt_sysfs_add(pcrypt->pinst, name); + if (ret) + goto err_unregister_notifier; + + put_online_cpus(); + + return ret; + +err_unregister_notifier: + padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); +err_free_cpumask: + free_cpumask_var(mask->mask); + kfree(mask); +err_free_padata: + padata_free(pcrypt->pinst); +err_destroy_workqueue: + destroy_workqueue(pcrypt->wq); +err: + put_online_cpus(); + + return ret; +} + +static void pcrypt_fini_padata(struct padata_pcrypt *pcrypt) +{ + free_cpumask_var(pcrypt->cb_cpumask->mask); + kfree(pcrypt->cb_cpumask); + + padata_stop(pcrypt->pinst); + padata_unregister_cpumask_notifier(pcrypt->pinst, &pcrypt->nblock); + destroy_workqueue(pcrypt->wq); + padata_free(pcrypt->pinst); +} + +static struct crypto_template pcrypt_tmpl = { + .name = "pcrypt", + .alloc = pcrypt_alloc, + .free = pcrypt_free, + .module = THIS_MODULE, +}; + +static int __init pcrypt_init(void) +{ + int err = -ENOMEM; + + pcrypt_kset = kset_create_and_add("pcrypt", NULL, kernel_kobj); + if (!pcrypt_kset) + goto err; + + err = pcrypt_init_padata(&pencrypt, "pencrypt"); + if (err) + goto err_unreg_kset; + + err = pcrypt_init_padata(&pdecrypt, "pdecrypt"); + if (err) + goto err_deinit_pencrypt; + + padata_start(pencrypt.pinst); + padata_start(pdecrypt.pinst); + + return crypto_register_template(&pcrypt_tmpl); + +err_deinit_pencrypt: + pcrypt_fini_padata(&pencrypt); +err_unreg_kset: + kset_unregister(pcrypt_kset); +err: + return err; +} + +static void __exit pcrypt_exit(void) +{ + pcrypt_fini_padata(&pencrypt); + pcrypt_fini_padata(&pdecrypt); + + kset_unregister(pcrypt_kset); + crypto_unregister_template(&pcrypt_tmpl); +} + +module_init(pcrypt_init); +module_exit(pcrypt_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Steffen Klassert <steffen.klassert@secunet.com>"); +MODULE_DESCRIPTION("Parallel crypto wrapper"); diff --git a/crypto/proc.c b/crypto/proc.c index 5dc07e442fca..58fef67d4f4d 100644 --- a/crypto/proc.c +++ b/crypto/proc.c @@ -25,28 +25,22 @@ #ifdef CONFIG_CRYPTO_FIPS static struct ctl_table crypto_sysctl_table[] = { { - .ctl_name = CTL_UNNUMBERED, .procname = "fips_enabled", .data = &fips_enabled, .maxlen = sizeof(int), .mode = 0444, - .proc_handler = &proc_dointvec - }, - { - .ctl_name = 0, + .proc_handler = proc_dointvec }, + {} }; static struct ctl_table crypto_dir_table[] = { { - .ctl_name = CTL_UNNUMBERED, .procname = "crypto", .mode = 0555, .child = crypto_sysctl_table }, - { - .ctl_name = 0, - }, + {} }; static struct ctl_table_header *crypto_sysctls; @@ -115,13 +109,6 @@ static int c_show(struct seq_file *m, void *p) seq_printf(m, "max keysize : %u\n", alg->cra_cipher.cia_max_keysize); break; - - case CRYPTO_ALG_TYPE_DIGEST: - seq_printf(m, "type : digest\n"); - seq_printf(m, "blocksize : %u\n", alg->cra_blocksize); - seq_printf(m, "digestsize : %u\n", - alg->cra_digest.dia_digestsize); - break; case CRYPTO_ALG_TYPE_COMPRESS: seq_printf(m, "type : compression\n"); break; diff --git a/crypto/rng.c b/crypto/rng.c index 6e94bc735578..f93cb5311182 100644 --- a/crypto/rng.c +++ b/crypto/rng.c @@ -19,6 +19,7 @@ #include <linux/mutex.h> #include <linux/random.h> #include <linux/seq_file.h> +#include <linux/slab.h> #include <linux/string.h> static DEFINE_MUTEX(crypto_default_rng_lock); @@ -123,4 +124,4 @@ void crypto_put_default_rng(void) EXPORT_SYMBOL_GPL(crypto_put_default_rng); MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Random Number Genertor"); +MODULE_DESCRIPTION("Random Number Generator"); diff --git a/crypto/scatterwalk.c b/crypto/scatterwalk.c index 3de89a424401..41e529af0773 100644 --- a/crypto/scatterwalk.c +++ b/crypto/scatterwalk.c @@ -68,7 +68,7 @@ static void scatterwalk_pagedone(struct scatter_walk *walk, int out, void scatterwalk_done(struct scatter_walk *walk, int out, int more) { - if (!offset_in_page(walk->offset) || !more) + if (!(scatterwalk_pagelen(walk) & (PAGE_SIZE - 1)) || !more) scatterwalk_pagedone(walk, out, more); } EXPORT_SYMBOL_GPL(scatterwalk_done); diff --git a/crypto/seqiv.c b/crypto/seqiv.c index 5a013a8bf87a..4c4491229417 100644 --- a/crypto/seqiv.c +++ b/crypto/seqiv.c @@ -20,6 +20,7 @@ #include <linux/init.h> #include <linux/kernel.h> #include <linux/module.h> +#include <linux/slab.h> #include <linux/spinlock.h> #include <linux/string.h> diff --git a/crypto/sha1_generic.c b/crypto/sha1_generic.c index 9efef20454cb..0416091bf45a 100644 --- a/crypto/sha1_generic.c +++ b/crypto/sha1_generic.c @@ -25,31 +25,21 @@ #include <crypto/sha.h> #include <asm/byteorder.h> -struct sha1_ctx { - u64 count; - u32 state[5]; - u8 buffer[64]; -}; - static int sha1_init(struct shash_desc *desc) { - struct sha1_ctx *sctx = shash_desc_ctx(desc); + struct sha1_state *sctx = shash_desc_ctx(desc); - static const struct sha1_ctx initstate = { - 0, - { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, - { 0, } + *sctx = (struct sha1_state){ + .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, }; - *sctx = initstate; - return 0; } static int sha1_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha1_ctx *sctx = shash_desc_ctx(desc); + struct sha1_state *sctx = shash_desc_ctx(desc); unsigned int partial, done; const u8 *src; @@ -85,7 +75,7 @@ static int sha1_update(struct shash_desc *desc, const u8 *data, /* Add padding and return the message digest. */ static int sha1_final(struct shash_desc *desc, u8 *out) { - struct sha1_ctx *sctx = shash_desc_ctx(desc); + struct sha1_state *sctx = shash_desc_ctx(desc); __be32 *dst = (__be32 *)out; u32 i, index, padlen; __be64 bits; @@ -111,12 +101,31 @@ static int sha1_final(struct shash_desc *desc, u8 *out) return 0; } +static int sha1_export(struct shash_desc *desc, void *out) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + memcpy(out, sctx, sizeof(*sctx)); + return 0; +} + +static int sha1_import(struct shash_desc *desc, const void *in) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + memcpy(sctx, in, sizeof(*sctx)); + return 0; +} + static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_init, .update = sha1_update, .final = sha1_final, - .descsize = sizeof(struct sha1_ctx), + .export = sha1_export, + .import = sha1_import, + .descsize = sizeof(struct sha1_state), + .statesize = sizeof(struct sha1_state), .base = { .cra_name = "sha1", .cra_driver_name= "sha1-generic", diff --git a/crypto/sha256_generic.c b/crypto/sha256_generic.c index 6349d8339d37..c48459ebf05b 100644 --- a/crypto/sha256_generic.c +++ b/crypto/sha256_generic.c @@ -25,12 +25,6 @@ #include <crypto/sha.h> #include <asm/byteorder.h> -struct sha256_ctx { - u32 count[2]; - u32 state[8]; - u8 buf[128]; -}; - static inline u32 Ch(u32 x, u32 y, u32 z) { return z ^ (x & (y ^ z)); @@ -222,7 +216,7 @@ static void sha256_transform(u32 *state, const u8 *input) static int sha224_init(struct shash_desc *desc) { - struct sha256_ctx *sctx = shash_desc_ctx(desc); + struct sha256_state *sctx = shash_desc_ctx(desc); sctx->state[0] = SHA224_H0; sctx->state[1] = SHA224_H1; sctx->state[2] = SHA224_H2; @@ -231,15 +225,14 @@ static int sha224_init(struct shash_desc *desc) sctx->state[5] = SHA224_H5; sctx->state[6] = SHA224_H6; sctx->state[7] = SHA224_H7; - sctx->count[0] = 0; - sctx->count[1] = 0; + sctx->count = 0; return 0; } static int sha256_init(struct shash_desc *desc) { - struct sha256_ctx *sctx = shash_desc_ctx(desc); + struct sha256_state *sctx = shash_desc_ctx(desc); sctx->state[0] = SHA256_H0; sctx->state[1] = SHA256_H1; sctx->state[2] = SHA256_H2; @@ -248,7 +241,7 @@ static int sha256_init(struct shash_desc *desc) sctx->state[5] = SHA256_H5; sctx->state[6] = SHA256_H6; sctx->state[7] = SHA256_H7; - sctx->count[0] = sctx->count[1] = 0; + sctx->count = 0; return 0; } @@ -256,58 +249,54 @@ static int sha256_init(struct shash_desc *desc) static int sha256_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha256_ctx *sctx = shash_desc_ctx(desc); - unsigned int i, index, part_len; - - /* Compute number of bytes mod 128 */ - index = (unsigned int)((sctx->count[0] >> 3) & 0x3f); - - /* Update number of bits */ - if ((sctx->count[0] += (len << 3)) < (len << 3)) { - sctx->count[1]++; - sctx->count[1] += (len >> 29); - } - - part_len = 64 - index; - - /* Transform as many times as possible. */ - if (len >= part_len) { - memcpy(&sctx->buf[index], data, part_len); - sha256_transform(sctx->state, sctx->buf); - - for (i = part_len; i + 63 < len; i += 64) - sha256_transform(sctx->state, &data[i]); - index = 0; - } else { - i = 0; + struct sha256_state *sctx = shash_desc_ctx(desc); + unsigned int partial, done; + const u8 *src; + + partial = sctx->count & 0x3f; + sctx->count += len; + done = 0; + src = data; + + if ((partial + len) > 63) { + if (partial) { + done = -partial; + memcpy(sctx->buf + partial, data, done + 64); + src = sctx->buf; + } + + do { + sha256_transform(sctx->state, src); + done += 64; + src = data + done; + } while (done + 63 < len); + + partial = 0; } - - /* Buffer remaining input */ - memcpy(&sctx->buf[index], &data[i], len-i); + memcpy(sctx->buf + partial, src, len - done); return 0; } static int sha256_final(struct shash_desc *desc, u8 *out) { - struct sha256_ctx *sctx = shash_desc_ctx(desc); + struct sha256_state *sctx = shash_desc_ctx(desc); __be32 *dst = (__be32 *)out; - __be32 bits[2]; + __be64 bits; unsigned int index, pad_len; int i; static const u8 padding[64] = { 0x80, }; /* Save number of bits */ - bits[1] = cpu_to_be32(sctx->count[0]); - bits[0] = cpu_to_be32(sctx->count[1]); + bits = cpu_to_be64(sctx->count << 3); /* Pad out to 56 mod 64. */ - index = (sctx->count[0] >> 3) & 0x3f; + index = sctx->count & 0x3f; pad_len = (index < 56) ? (56 - index) : ((64+56) - index); sha256_update(desc, padding, pad_len); /* Append length (before padding) */ - sha256_update(desc, (const u8 *)bits, sizeof(bits)); + sha256_update(desc, (const u8 *)&bits, sizeof(bits)); /* Store state in digest */ for (i = 0; i < 8; i++) @@ -331,12 +320,31 @@ static int sha224_final(struct shash_desc *desc, u8 *hash) return 0; } +static int sha256_export(struct shash_desc *desc, void *out) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + + memcpy(out, sctx, sizeof(*sctx)); + return 0; +} + +static int sha256_import(struct shash_desc *desc, const void *in) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + + memcpy(sctx, in, sizeof(*sctx)); + return 0; +} + static struct shash_alg sha256 = { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_init, .update = sha256_update, .final = sha256_final, - .descsize = sizeof(struct sha256_ctx), + .export = sha256_export, + .import = sha256_import, + .descsize = sizeof(struct sha256_state), + .statesize = sizeof(struct sha256_state), .base = { .cra_name = "sha256", .cra_driver_name= "sha256-generic", @@ -351,7 +359,7 @@ static struct shash_alg sha224 = { .init = sha224_init, .update = sha256_update, .final = sha224_final, - .descsize = sizeof(struct sha256_ctx), + .descsize = sizeof(struct sha256_state), .base = { .cra_name = "sha224", .cra_driver_name= "sha224-generic", diff --git a/crypto/sha512_generic.c b/crypto/sha512_generic.c index 3bea38d12242..9ed9f60316e5 100644 --- a/crypto/sha512_generic.c +++ b/crypto/sha512_generic.c @@ -21,12 +21,6 @@ #include <linux/percpu.h> #include <asm/byteorder.h> -struct sha512_ctx { - u64 state[8]; - u32 count[4]; - u8 buf[128]; -}; - static DEFINE_PER_CPU(u64[80], msg_schedule); static inline u64 Ch(u64 x, u64 y, u64 z) @@ -141,7 +135,7 @@ sha512_transform(u64 *state, const u8 *input) static int sha512_init(struct shash_desc *desc) { - struct sha512_ctx *sctx = shash_desc_ctx(desc); + struct sha512_state *sctx = shash_desc_ctx(desc); sctx->state[0] = SHA512_H0; sctx->state[1] = SHA512_H1; sctx->state[2] = SHA512_H2; @@ -150,7 +144,7 @@ sha512_init(struct shash_desc *desc) sctx->state[5] = SHA512_H5; sctx->state[6] = SHA512_H6; sctx->state[7] = SHA512_H7; - sctx->count[0] = sctx->count[1] = sctx->count[2] = sctx->count[3] = 0; + sctx->count[0] = sctx->count[1] = 0; return 0; } @@ -158,7 +152,7 @@ sha512_init(struct shash_desc *desc) static int sha384_init(struct shash_desc *desc) { - struct sha512_ctx *sctx = shash_desc_ctx(desc); + struct sha512_state *sctx = shash_desc_ctx(desc); sctx->state[0] = SHA384_H0; sctx->state[1] = SHA384_H1; sctx->state[2] = SHA384_H2; @@ -167,7 +161,7 @@ sha384_init(struct shash_desc *desc) sctx->state[5] = SHA384_H5; sctx->state[6] = SHA384_H6; sctx->state[7] = SHA384_H7; - sctx->count[0] = sctx->count[1] = sctx->count[2] = sctx->count[3] = 0; + sctx->count[0] = sctx->count[1] = 0; return 0; } @@ -175,20 +169,16 @@ sha384_init(struct shash_desc *desc) static int sha512_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha512_ctx *sctx = shash_desc_ctx(desc); + struct sha512_state *sctx = shash_desc_ctx(desc); unsigned int i, index, part_len; /* Compute number of bytes mod 128 */ - index = (unsigned int)((sctx->count[0] >> 3) & 0x7F); - - /* Update number of bits */ - if ((sctx->count[0] += (len << 3)) < (len << 3)) { - if ((sctx->count[1] += 1) < 1) - if ((sctx->count[2] += 1) < 1) - sctx->count[3]++; - sctx->count[1] += (len >> 29); - } + index = sctx->count[0] & 0x7f; + + /* Update number of bytes */ + if (!(sctx->count[0] += len)) + sctx->count[1]++; part_len = 128 - index; @@ -214,21 +204,19 @@ sha512_update(struct shash_desc *desc, const u8 *data, unsigned int len) static int sha512_final(struct shash_desc *desc, u8 *hash) { - struct sha512_ctx *sctx = shash_desc_ctx(desc); + struct sha512_state *sctx = shash_desc_ctx(desc); static u8 padding[128] = { 0x80, }; __be64 *dst = (__be64 *)hash; - __be32 bits[4]; + __be64 bits[2]; unsigned int index, pad_len; int i; /* Save number of bits */ - bits[3] = cpu_to_be32(sctx->count[0]); - bits[2] = cpu_to_be32(sctx->count[1]); - bits[1] = cpu_to_be32(sctx->count[2]); - bits[0] = cpu_to_be32(sctx->count[3]); + bits[1] = cpu_to_be64(sctx->count[0] << 3); + bits[0] = cpu_to_be64(sctx->count[1] << 3 | sctx->count[0] >> 61); /* Pad out to 112 mod 128. */ - index = (sctx->count[0] >> 3) & 0x7f; + index = sctx->count[0] & 0x7f; pad_len = (index < 112) ? (112 - index) : ((128+112) - index); sha512_update(desc, padding, pad_len); @@ -240,7 +228,7 @@ sha512_final(struct shash_desc *desc, u8 *hash) dst[i] = cpu_to_be64(sctx->state[i]); /* Zeroize sensitive information. */ - memset(sctx, 0, sizeof(struct sha512_ctx)); + memset(sctx, 0, sizeof(struct sha512_state)); return 0; } @@ -262,7 +250,7 @@ static struct shash_alg sha512 = { .init = sha512_init, .update = sha512_update, .final = sha512_final, - .descsize = sizeof(struct sha512_ctx), + .descsize = sizeof(struct sha512_state), .base = { .cra_name = "sha512", .cra_flags = CRYPTO_ALG_TYPE_SHASH, @@ -276,7 +264,7 @@ static struct shash_alg sha384 = { .init = sha384_init, .update = sha512_update, .final = sha384_final, - .descsize = sizeof(struct sha512_ctx), + .descsize = sizeof(struct sha512_state), .base = { .cra_name = "sha384", .cra_flags = CRYPTO_ALG_TYPE_SHASH, diff --git a/crypto/shash.c b/crypto/shash.c index 2ccc8b0076ce..22fd9433141f 100644 --- a/crypto/shash.c +++ b/crypto/shash.c @@ -22,6 +22,12 @@ static const struct crypto_type crypto_shash_type; +static int shash_no_setkey(struct crypto_shash *tfm, const u8 *key, + unsigned int keylen) +{ + return -ENOSYS; +} + static int shash_setkey_unaligned(struct crypto_shash *tfm, const u8 *key, unsigned int keylen) { @@ -31,7 +37,7 @@ static int shash_setkey_unaligned(struct crypto_shash *tfm, const u8 *key, u8 *buffer, *alignbuffer; int err; - absize = keylen + (alignmask & ~(CRYPTO_MINALIGN - 1)); + absize = keylen + (alignmask & ~(crypto_tfm_ctx_alignment() - 1)); buffer = kmalloc(absize, GFP_KERNEL); if (!buffer) return -ENOMEM; @@ -39,8 +45,7 @@ static int shash_setkey_unaligned(struct crypto_shash *tfm, const u8 *key, alignbuffer = (u8 *)ALIGN((unsigned long)buffer, alignmask + 1); memcpy(alignbuffer, key, keylen); err = shash->setkey(tfm, alignbuffer, keylen); - memset(alignbuffer, 0, keylen); - kfree(buffer); + kzfree(buffer); return err; } @@ -50,9 +55,6 @@ int crypto_shash_setkey(struct crypto_shash *tfm, const u8 *key, struct shash_alg *shash = crypto_shash_alg(tfm); unsigned long alignmask = crypto_shash_alignmask(tfm); - if (!shash->setkey) - return -ENOSYS; - if ((unsigned long)key & alignmask) return shash_setkey_unaligned(tfm, key, keylen); @@ -74,15 +76,19 @@ static int shash_update_unaligned(struct shash_desc *desc, const u8 *data, unsigned long alignmask = crypto_shash_alignmask(tfm); unsigned int unaligned_len = alignmask + 1 - ((unsigned long)data & alignmask); - u8 buf[shash_align_buffer_size(unaligned_len, alignmask)] + u8 ubuf[shash_align_buffer_size(unaligned_len, alignmask)] __attribute__ ((aligned)); + u8 *buf = PTR_ALIGN(&ubuf[0], alignmask + 1); + int err; if (unaligned_len > len) unaligned_len = len; memcpy(buf, data, unaligned_len); + err = shash->update(desc, buf, unaligned_len); + memset(buf, 0, unaligned_len); - return shash->update(desc, buf, unaligned_len) ?: + return err ?: shash->update(desc, data + unaligned_len, len - unaligned_len); } @@ -106,12 +112,19 @@ static int shash_final_unaligned(struct shash_desc *desc, u8 *out) unsigned long alignmask = crypto_shash_alignmask(tfm); struct shash_alg *shash = crypto_shash_alg(tfm); unsigned int ds = crypto_shash_digestsize(tfm); - u8 buf[shash_align_buffer_size(ds, alignmask)] + u8 ubuf[shash_align_buffer_size(ds, alignmask)] __attribute__ ((aligned)); + u8 *buf = PTR_ALIGN(&ubuf[0], alignmask + 1); int err; err = shash->final(desc, buf); + if (err) + goto out; + memcpy(out, buf, ds); + +out: + memset(buf, 0, ds); return err; } @@ -142,8 +155,7 @@ int crypto_shash_finup(struct shash_desc *desc, const u8 *data, struct shash_alg *shash = crypto_shash_alg(tfm); unsigned long alignmask = crypto_shash_alignmask(tfm); - if (((unsigned long)data | (unsigned long)out) & alignmask || - !shash->finup) + if (((unsigned long)data | (unsigned long)out) & alignmask) return shash_finup_unaligned(desc, data, len, out); return shash->finup(desc, data, len, out); @@ -154,8 +166,7 @@ static int shash_digest_unaligned(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { return crypto_shash_init(desc) ?: - crypto_shash_update(desc, data, len) ?: - crypto_shash_final(desc, out); + crypto_shash_finup(desc, data, len, out); } int crypto_shash_digest(struct shash_desc *desc, const u8 *data, @@ -165,27 +176,24 @@ int crypto_shash_digest(struct shash_desc *desc, const u8 *data, struct shash_alg *shash = crypto_shash_alg(tfm); unsigned long alignmask = crypto_shash_alignmask(tfm); - if (((unsigned long)data | (unsigned long)out) & alignmask || - !shash->digest) + if (((unsigned long)data | (unsigned long)out) & alignmask) return shash_digest_unaligned(desc, data, len, out); return shash->digest(desc, data, len, out); } EXPORT_SYMBOL_GPL(crypto_shash_digest); -int crypto_shash_import(struct shash_desc *desc, const u8 *in) +static int shash_default_export(struct shash_desc *desc, void *out) { - struct crypto_shash *tfm = desc->tfm; - struct shash_alg *alg = crypto_shash_alg(tfm); - - memcpy(shash_desc_ctx(desc), in, crypto_shash_descsize(tfm)); - - if (alg->reinit) - alg->reinit(desc); + memcpy(out, shash_desc_ctx(desc), crypto_shash_descsize(desc->tfm)); + return 0; +} +static int shash_default_import(struct shash_desc *desc, const void *in) +{ + memcpy(shash_desc_ctx(desc), in, crypto_shash_descsize(desc->tfm)); return 0; } -EXPORT_SYMBOL_GPL(crypto_shash_import); static int shash_async_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) @@ -206,9 +214,8 @@ static int shash_async_init(struct ahash_request *req) return crypto_shash_init(desc); } -static int shash_async_update(struct ahash_request *req) +int shash_ahash_update(struct ahash_request *req, struct shash_desc *desc) { - struct shash_desc *desc = ahash_request_ctx(req); struct crypto_hash_walk walk; int nbytes; @@ -218,13 +225,51 @@ static int shash_async_update(struct ahash_request *req) return nbytes; } +EXPORT_SYMBOL_GPL(shash_ahash_update); + +static int shash_async_update(struct ahash_request *req) +{ + return shash_ahash_update(req, ahash_request_ctx(req)); +} static int shash_async_final(struct ahash_request *req) { return crypto_shash_final(ahash_request_ctx(req), req->result); } -static int shash_async_digest(struct ahash_request *req) +int shash_ahash_finup(struct ahash_request *req, struct shash_desc *desc) +{ + struct crypto_hash_walk walk; + int nbytes; + + nbytes = crypto_hash_walk_first(req, &walk); + if (!nbytes) + return crypto_shash_final(desc, req->result); + + do { + nbytes = crypto_hash_walk_last(&walk) ? + crypto_shash_finup(desc, walk.data, nbytes, + req->result) : + crypto_shash_update(desc, walk.data, nbytes); + nbytes = crypto_hash_walk_done(&walk, nbytes); + } while (nbytes > 0); + + return nbytes; +} +EXPORT_SYMBOL_GPL(shash_ahash_finup); + +static int shash_async_finup(struct ahash_request *req) +{ + struct crypto_shash **ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + struct shash_desc *desc = ahash_request_ctx(req); + + desc->tfm = *ctx; + desc->flags = req->base.flags; + + return shash_ahash_finup(req, desc); +} + +int shash_ahash_digest(struct ahash_request *req, struct shash_desc *desc) { struct scatterlist *sg = req->src; unsigned int offset = sg->offset; @@ -232,34 +277,40 @@ static int shash_async_digest(struct ahash_request *req) int err; if (nbytes < min(sg->length, ((unsigned int)(PAGE_SIZE)) - offset)) { - struct crypto_shash **ctx = - crypto_ahash_ctx(crypto_ahash_reqtfm(req)); - struct shash_desc *desc = ahash_request_ctx(req); void *data; - desc->tfm = *ctx; - desc->flags = req->base.flags; - data = crypto_kmap(sg_page(sg), 0); err = crypto_shash_digest(desc, data + offset, nbytes, req->result); crypto_kunmap(data, 0); crypto_yield(desc->flags); - goto out; - } + } else + err = crypto_shash_init(desc) ?: + shash_ahash_finup(req, desc); - err = shash_async_init(req); - if (err) - goto out; + return err; +} +EXPORT_SYMBOL_GPL(shash_ahash_digest); - err = shash_async_update(req); - if (err) - goto out; +static int shash_async_digest(struct ahash_request *req) +{ + struct crypto_shash **ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + struct shash_desc *desc = ahash_request_ctx(req); - err = shash_async_final(req); + desc->tfm = *ctx; + desc->flags = req->base.flags; -out: - return err; + return shash_ahash_digest(req, desc); +} + +static int shash_async_export(struct ahash_request *req, void *out) +{ + return crypto_shash_export(ahash_request_ctx(req), out); +} + +static int shash_async_import(struct ahash_request *req, const void *in) +{ + return crypto_shash_import(ahash_request_ctx(req), in); } static void crypto_exit_shash_ops_async(struct crypto_tfm *tfm) @@ -269,11 +320,11 @@ static void crypto_exit_shash_ops_async(struct crypto_tfm *tfm) crypto_free_shash(*ctx); } -static int crypto_init_shash_ops_async(struct crypto_tfm *tfm) +int crypto_init_shash_ops_async(struct crypto_tfm *tfm) { struct crypto_alg *calg = tfm->__crt_alg; struct shash_alg *alg = __crypto_shash_alg(calg); - struct ahash_tfm *crt = &tfm->crt_ahash; + struct crypto_ahash *crt = __crypto_ahash_cast(tfm); struct crypto_shash **ctx = crypto_tfm_ctx(tfm); struct crypto_shash *shash; @@ -291,11 +342,17 @@ static int crypto_init_shash_ops_async(struct crypto_tfm *tfm) crt->init = shash_async_init; crt->update = shash_async_update; - crt->final = shash_async_final; + crt->final = shash_async_final; + crt->finup = shash_async_finup; crt->digest = shash_async_digest; - crt->setkey = shash_async_setkey; - crt->digestsize = alg->digestsize; + if (alg->setkey) + crt->setkey = shash_async_setkey; + if (alg->export) + crt->export = shash_async_export; + if (alg->import) + crt->import = shash_async_import; + crt->reqsize = sizeof(struct shash_desc) + crypto_shash_descsize(shash); return 0; @@ -304,14 +361,16 @@ static int crypto_init_shash_ops_async(struct crypto_tfm *tfm) static int shash_compat_setkey(struct crypto_hash *tfm, const u8 *key, unsigned int keylen) { - struct shash_desc *desc = crypto_hash_ctx(tfm); + struct shash_desc **descp = crypto_hash_ctx(tfm); + struct shash_desc *desc = *descp; return crypto_shash_setkey(desc->tfm, key, keylen); } static int shash_compat_init(struct hash_desc *hdesc) { - struct shash_desc *desc = crypto_hash_ctx(hdesc->tfm); + struct shash_desc **descp = crypto_hash_ctx(hdesc->tfm); + struct shash_desc *desc = *descp; desc->flags = hdesc->flags; @@ -321,7 +380,8 @@ static int shash_compat_init(struct hash_desc *hdesc) static int shash_compat_update(struct hash_desc *hdesc, struct scatterlist *sg, unsigned int len) { - struct shash_desc *desc = crypto_hash_ctx(hdesc->tfm); + struct shash_desc **descp = crypto_hash_ctx(hdesc->tfm); + struct shash_desc *desc = *descp; struct crypto_hash_walk walk; int nbytes; @@ -334,7 +394,9 @@ static int shash_compat_update(struct hash_desc *hdesc, struct scatterlist *sg, static int shash_compat_final(struct hash_desc *hdesc, u8 *out) { - return crypto_shash_final(crypto_hash_ctx(hdesc->tfm), out); + struct shash_desc **descp = crypto_hash_ctx(hdesc->tfm); + + return crypto_shash_final(*descp, out); } static int shash_compat_digest(struct hash_desc *hdesc, struct scatterlist *sg, @@ -344,7 +406,8 @@ static int shash_compat_digest(struct hash_desc *hdesc, struct scatterlist *sg, int err; if (nbytes < min(sg->length, ((unsigned int)(PAGE_SIZE)) - offset)) { - struct shash_desc *desc = crypto_hash_ctx(hdesc->tfm); + struct shash_desc **descp = crypto_hash_ctx(hdesc->tfm); + struct shash_desc *desc = *descp; void *data; desc->flags = hdesc->flags; @@ -372,9 +435,11 @@ out: static void crypto_exit_shash_ops_compat(struct crypto_tfm *tfm) { - struct shash_desc *desc= crypto_tfm_ctx(tfm); + struct shash_desc **descp = crypto_tfm_ctx(tfm); + struct shash_desc *desc = *descp; crypto_free_shash(desc->tfm); + kzfree(desc); } static int crypto_init_shash_ops_compat(struct crypto_tfm *tfm) @@ -382,8 +447,9 @@ static int crypto_init_shash_ops_compat(struct crypto_tfm *tfm) struct hash_tfm *crt = &tfm->crt_hash; struct crypto_alg *calg = tfm->__crt_alg; struct shash_alg *alg = __crypto_shash_alg(calg); - struct shash_desc *desc = crypto_tfm_ctx(tfm); + struct shash_desc **descp = crypto_tfm_ctx(tfm); struct crypto_shash *shash; + struct shash_desc *desc; if (!crypto_mod_get(calg)) return -EAGAIN; @@ -394,6 +460,14 @@ static int crypto_init_shash_ops_compat(struct crypto_tfm *tfm) return PTR_ERR(shash); } + desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(shash), + GFP_KERNEL); + if (!desc) { + crypto_free_shash(shash); + return -ENOMEM; + } + + *descp = desc; desc->tfm = shash; tfm->exit = crypto_exit_shash_ops_compat; @@ -413,8 +487,6 @@ static int crypto_init_shash_ops(struct crypto_tfm *tfm, u32 type, u32 mask) switch (mask & CRYPTO_ALG_TYPE_MASK) { case CRYPTO_ALG_TYPE_HASH_MASK: return crypto_init_shash_ops_compat(tfm); - case CRYPTO_ALG_TYPE_AHASH_MASK: - return crypto_init_shash_ops_async(tfm); } return -EINVAL; @@ -423,26 +495,23 @@ static int crypto_init_shash_ops(struct crypto_tfm *tfm, u32 type, u32 mask) static unsigned int crypto_shash_ctxsize(struct crypto_alg *alg, u32 type, u32 mask) { - struct shash_alg *salg = __crypto_shash_alg(alg); - switch (mask & CRYPTO_ALG_TYPE_MASK) { case CRYPTO_ALG_TYPE_HASH_MASK: - return sizeof(struct shash_desc) + salg->descsize; - case CRYPTO_ALG_TYPE_AHASH_MASK: - return sizeof(struct crypto_shash *); + return sizeof(struct shash_desc *); } return 0; } -static int crypto_shash_init_tfm(struct crypto_tfm *tfm, - const struct crypto_type *frontend) +static int crypto_shash_init_tfm(struct crypto_tfm *tfm) { + struct crypto_shash *hash = __crypto_shash_cast(tfm); + + hash->descsize = crypto_shash_alg(hash)->descsize; return 0; } -static unsigned int crypto_shash_extsize(struct crypto_alg *alg, - const struct crypto_type *frontend) +static unsigned int crypto_shash_extsize(struct crypto_alg *alg) { return alg->cra_ctxsize; } @@ -456,7 +525,6 @@ static void crypto_shash_show(struct seq_file *m, struct crypto_alg *alg) seq_printf(m, "type : shash\n"); seq_printf(m, "blocksize : %u\n", alg->cra_blocksize); seq_printf(m, "digestsize : %u\n", salg->digestsize); - seq_printf(m, "descsize : %u\n", salg->descsize); } static const struct crypto_type crypto_shash_type = { @@ -480,18 +548,43 @@ struct crypto_shash *crypto_alloc_shash(const char *alg_name, u32 type, } EXPORT_SYMBOL_GPL(crypto_alloc_shash); -int crypto_register_shash(struct shash_alg *alg) +static int shash_prepare_alg(struct shash_alg *alg) { struct crypto_alg *base = &alg->base; if (alg->digestsize > PAGE_SIZE / 8 || - alg->descsize > PAGE_SIZE / 8) + alg->descsize > PAGE_SIZE / 8 || + alg->statesize > PAGE_SIZE / 8) return -EINVAL; base->cra_type = &crypto_shash_type; base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK; base->cra_flags |= CRYPTO_ALG_TYPE_SHASH; + if (!alg->finup) + alg->finup = shash_finup_unaligned; + if (!alg->digest) + alg->digest = shash_digest_unaligned; + if (!alg->export) { + alg->export = shash_default_export; + alg->import = shash_default_import; + alg->statesize = alg->descsize; + } + if (!alg->setkey) + alg->setkey = shash_no_setkey; + + return 0; +} + +int crypto_register_shash(struct shash_alg *alg) +{ + struct crypto_alg *base = &alg->base; + int err; + + err = shash_prepare_alg(alg); + if (err) + return err; + return crypto_register_alg(base); } EXPORT_SYMBOL_GPL(crypto_register_shash); @@ -502,5 +595,44 @@ int crypto_unregister_shash(struct shash_alg *alg) } EXPORT_SYMBOL_GPL(crypto_unregister_shash); +int shash_register_instance(struct crypto_template *tmpl, + struct shash_instance *inst) +{ + int err; + + err = shash_prepare_alg(&inst->alg); + if (err) + return err; + + return crypto_register_instance(tmpl, shash_crypto_instance(inst)); +} +EXPORT_SYMBOL_GPL(shash_register_instance); + +void shash_free_instance(struct crypto_instance *inst) +{ + crypto_drop_spawn(crypto_instance_ctx(inst)); + kfree(shash_instance(inst)); +} +EXPORT_SYMBOL_GPL(shash_free_instance); + +int crypto_init_shash_spawn(struct crypto_shash_spawn *spawn, + struct shash_alg *alg, + struct crypto_instance *inst) +{ + return crypto_init_spawn2(&spawn->base, &alg->base, inst, + &crypto_shash_type); +} +EXPORT_SYMBOL_GPL(crypto_init_shash_spawn); + +struct shash_alg *shash_attr_alg(struct rtattr *rta, u32 type, u32 mask) +{ + struct crypto_alg *alg; + + alg = crypto_attr_alg2(rta, &crypto_shash_type, type, mask); + return IS_ERR(alg) ? ERR_CAST(alg) : + container_of(alg, struct shash_alg, base); +} +EXPORT_SYMBOL_GPL(shash_attr_alg); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Synchronous cryptographic hash type"); diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index d59ba5079d14..3ca68f9fc14d 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -18,8 +18,8 @@ #include <crypto/hash.h> #include <linux/err.h> #include <linux/init.h> +#include <linux/gfp.h> #include <linux/module.h> -#include <linux/slab.h> #include <linux/scatterlist.h> #include <linux/string.h> #include <linux/moduleparam.h> @@ -45,6 +45,9 @@ */ static unsigned int sec; +static char *alg = NULL; +static u32 type; +static u32 mask; static int mode; static char *tvmem[TVMEMSIZE]; @@ -391,6 +394,17 @@ out: return 0; } +static void test_hash_sg_init(struct scatterlist *sg) +{ + int i; + + sg_init_table(sg, TVMEMSIZE); + for (i = 0; i < TVMEMSIZE; i++) { + sg_set_buf(sg + i, tvmem[i], PAGE_SIZE); + memset(tvmem[i], 0xff, PAGE_SIZE); + } +} + static void test_hash_speed(const char *algo, unsigned int sec, struct hash_speed *speed) { @@ -420,12 +434,7 @@ static void test_hash_speed(const char *algo, unsigned int sec, goto out; } - sg_init_table(sg, TVMEMSIZE); - for (i = 0; i < TVMEMSIZE; i++) { - sg_set_buf(sg + i, tvmem[i], PAGE_SIZE); - memset(tvmem[i], 0xff, PAGE_SIZE); - } - + test_hash_sg_init(sg); for (i = 0; speed[i].blen != 0; i++) { if (speed[i].blen > TVMEMSIZE * PAGE_SIZE) { printk(KERN_ERR @@ -434,6 +443,9 @@ static void test_hash_speed(const char *algo, unsigned int sec, goto out; } + if (speed[i].klen) + crypto_hash_setkey(tfm, tvmem[0], speed[i].klen); + printk(KERN_INFO "test%3u " "(%5u byte blocks,%5u bytes per update,%4u updates): ", i, speed[i].blen, speed[i].plen, speed[i].blen / speed[i].plen); @@ -455,6 +467,250 @@ out: crypto_free_hash(tfm); } +struct tcrypt_result { + struct completion completion; + int err; +}; + +static void tcrypt_complete(struct crypto_async_request *req, int err) +{ + struct tcrypt_result *res = req->data; + + if (err == -EINPROGRESS) + return; + + res->err = err; + complete(&res->completion); +} + +static inline int do_one_ahash_op(struct ahash_request *req, int ret) +{ + if (ret == -EINPROGRESS || ret == -EBUSY) { + struct tcrypt_result *tr = req->base.data; + + ret = wait_for_completion_interruptible(&tr->completion); + if (!ret) + ret = tr->err; + INIT_COMPLETION(tr->completion); + } + return ret; +} + +static int test_ahash_jiffies_digest(struct ahash_request *req, int blen, + char *out, int sec) +{ + unsigned long start, end; + int bcount; + int ret; + + for (start = jiffies, end = start + sec * HZ, bcount = 0; + time_before(jiffies, end); bcount++) { + ret = do_one_ahash_op(req, crypto_ahash_digest(req)); + if (ret) + return ret; + } + + printk("%6u opers/sec, %9lu bytes/sec\n", + bcount / sec, ((long)bcount * blen) / sec); + + return 0; +} + +static int test_ahash_jiffies(struct ahash_request *req, int blen, + int plen, char *out, int sec) +{ + unsigned long start, end; + int bcount, pcount; + int ret; + + if (plen == blen) + return test_ahash_jiffies_digest(req, blen, out, sec); + + for (start = jiffies, end = start + sec * HZ, bcount = 0; + time_before(jiffies, end); bcount++) { + ret = crypto_ahash_init(req); + if (ret) + return ret; + for (pcount = 0; pcount < blen; pcount += plen) { + ret = do_one_ahash_op(req, crypto_ahash_update(req)); + if (ret) + return ret; + } + /* we assume there is enough space in 'out' for the result */ + ret = do_one_ahash_op(req, crypto_ahash_final(req)); + if (ret) + return ret; + } + + pr_cont("%6u opers/sec, %9lu bytes/sec\n", + bcount / sec, ((long)bcount * blen) / sec); + + return 0; +} + +static int test_ahash_cycles_digest(struct ahash_request *req, int blen, + char *out) +{ + unsigned long cycles = 0; + int ret, i; + + /* Warm-up run. */ + for (i = 0; i < 4; i++) { + ret = do_one_ahash_op(req, crypto_ahash_digest(req)); + if (ret) + goto out; + } + + /* The real thing. */ + for (i = 0; i < 8; i++) { + cycles_t start, end; + + start = get_cycles(); + + ret = do_one_ahash_op(req, crypto_ahash_digest(req)); + if (ret) + goto out; + + end = get_cycles(); + + cycles += end - start; + } + +out: + if (ret) + return ret; + + pr_cont("%6lu cycles/operation, %4lu cycles/byte\n", + cycles / 8, cycles / (8 * blen)); + + return 0; +} + +static int test_ahash_cycles(struct ahash_request *req, int blen, + int plen, char *out) +{ + unsigned long cycles = 0; + int i, pcount, ret; + + if (plen == blen) + return test_ahash_cycles_digest(req, blen, out); + + /* Warm-up run. */ + for (i = 0; i < 4; i++) { + ret = crypto_ahash_init(req); + if (ret) + goto out; + for (pcount = 0; pcount < blen; pcount += plen) { + ret = do_one_ahash_op(req, crypto_ahash_update(req)); + if (ret) + goto out; + } + ret = do_one_ahash_op(req, crypto_ahash_final(req)); + if (ret) + goto out; + } + + /* The real thing. */ + for (i = 0; i < 8; i++) { + cycles_t start, end; + + start = get_cycles(); + + ret = crypto_ahash_init(req); + if (ret) + goto out; + for (pcount = 0; pcount < blen; pcount += plen) { + ret = do_one_ahash_op(req, crypto_ahash_update(req)); + if (ret) + goto out; + } + ret = do_one_ahash_op(req, crypto_ahash_final(req)); + if (ret) + goto out; + + end = get_cycles(); + + cycles += end - start; + } + +out: + if (ret) + return ret; + + pr_cont("%6lu cycles/operation, %4lu cycles/byte\n", + cycles / 8, cycles / (8 * blen)); + + return 0; +} + +static void test_ahash_speed(const char *algo, unsigned int sec, + struct hash_speed *speed) +{ + struct scatterlist sg[TVMEMSIZE]; + struct tcrypt_result tresult; + struct ahash_request *req; + struct crypto_ahash *tfm; + static char output[1024]; + int i, ret; + + printk(KERN_INFO "\ntesting speed of async %s\n", algo); + + tfm = crypto_alloc_ahash(algo, 0, 0); + if (IS_ERR(tfm)) { + pr_err("failed to load transform for %s: %ld\n", + algo, PTR_ERR(tfm)); + return; + } + + if (crypto_ahash_digestsize(tfm) > sizeof(output)) { + pr_err("digestsize(%u) > outputbuffer(%zu)\n", + crypto_ahash_digestsize(tfm), sizeof(output)); + goto out; + } + + test_hash_sg_init(sg); + req = ahash_request_alloc(tfm, GFP_KERNEL); + if (!req) { + pr_err("ahash request allocation failure\n"); + goto out; + } + + init_completion(&tresult.completion); + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, + tcrypt_complete, &tresult); + + for (i = 0; speed[i].blen != 0; i++) { + if (speed[i].blen > TVMEMSIZE * PAGE_SIZE) { + pr_err("template (%u) too big for tvmem (%lu)\n", + speed[i].blen, TVMEMSIZE * PAGE_SIZE); + break; + } + + pr_info("test%3u " + "(%5u byte blocks,%5u bytes per update,%4u updates): ", + i, speed[i].blen, speed[i].plen, speed[i].blen / speed[i].plen); + + ahash_request_set_crypt(req, sg, output, speed[i].plen); + + if (sec) + ret = test_ahash_jiffies(req, speed[i].blen, + speed[i].plen, output, sec); + else + ret = test_ahash_cycles(req, speed[i].blen, + speed[i].plen, output); + + if (ret) { + pr_err("hashing failed ret=%d\n", ret); + break; + } + } + + ahash_request_free(req); + +out: + crypto_free_ahash(tfm); +} + static void test_available(void) { char **name = check; @@ -716,6 +972,10 @@ static int do_test(int m) ret += tcrypt_test("hmac(rmd160)"); break; + case 109: + ret += tcrypt_test("vmac(aes)"); + break; + case 150: ret += tcrypt_test("ansi_cprng"); break; @@ -874,9 +1134,87 @@ static int do_test(int m) test_hash_speed("rmd320", sec, generic_hash_speed_template); if (mode > 300 && mode < 400) break; + case 318: + test_hash_speed("ghash-generic", sec, hash_speed_template_16); + if (mode > 300 && mode < 400) break; + case 399: break; + case 400: + /* fall through */ + + case 401: + test_ahash_speed("md4", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + case 402: + test_ahash_speed("md5", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + case 403: + test_ahash_speed("sha1", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + case 404: + test_ahash_speed("sha256", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + case 405: + test_ahash_speed("sha384", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + case 406: + test_ahash_speed("sha512", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + case 407: + test_ahash_speed("wp256", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + case 408: + test_ahash_speed("wp384", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + case 409: + test_ahash_speed("wp512", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + case 410: + test_ahash_speed("tgr128", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + case 411: + test_ahash_speed("tgr160", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + case 412: + test_ahash_speed("tgr192", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + case 413: + test_ahash_speed("sha224", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + case 414: + test_ahash_speed("rmd128", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + case 415: + test_ahash_speed("rmd160", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + case 416: + test_ahash_speed("rmd256", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + case 417: + test_ahash_speed("rmd320", sec, generic_hash_speed_template); + if (mode > 400 && mode < 500) break; + + case 499: + break; + case 1000: test_available(); break; @@ -885,6 +1223,12 @@ static int do_test(int m) return ret; } +static int do_alg_test(const char *alg, u32 type, u32 mask) +{ + return crypto_has_alg(alg, type, mask ?: CRYPTO_ALG_TYPE_MASK) ? + 0 : -ENOENT; +} + static int __init tcrypt_mod_init(void) { int err = -ENOMEM; @@ -896,7 +1240,11 @@ static int __init tcrypt_mod_init(void) goto err_free_tv; } - err = do_test(mode); + if (alg) + err = do_alg_test(alg, type, mask); + else + err = do_test(mode); + if (err) { printk(KERN_ERR "tcrypt: one or more tests failed!\n"); goto err_free_tv; @@ -928,6 +1276,9 @@ static void __exit tcrypt_mod_fini(void) { } module_init(tcrypt_mod_init); module_exit(tcrypt_mod_fini); +module_param(alg, charp, 0); +module_param(type, uint, 0); +module_param(mask, uint, 0); module_param(mode, int, 0); module_param(sec, uint, 0); MODULE_PARM_DESC(sec, "Length in seconds of speed tests " diff --git a/crypto/tcrypt.h b/crypto/tcrypt.h index 966bbfaf95b1..10cb925132c9 100644 --- a/crypto/tcrypt.h +++ b/crypto/tcrypt.h @@ -25,6 +25,7 @@ struct cipher_speed_template { struct hash_speed { unsigned int blen; /* buffer length */ unsigned int plen; /* per-update length */ + unsigned int klen; /* key length */ }; /* @@ -83,4 +84,32 @@ static struct hash_speed generic_hash_speed_template[] = { { .blen = 0, .plen = 0, } }; +static struct hash_speed hash_speed_template_16[] = { + { .blen = 16, .plen = 16, .klen = 16, }, + { .blen = 64, .plen = 16, .klen = 16, }, + { .blen = 64, .plen = 64, .klen = 16, }, + { .blen = 256, .plen = 16, .klen = 16, }, + { .blen = 256, .plen = 64, .klen = 16, }, + { .blen = 256, .plen = 256, .klen = 16, }, + { .blen = 1024, .plen = 16, .klen = 16, }, + { .blen = 1024, .plen = 256, .klen = 16, }, + { .blen = 1024, .plen = 1024, .klen = 16, }, + { .blen = 2048, .plen = 16, .klen = 16, }, + { .blen = 2048, .plen = 256, .klen = 16, }, + { .blen = 2048, .plen = 1024, .klen = 16, }, + { .blen = 2048, .plen = 2048, .klen = 16, }, + { .blen = 4096, .plen = 16, .klen = 16, }, + { .blen = 4096, .plen = 256, .klen = 16, }, + { .blen = 4096, .plen = 1024, .klen = 16, }, + { .blen = 4096, .plen = 4096, .klen = 16, }, + { .blen = 8192, .plen = 16, .klen = 16, }, + { .blen = 8192, .plen = 256, .klen = 16, }, + { .blen = 8192, .plen = 1024, .klen = 16, }, + { .blen = 8192, .plen = 4096, .klen = 16, }, + { .blen = 8192, .plen = 8192, .klen = 16, }, + + /* End marker */ + { .blen = 0, .plen = 0, .klen = 0, } +}; + #endif /* _CRYPTO_TCRYPT_H */ diff --git a/crypto/testmgr.c b/crypto/testmgr.c index e9e9d84293b9..fa8c8f78c8d4 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -22,6 +22,17 @@ #include <crypto/rng.h> #include "internal.h" + +#ifdef CONFIG_CRYPTO_MANAGER_DISABLE_TESTS + +/* a perfect nop */ +int alg_test(const char *driver, const char *alg, u32 type, u32 mask) +{ + return 0; +} + +#else + #include "testmgr.h" /* @@ -153,8 +164,21 @@ static void testmgr_free_buf(char *buf[XBUFSIZE]) free_page((unsigned long)buf[i]); } +static int do_one_async_hash_op(struct ahash_request *req, + struct tcrypt_result *tr, + int ret) +{ + if (ret == -EINPROGRESS || ret == -EBUSY) { + ret = wait_for_completion_interruptible(&tr->completion); + if (!ret) + ret = tr->err; + INIT_COMPLETION(tr->completion); + } + return ret; +} + static int test_hash(struct crypto_ahash *tfm, struct hash_testvec *template, - unsigned int tcount) + unsigned int tcount, bool use_digest) { const char *algo = crypto_tfm_alg_driver_name(crypto_ahash_tfm(tfm)); unsigned int i, j, k, temp; @@ -190,10 +214,6 @@ static int test_hash(struct crypto_ahash *tfm, struct hash_testvec *template, hash_buff = xbuf[0]; - ret = -EINVAL; - if (WARN_ON(template[i].psize > PAGE_SIZE)) - goto out; - memcpy(hash_buff, template[i].plaintext, template[i].psize); sg_init_one(&sg[0], hash_buff, template[i].psize); @@ -210,23 +230,36 @@ static int test_hash(struct crypto_ahash *tfm, struct hash_testvec *template, } ahash_request_set_crypt(req, sg, result, template[i].psize); - ret = crypto_ahash_digest(req); - switch (ret) { - case 0: - break; - case -EINPROGRESS: - case -EBUSY: - ret = wait_for_completion_interruptible( - &tresult.completion); - if (!ret && !(ret = tresult.err)) { - INIT_COMPLETION(tresult.completion); - break; + if (use_digest) { + ret = do_one_async_hash_op(req, &tresult, + crypto_ahash_digest(req)); + if (ret) { + pr_err("alg: hash: digest failed on test %d " + "for %s: ret=%d\n", j, algo, -ret); + goto out; + } + } else { + ret = do_one_async_hash_op(req, &tresult, + crypto_ahash_init(req)); + if (ret) { + pr_err("alt: hash: init failed on test %d " + "for %s: ret=%d\n", j, algo, -ret); + goto out; + } + ret = do_one_async_hash_op(req, &tresult, + crypto_ahash_update(req)); + if (ret) { + pr_err("alt: hash: update failed on test %d " + "for %s: ret=%d\n", j, algo, -ret); + goto out; + } + ret = do_one_async_hash_op(req, &tresult, + crypto_ahash_final(req)); + if (ret) { + pr_err("alt: hash: final failed on test %d " + "for %s: ret=%d\n", j, algo, -ret); + goto out; } - /* fall through */ - default: - printk(KERN_ERR "alg: hash: digest failed on test %d " - "for %s: ret=%d\n", j, algo, -ret); - goto out; } if (memcmp(result, template[i].digest, @@ -1205,7 +1238,7 @@ static int test_cprng(struct crypto_rng *tfm, struct cprng_testvec *template, unsigned int tcount) { const char *algo = crypto_tfm_alg_driver_name(crypto_rng_tfm(tfm)); - int err, i, j, seedsize; + int err = 0, i, j, seedsize; u8 *seed; char result[32]; @@ -1406,7 +1439,11 @@ static int alg_test_hash(const struct alg_test_desc *desc, const char *driver, return PTR_ERR(tfm); } - err = test_hash(tfm, desc->suite.hash.vecs, desc->suite.hash.count); + err = test_hash(tfm, desc->suite.hash.vecs, + desc->suite.hash.count, true); + if (!err) + err = test_hash(tfm, desc->suite.hash.vecs, + desc->suite.hash.count, false); crypto_free_ahash(tfm); return err; @@ -1481,9 +1518,54 @@ static int alg_test_cprng(const struct alg_test_desc *desc, const char *driver, return err; } +static int alg_test_null(const struct alg_test_desc *desc, + const char *driver, u32 type, u32 mask) +{ + return 0; +} + /* Please keep this list sorted by algorithm name. */ static const struct alg_test_desc alg_test_descs[] = { { + .alg = "__driver-cbc-aes-aesni", + .test = alg_test_null, + .suite = { + .cipher = { + .enc = { + .vecs = NULL, + .count = 0 + }, + .dec = { + .vecs = NULL, + .count = 0 + } + } + } + }, { + .alg = "__driver-ecb-aes-aesni", + .test = alg_test_null, + .suite = { + .cipher = { + .enc = { + .vecs = NULL, + .count = 0 + }, + .dec = { + .vecs = NULL, + .count = 0 + } + } + } + }, { + .alg = "__ghash-pclmulqdqni", + .test = alg_test_null, + .suite = { + .hash = { + .vecs = NULL, + .count = 0 + } + } + }, { .alg = "ansi_cprng", .test = alg_test_cprng, .fips_allowed = 1, @@ -1627,6 +1709,30 @@ static const struct alg_test_desc alg_test_descs[] = { } } }, { + .alg = "cryptd(__driver-ecb-aes-aesni)", + .test = alg_test_null, + .suite = { + .cipher = { + .enc = { + .vecs = NULL, + .count = 0 + }, + .dec = { + .vecs = NULL, + .count = 0 + } + } + } + }, { + .alg = "cryptd(__ghash-pclmulqdqni)", + .test = alg_test_null, + .suite = { + .hash = { + .vecs = NULL, + .count = 0 + } + } + }, { .alg = "ctr(aes)", .test = alg_test_skcipher, .fips_allowed = 1, @@ -1673,6 +1779,21 @@ static const struct alg_test_desc alg_test_descs[] = { } } }, { + .alg = "ecb(__aes-aesni)", + .test = alg_test_null, + .suite = { + .cipher = { + .enc = { + .vecs = NULL, + .count = 0 + }, + .dec = { + .vecs = NULL, + .count = 0 + } + } + } + }, { .alg = "ecb(aes)", .test = alg_test_skcipher, .fips_allowed = 1, @@ -1947,6 +2068,15 @@ static const struct alg_test_desc alg_test_descs[] = { } } }, { + .alg = "ghash", + .test = alg_test_hash, + .suite = { + .hash = { + .vecs = ghash_tv_template, + .count = GHASH_TEST_VECTORS + } + } + }, { .alg = "hmac(md5)", .test = alg_test_hash, .suite = { @@ -2252,6 +2382,15 @@ static const struct alg_test_desc alg_test_descs[] = { } } }, { + .alg = "vmac(aes)", + .test = alg_test_hash, + .suite = { + .hash = { + .vecs = aes_vmac128_tv_template, + .count = VMAC_AES_TEST_VECTORS + } + } + }, { .alg = "wp256", .test = alg_test_hash, .suite = { @@ -2348,6 +2487,7 @@ static int alg_find_test(const char *alg) int alg_test(const char *driver, const char *alg, u32 type, u32 mask) { int i; + int j; int rc; if ((type & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_CIPHER) { @@ -2369,14 +2509,22 @@ int alg_test(const char *driver, const char *alg, u32 type, u32 mask) } i = alg_find_test(alg); - if (i < 0) + j = alg_find_test(driver); + if (i < 0 && j < 0) goto notest; - if (fips_enabled && !alg_test_descs[i].fips_allowed) + if (fips_enabled && ((i >= 0 && !alg_test_descs[i].fips_allowed) || + (j >= 0 && !alg_test_descs[j].fips_allowed))) goto non_fips_alg; - rc = alg_test_descs[i].test(alg_test_descs + i, driver, - type, mask); + rc = 0; + if (i >= 0) + rc |= alg_test_descs[i].test(alg_test_descs + i, driver, + type, mask); + if (j >= 0) + rc |= alg_test_descs[j].test(alg_test_descs + j, driver, + type, mask); + test_done: if (fips_enabled && rc) panic("%s: %s alg self test failed in fips mode!\n", driver, alg); @@ -2393,4 +2541,7 @@ notest: non_fips_alg: return -EINVAL; } + +#endif /* CONFIG_CRYPTO_MANAGER_DISABLE_TESTS */ + EXPORT_SYMBOL_GPL(alg_test); diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 69316228fc19..74e35377fd30 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -1003,6 +1003,21 @@ static struct hash_testvec tgr128_tv_template[] = { }, }; +#define GHASH_TEST_VECTORS 1 + +static struct hash_testvec ghash_tv_template[] = +{ + { + + .key = "\xdf\xa6\xbf\x4d\xed\x81\xdb\x03\xff\xca\xff\x95\xf8\x30\xf0\x61", + .ksize = 16, + .plaintext = "\x95\x2b\x2a\x56\xa5\x60\x04a\xc0\xb3\x2b\x66\x56\xa0\x5b\x40\xb6", + .psize = 16, + .digest = "\xda\x53\xeb\x0a\xd2\xc5\x5b\xb6" + "\x4f\xc4\x80\x2c\xc3\xfe\xda\x60", + }, +}; + /* * HMAC-MD5 test vectors from RFC2202 * (These need to be fixed to not use strlen). @@ -1654,6 +1669,78 @@ static struct hash_testvec aes_xcbc128_tv_template[] = { } }; +#define VMAC_AES_TEST_VECTORS 8 +static char vmac_string1[128] = {'\x01', '\x01', '\x01', '\x01', + '\x02', '\x03', '\x02', '\x02', + '\x02', '\x04', '\x01', '\x07', + '\x04', '\x01', '\x04', '\x03',}; +static char vmac_string2[128] = {'a', 'b', 'c',}; +static char vmac_string3[128] = {'a', 'b', 'c', 'a', 'b', 'c', + 'a', 'b', 'c', 'a', 'b', 'c', + 'a', 'b', 'c', 'a', 'b', 'c', + 'a', 'b', 'c', 'a', 'b', 'c', + 'a', 'b', 'c', 'a', 'b', 'c', + 'a', 'b', 'c', 'a', 'b', 'c', + 'a', 'b', 'c', 'a', 'b', 'c', + 'a', 'b', 'c', 'a', 'b', 'c', + }; + +static struct hash_testvec aes_vmac128_tv_template[] = { + { + .key = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + .plaintext = NULL, + .digest = "\x07\x58\x80\x35\x77\xa4\x7b\x54", + .psize = 0, + .ksize = 16, + }, { + .key = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + .plaintext = vmac_string1, + .digest = "\xce\xf5\x3c\xd3\xae\x68\x8c\xa1", + .psize = 128, + .ksize = 16, + }, { + .key = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + .plaintext = vmac_string2, + .digest = "\xc9\x27\xb0\x73\x81\xbd\x14\x2d", + .psize = 128, + .ksize = 16, + }, { + .key = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + .plaintext = vmac_string3, + .digest = "\x8d\x1a\x95\x8c\x98\x47\x0b\x19", + .psize = 128, + .ksize = 16, + }, { + .key = "abcdefghijklmnop", + .plaintext = NULL, + .digest = "\x3b\x89\xa1\x26\x9e\x55\x8f\x84", + .psize = 0, + .ksize = 16, + }, { + .key = "abcdefghijklmnop", + .plaintext = vmac_string1, + .digest = "\xab\x5e\xab\xb0\xf6\x8d\x74\xc2", + .psize = 128, + .ksize = 16, + }, { + .key = "abcdefghijklmnop", + .plaintext = vmac_string2, + .digest = "\x11\x15\x68\x42\x3d\x7b\x09\xdf", + .psize = 128, + .ksize = 16, + }, { + .key = "abcdefghijklmnop", + .plaintext = vmac_string3, + .digest = "\x8b\x32\x8f\xe1\xed\x8f\xfa\xd4", + .psize = 128, + .ksize = 16, + }, +}; + /* * SHA384 HMAC test vectors from RFC4231 */ diff --git a/crypto/twofish.c b/crypto/twofish_generic.c index dfcda231f87a..1f07b843e07c 100644 --- a/crypto/twofish.c +++ b/crypto/twofish_generic.c @@ -212,3 +212,4 @@ module_exit(twofish_mod_fini); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION ("Twofish Cipher Algorithm"); +MODULE_ALIAS("twofish"); diff --git a/crypto/vmac.c b/crypto/vmac.c new file mode 100644 index 000000000000..0999274a27ac --- /dev/null +++ b/crypto/vmac.c @@ -0,0 +1,675 @@ +/* + * Modified to interface to the Linux kernel + * Copyright (c) 2009, Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple + * Place - Suite 330, Boston, MA 02111-1307 USA. + */ + +/* -------------------------------------------------------------------------- + * VMAC and VHASH Implementation by Ted Krovetz (tdk@acm.org) and Wei Dai. + * This implementation is herby placed in the public domain. + * The authors offers no warranty. Use at your own risk. + * Please send bug reports to the authors. + * Last modified: 17 APR 08, 1700 PDT + * ----------------------------------------------------------------------- */ + +#include <linux/init.h> +#include <linux/types.h> +#include <linux/crypto.h> +#include <linux/scatterlist.h> +#include <asm/byteorder.h> +#include <crypto/scatterwalk.h> +#include <crypto/vmac.h> +#include <crypto/internal/hash.h> + +/* + * Constants and masks + */ +#define UINT64_C(x) x##ULL +const u64 p64 = UINT64_C(0xfffffffffffffeff); /* 2^64 - 257 prime */ +const u64 m62 = UINT64_C(0x3fffffffffffffff); /* 62-bit mask */ +const u64 m63 = UINT64_C(0x7fffffffffffffff); /* 63-bit mask */ +const u64 m64 = UINT64_C(0xffffffffffffffff); /* 64-bit mask */ +const u64 mpoly = UINT64_C(0x1fffffff1fffffff); /* Poly key mask */ + +#define pe64_to_cpup le64_to_cpup /* Prefer little endian */ + +#ifdef __LITTLE_ENDIAN +#define INDEX_HIGH 1 +#define INDEX_LOW 0 +#else +#define INDEX_HIGH 0 +#define INDEX_LOW 1 +#endif + +/* + * The following routines are used in this implementation. They are + * written via macros to simulate zero-overhead call-by-reference. + * + * MUL64: 64x64->128-bit multiplication + * PMUL64: assumes top bits cleared on inputs + * ADD128: 128x128->128-bit addition + */ + +#define ADD128(rh, rl, ih, il) \ + do { \ + u64 _il = (il); \ + (rl) += (_il); \ + if ((rl) < (_il)) \ + (rh)++; \ + (rh) += (ih); \ + } while (0) + +#define MUL32(i1, i2) ((u64)(u32)(i1)*(u32)(i2)) + +#define PMUL64(rh, rl, i1, i2) /* Assumes m doesn't overflow */ \ + do { \ + u64 _i1 = (i1), _i2 = (i2); \ + u64 m = MUL32(_i1, _i2>>32) + MUL32(_i1>>32, _i2); \ + rh = MUL32(_i1>>32, _i2>>32); \ + rl = MUL32(_i1, _i2); \ + ADD128(rh, rl, (m >> 32), (m << 32)); \ + } while (0) + +#define MUL64(rh, rl, i1, i2) \ + do { \ + u64 _i1 = (i1), _i2 = (i2); \ + u64 m1 = MUL32(_i1, _i2>>32); \ + u64 m2 = MUL32(_i1>>32, _i2); \ + rh = MUL32(_i1>>32, _i2>>32); \ + rl = MUL32(_i1, _i2); \ + ADD128(rh, rl, (m1 >> 32), (m1 << 32)); \ + ADD128(rh, rl, (m2 >> 32), (m2 << 32)); \ + } while (0) + +/* + * For highest performance the L1 NH and L2 polynomial hashes should be + * carefully implemented to take advantage of one's target architechture. + * Here these two hash functions are defined multiple time; once for + * 64-bit architectures, once for 32-bit SSE2 architectures, and once + * for the rest (32-bit) architectures. + * For each, nh_16 *must* be defined (works on multiples of 16 bytes). + * Optionally, nh_vmac_nhbytes can be defined (for multiples of + * VMAC_NHBYTES), and nh_16_2 and nh_vmac_nhbytes_2 (versions that do two + * NH computations at once). + */ + +#ifdef CONFIG_64BIT + +#define nh_16(mp, kp, nw, rh, rl) \ + do { \ + int i; u64 th, tl; \ + rh = rl = 0; \ + for (i = 0; i < nw; i += 2) { \ + MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \ + pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \ + ADD128(rh, rl, th, tl); \ + } \ + } while (0) + +#define nh_16_2(mp, kp, nw, rh, rl, rh1, rl1) \ + do { \ + int i; u64 th, tl; \ + rh1 = rl1 = rh = rl = 0; \ + for (i = 0; i < nw; i += 2) { \ + MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \ + pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \ + ADD128(rh, rl, th, tl); \ + MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2], \ + pe64_to_cpup((mp)+i+1)+(kp)[i+3]); \ + ADD128(rh1, rl1, th, tl); \ + } \ + } while (0) + +#if (VMAC_NHBYTES >= 64) /* These versions do 64-bytes of message at a time */ +#define nh_vmac_nhbytes(mp, kp, nw, rh, rl) \ + do { \ + int i; u64 th, tl; \ + rh = rl = 0; \ + for (i = 0; i < nw; i += 8) { \ + MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \ + pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \ + ADD128(rh, rl, th, tl); \ + MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2], \ + pe64_to_cpup((mp)+i+3)+(kp)[i+3]); \ + ADD128(rh, rl, th, tl); \ + MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4], \ + pe64_to_cpup((mp)+i+5)+(kp)[i+5]); \ + ADD128(rh, rl, th, tl); \ + MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6], \ + pe64_to_cpup((mp)+i+7)+(kp)[i+7]); \ + ADD128(rh, rl, th, tl); \ + } \ + } while (0) + +#define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh1, rl1) \ + do { \ + int i; u64 th, tl; \ + rh1 = rl1 = rh = rl = 0; \ + for (i = 0; i < nw; i += 8) { \ + MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i], \ + pe64_to_cpup((mp)+i+1)+(kp)[i+1]); \ + ADD128(rh, rl, th, tl); \ + MUL64(th, tl, pe64_to_cpup((mp)+i)+(kp)[i+2], \ + pe64_to_cpup((mp)+i+1)+(kp)[i+3]); \ + ADD128(rh1, rl1, th, tl); \ + MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+2], \ + pe64_to_cpup((mp)+i+3)+(kp)[i+3]); \ + ADD128(rh, rl, th, tl); \ + MUL64(th, tl, pe64_to_cpup((mp)+i+2)+(kp)[i+4], \ + pe64_to_cpup((mp)+i+3)+(kp)[i+5]); \ + ADD128(rh1, rl1, th, tl); \ + MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+4], \ + pe64_to_cpup((mp)+i+5)+(kp)[i+5]); \ + ADD128(rh, rl, th, tl); \ + MUL64(th, tl, pe64_to_cpup((mp)+i+4)+(kp)[i+6], \ + pe64_to_cpup((mp)+i+5)+(kp)[i+7]); \ + ADD128(rh1, rl1, th, tl); \ + MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+6], \ + pe64_to_cpup((mp)+i+7)+(kp)[i+7]); \ + ADD128(rh, rl, th, tl); \ + MUL64(th, tl, pe64_to_cpup((mp)+i+6)+(kp)[i+8], \ + pe64_to_cpup((mp)+i+7)+(kp)[i+9]); \ + ADD128(rh1, rl1, th, tl); \ + } \ + } while (0) +#endif + +#define poly_step(ah, al, kh, kl, mh, ml) \ + do { \ + u64 t1h, t1l, t2h, t2l, t3h, t3l, z = 0; \ + /* compute ab*cd, put bd into result registers */ \ + PMUL64(t3h, t3l, al, kh); \ + PMUL64(t2h, t2l, ah, kl); \ + PMUL64(t1h, t1l, ah, 2*kh); \ + PMUL64(ah, al, al, kl); \ + /* add 2 * ac to result */ \ + ADD128(ah, al, t1h, t1l); \ + /* add together ad + bc */ \ + ADD128(t2h, t2l, t3h, t3l); \ + /* now (ah,al), (t2l,2*t2h) need summing */ \ + /* first add the high registers, carrying into t2h */ \ + ADD128(t2h, ah, z, t2l); \ + /* double t2h and add top bit of ah */ \ + t2h = 2 * t2h + (ah >> 63); \ + ah &= m63; \ + /* now add the low registers */ \ + ADD128(ah, al, mh, ml); \ + ADD128(ah, al, z, t2h); \ + } while (0) + +#else /* ! CONFIG_64BIT */ + +#ifndef nh_16 +#define nh_16(mp, kp, nw, rh, rl) \ + do { \ + u64 t1, t2, m1, m2, t; \ + int i; \ + rh = rl = t = 0; \ + for (i = 0; i < nw; i += 2) { \ + t1 = pe64_to_cpup(mp+i) + kp[i]; \ + t2 = pe64_to_cpup(mp+i+1) + kp[i+1]; \ + m2 = MUL32(t1 >> 32, t2); \ + m1 = MUL32(t1, t2 >> 32); \ + ADD128(rh, rl, MUL32(t1 >> 32, t2 >> 32), \ + MUL32(t1, t2)); \ + rh += (u64)(u32)(m1 >> 32) \ + + (u32)(m2 >> 32); \ + t += (u64)(u32)m1 + (u32)m2; \ + } \ + ADD128(rh, rl, (t >> 32), (t << 32)); \ + } while (0) +#endif + +static void poly_step_func(u64 *ahi, u64 *alo, + const u64 *kh, const u64 *kl, + const u64 *mh, const u64 *ml) +{ +#define a0 (*(((u32 *)alo)+INDEX_LOW)) +#define a1 (*(((u32 *)alo)+INDEX_HIGH)) +#define a2 (*(((u32 *)ahi)+INDEX_LOW)) +#define a3 (*(((u32 *)ahi)+INDEX_HIGH)) +#define k0 (*(((u32 *)kl)+INDEX_LOW)) +#define k1 (*(((u32 *)kl)+INDEX_HIGH)) +#define k2 (*(((u32 *)kh)+INDEX_LOW)) +#define k3 (*(((u32 *)kh)+INDEX_HIGH)) + + u64 p, q, t; + u32 t2; + + p = MUL32(a3, k3); + p += p; + p += *(u64 *)mh; + p += MUL32(a0, k2); + p += MUL32(a1, k1); + p += MUL32(a2, k0); + t = (u32)(p); + p >>= 32; + p += MUL32(a0, k3); + p += MUL32(a1, k2); + p += MUL32(a2, k1); + p += MUL32(a3, k0); + t |= ((u64)((u32)p & 0x7fffffff)) << 32; + p >>= 31; + p += (u64)(((u32 *)ml)[INDEX_LOW]); + p += MUL32(a0, k0); + q = MUL32(a1, k3); + q += MUL32(a2, k2); + q += MUL32(a3, k1); + q += q; + p += q; + t2 = (u32)(p); + p >>= 32; + p += (u64)(((u32 *)ml)[INDEX_HIGH]); + p += MUL32(a0, k1); + p += MUL32(a1, k0); + q = MUL32(a2, k3); + q += MUL32(a3, k2); + q += q; + p += q; + *(u64 *)(alo) = (p << 32) | t2; + p >>= 32; + *(u64 *)(ahi) = p + t; + +#undef a0 +#undef a1 +#undef a2 +#undef a3 +#undef k0 +#undef k1 +#undef k2 +#undef k3 +} + +#define poly_step(ah, al, kh, kl, mh, ml) \ + poly_step_func(&(ah), &(al), &(kh), &(kl), &(mh), &(ml)) + +#endif /* end of specialized NH and poly definitions */ + +/* At least nh_16 is defined. Defined others as needed here */ +#ifndef nh_16_2 +#define nh_16_2(mp, kp, nw, rh, rl, rh2, rl2) \ + do { \ + nh_16(mp, kp, nw, rh, rl); \ + nh_16(mp, ((kp)+2), nw, rh2, rl2); \ + } while (0) +#endif +#ifndef nh_vmac_nhbytes +#define nh_vmac_nhbytes(mp, kp, nw, rh, rl) \ + nh_16(mp, kp, nw, rh, rl) +#endif +#ifndef nh_vmac_nhbytes_2 +#define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh2, rl2) \ + do { \ + nh_vmac_nhbytes(mp, kp, nw, rh, rl); \ + nh_vmac_nhbytes(mp, ((kp)+2), nw, rh2, rl2); \ + } while (0) +#endif + +static void vhash_abort(struct vmac_ctx *ctx) +{ + ctx->polytmp[0] = ctx->polykey[0] ; + ctx->polytmp[1] = ctx->polykey[1] ; + ctx->first_block_processed = 0; +} + +static u64 l3hash(u64 p1, u64 p2, u64 k1, u64 k2, u64 len) +{ + u64 rh, rl, t, z = 0; + + /* fully reduce (p1,p2)+(len,0) mod p127 */ + t = p1 >> 63; + p1 &= m63; + ADD128(p1, p2, len, t); + /* At this point, (p1,p2) is at most 2^127+(len<<64) */ + t = (p1 > m63) + ((p1 == m63) && (p2 == m64)); + ADD128(p1, p2, z, t); + p1 &= m63; + + /* compute (p1,p2)/(2^64-2^32) and (p1,p2)%(2^64-2^32) */ + t = p1 + (p2 >> 32); + t += (t >> 32); + t += (u32)t > 0xfffffffeu; + p1 += (t >> 32); + p2 += (p1 << 32); + + /* compute (p1+k1)%p64 and (p2+k2)%p64 */ + p1 += k1; + p1 += (0 - (p1 < k1)) & 257; + p2 += k2; + p2 += (0 - (p2 < k2)) & 257; + + /* compute (p1+k1)*(p2+k2)%p64 */ + MUL64(rh, rl, p1, p2); + t = rh >> 56; + ADD128(t, rl, z, rh); + rh <<= 8; + ADD128(t, rl, z, rh); + t += t << 8; + rl += t; + rl += (0 - (rl < t)) & 257; + rl += (0 - (rl > p64-1)) & 257; + return rl; +} + +static void vhash_update(const unsigned char *m, + unsigned int mbytes, /* Pos multiple of VMAC_NHBYTES */ + struct vmac_ctx *ctx) +{ + u64 rh, rl, *mptr; + const u64 *kptr = (u64 *)ctx->nhkey; + int i; + u64 ch, cl; + u64 pkh = ctx->polykey[0]; + u64 pkl = ctx->polykey[1]; + + mptr = (u64 *)m; + i = mbytes / VMAC_NHBYTES; /* Must be non-zero */ + + ch = ctx->polytmp[0]; + cl = ctx->polytmp[1]; + + if (!ctx->first_block_processed) { + ctx->first_block_processed = 1; + nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl); + rh &= m62; + ADD128(ch, cl, rh, rl); + mptr += (VMAC_NHBYTES/sizeof(u64)); + i--; + } + + while (i--) { + nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl); + rh &= m62; + poly_step(ch, cl, pkh, pkl, rh, rl); + mptr += (VMAC_NHBYTES/sizeof(u64)); + } + + ctx->polytmp[0] = ch; + ctx->polytmp[1] = cl; +} + +static u64 vhash(unsigned char m[], unsigned int mbytes, + u64 *tagl, struct vmac_ctx *ctx) +{ + u64 rh, rl, *mptr; + const u64 *kptr = (u64 *)ctx->nhkey; + int i, remaining; + u64 ch, cl; + u64 pkh = ctx->polykey[0]; + u64 pkl = ctx->polykey[1]; + + mptr = (u64 *)m; + i = mbytes / VMAC_NHBYTES; + remaining = mbytes % VMAC_NHBYTES; + + if (ctx->first_block_processed) { + ch = ctx->polytmp[0]; + cl = ctx->polytmp[1]; + } else if (i) { + nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, ch, cl); + ch &= m62; + ADD128(ch, cl, pkh, pkl); + mptr += (VMAC_NHBYTES/sizeof(u64)); + i--; + } else if (remaining) { + nh_16(mptr, kptr, 2*((remaining+15)/16), ch, cl); + ch &= m62; + ADD128(ch, cl, pkh, pkl); + mptr += (VMAC_NHBYTES/sizeof(u64)); + goto do_l3; + } else {/* Empty String */ + ch = pkh; cl = pkl; + goto do_l3; + } + + while (i--) { + nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl); + rh &= m62; + poly_step(ch, cl, pkh, pkl, rh, rl); + mptr += (VMAC_NHBYTES/sizeof(u64)); + } + if (remaining) { + nh_16(mptr, kptr, 2*((remaining+15)/16), rh, rl); + rh &= m62; + poly_step(ch, cl, pkh, pkl, rh, rl); + } + +do_l3: + vhash_abort(ctx); + remaining *= 8; + return l3hash(ch, cl, ctx->l3key[0], ctx->l3key[1], remaining); +} + +static u64 vmac(unsigned char m[], unsigned int mbytes, + unsigned char n[16], u64 *tagl, + struct vmac_ctx_t *ctx) +{ + u64 *in_n, *out_p; + u64 p, h; + int i; + + in_n = ctx->__vmac_ctx.cached_nonce; + out_p = ctx->__vmac_ctx.cached_aes; + + i = n[15] & 1; + if ((*(u64 *)(n+8) != in_n[1]) || (*(u64 *)(n) != in_n[0])) { + in_n[0] = *(u64 *)(n); + in_n[1] = *(u64 *)(n+8); + ((unsigned char *)in_n)[15] &= 0xFE; + crypto_cipher_encrypt_one(ctx->child, + (unsigned char *)out_p, (unsigned char *)in_n); + + ((unsigned char *)in_n)[15] |= (unsigned char)(1-i); + } + p = be64_to_cpup(out_p + i); + h = vhash(m, mbytes, (u64 *)0, &ctx->__vmac_ctx); + return le64_to_cpu(p + h); +} + +static int vmac_set_key(unsigned char user_key[], struct vmac_ctx_t *ctx) +{ + u64 in[2] = {0}, out[2]; + unsigned i; + int err = 0; + + err = crypto_cipher_setkey(ctx->child, user_key, VMAC_KEY_LEN); + if (err) + return err; + + /* Fill nh key */ + ((unsigned char *)in)[0] = 0x80; + for (i = 0; i < sizeof(ctx->__vmac_ctx.nhkey)/8; i += 2) { + crypto_cipher_encrypt_one(ctx->child, + (unsigned char *)out, (unsigned char *)in); + ctx->__vmac_ctx.nhkey[i] = be64_to_cpup(out); + ctx->__vmac_ctx.nhkey[i+1] = be64_to_cpup(out+1); + ((unsigned char *)in)[15] += 1; + } + + /* Fill poly key */ + ((unsigned char *)in)[0] = 0xC0; + in[1] = 0; + for (i = 0; i < sizeof(ctx->__vmac_ctx.polykey)/8; i += 2) { + crypto_cipher_encrypt_one(ctx->child, + (unsigned char *)out, (unsigned char *)in); + ctx->__vmac_ctx.polytmp[i] = + ctx->__vmac_ctx.polykey[i] = + be64_to_cpup(out) & mpoly; + ctx->__vmac_ctx.polytmp[i+1] = + ctx->__vmac_ctx.polykey[i+1] = + be64_to_cpup(out+1) & mpoly; + ((unsigned char *)in)[15] += 1; + } + + /* Fill ip key */ + ((unsigned char *)in)[0] = 0xE0; + in[1] = 0; + for (i = 0; i < sizeof(ctx->__vmac_ctx.l3key)/8; i += 2) { + do { + crypto_cipher_encrypt_one(ctx->child, + (unsigned char *)out, (unsigned char *)in); + ctx->__vmac_ctx.l3key[i] = be64_to_cpup(out); + ctx->__vmac_ctx.l3key[i+1] = be64_to_cpup(out+1); + ((unsigned char *)in)[15] += 1; + } while (ctx->__vmac_ctx.l3key[i] >= p64 + || ctx->__vmac_ctx.l3key[i+1] >= p64); + } + + /* Invalidate nonce/aes cache and reset other elements */ + ctx->__vmac_ctx.cached_nonce[0] = (u64)-1; /* Ensure illegal nonce */ + ctx->__vmac_ctx.cached_nonce[1] = (u64)0; /* Ensure illegal nonce */ + ctx->__vmac_ctx.first_block_processed = 0; + + return err; +} + +static int vmac_setkey(struct crypto_shash *parent, + const u8 *key, unsigned int keylen) +{ + struct vmac_ctx_t *ctx = crypto_shash_ctx(parent); + + if (keylen != VMAC_KEY_LEN) { + crypto_shash_set_flags(parent, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + return vmac_set_key((u8 *)key, ctx); +} + +static int vmac_init(struct shash_desc *pdesc) +{ + return 0; +} + +static int vmac_update(struct shash_desc *pdesc, const u8 *p, + unsigned int len) +{ + struct crypto_shash *parent = pdesc->tfm; + struct vmac_ctx_t *ctx = crypto_shash_ctx(parent); + + vhash_update(p, len, &ctx->__vmac_ctx); + + return 0; +} + +static int vmac_final(struct shash_desc *pdesc, u8 *out) +{ + struct crypto_shash *parent = pdesc->tfm; + struct vmac_ctx_t *ctx = crypto_shash_ctx(parent); + vmac_t mac; + u8 nonce[16] = {}; + + mac = vmac(NULL, 0, nonce, NULL, ctx); + memcpy(out, &mac, sizeof(vmac_t)); + memset(&mac, 0, sizeof(vmac_t)); + memset(&ctx->__vmac_ctx, 0, sizeof(struct vmac_ctx)); + return 0; +} + +static int vmac_init_tfm(struct crypto_tfm *tfm) +{ + struct crypto_cipher *cipher; + struct crypto_instance *inst = (void *)tfm->__crt_alg; + struct crypto_spawn *spawn = crypto_instance_ctx(inst); + struct vmac_ctx_t *ctx = crypto_tfm_ctx(tfm); + + cipher = crypto_spawn_cipher(spawn); + if (IS_ERR(cipher)) + return PTR_ERR(cipher); + + ctx->child = cipher; + return 0; +} + +static void vmac_exit_tfm(struct crypto_tfm *tfm) +{ + struct vmac_ctx_t *ctx = crypto_tfm_ctx(tfm); + crypto_free_cipher(ctx->child); +} + +static int vmac_create(struct crypto_template *tmpl, struct rtattr **tb) +{ + struct shash_instance *inst; + struct crypto_alg *alg; + int err; + + err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH); + if (err) + return err; + + alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER, + CRYPTO_ALG_TYPE_MASK); + if (IS_ERR(alg)) + return PTR_ERR(alg); + + inst = shash_alloc_instance("vmac", alg); + err = PTR_ERR(inst); + if (IS_ERR(inst)) + goto out_put_alg; + + err = crypto_init_spawn(shash_instance_ctx(inst), alg, + shash_crypto_instance(inst), + CRYPTO_ALG_TYPE_MASK); + if (err) + goto out_free_inst; + + inst->alg.base.cra_priority = alg->cra_priority; + inst->alg.base.cra_blocksize = alg->cra_blocksize; + inst->alg.base.cra_alignmask = alg->cra_alignmask; + + inst->alg.digestsize = sizeof(vmac_t); + inst->alg.base.cra_ctxsize = sizeof(struct vmac_ctx_t); + inst->alg.base.cra_init = vmac_init_tfm; + inst->alg.base.cra_exit = vmac_exit_tfm; + + inst->alg.init = vmac_init; + inst->alg.update = vmac_update; + inst->alg.final = vmac_final; + inst->alg.setkey = vmac_setkey; + + err = shash_register_instance(tmpl, inst); + if (err) { +out_free_inst: + shash_free_instance(shash_crypto_instance(inst)); + } + +out_put_alg: + crypto_mod_put(alg); + return err; +} + +static struct crypto_template vmac_tmpl = { + .name = "vmac", + .create = vmac_create, + .free = shash_free_instance, + .module = THIS_MODULE, +}; + +static int __init vmac_module_init(void) +{ + return crypto_register_template(&vmac_tmpl); +} + +static void __exit vmac_module_exit(void) +{ + crypto_unregister_template(&vmac_tmpl); +} + +module_init(vmac_module_init); +module_exit(vmac_module_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("VMAC hash algorithm"); + diff --git a/crypto/xcbc.c b/crypto/xcbc.c index b63b633e549c..bb7b67fba349 100644 --- a/crypto/xcbc.c +++ b/crypto/xcbc.c @@ -19,211 +19,142 @@ * Kazunori Miyazawa <miyazawa@linux-ipv6.org> */ -#include <crypto/scatterwalk.h> -#include <linux/crypto.h> +#include <crypto/internal/hash.h> #include <linux/err.h> -#include <linux/hardirq.h> #include <linux/kernel.h> -#include <linux/mm.h> -#include <linux/rtnetlink.h> -#include <linux/slab.h> -#include <linux/scatterlist.h> static u_int32_t ks[12] = {0x01010101, 0x01010101, 0x01010101, 0x01010101, 0x02020202, 0x02020202, 0x02020202, 0x02020202, 0x03030303, 0x03030303, 0x03030303, 0x03030303}; + /* * +------------------------ * | <parent tfm> * +------------------------ - * | crypto_xcbc_ctx + * | xcbc_tfm_ctx * +------------------------ - * | odds (block size) + * | consts (block size * 2) * +------------------------ - * | prev (block size) + */ +struct xcbc_tfm_ctx { + struct crypto_cipher *child; + u8 ctx[]; +}; + +/* * +------------------------ - * | key (block size) + * | <shash desc> * +------------------------ - * | consts (block size * 3) + * | xcbc_desc_ctx + * +------------------------ + * | odds (block size) + * +------------------------ + * | prev (block size) * +------------------------ */ -struct crypto_xcbc_ctx { - struct crypto_cipher *child; - u8 *odds; - u8 *prev; - u8 *key; - u8 *consts; - void (*xor)(u8 *a, const u8 *b, unsigned int bs); - unsigned int keylen; +struct xcbc_desc_ctx { unsigned int len; + u8 ctx[]; }; -static void xor_128(u8 *a, const u8 *b, unsigned int bs) -{ - ((u32 *)a)[0] ^= ((u32 *)b)[0]; - ((u32 *)a)[1] ^= ((u32 *)b)[1]; - ((u32 *)a)[2] ^= ((u32 *)b)[2]; - ((u32 *)a)[3] ^= ((u32 *)b)[3]; -} - -static int _crypto_xcbc_digest_setkey(struct crypto_hash *parent, - struct crypto_xcbc_ctx *ctx) +static int crypto_xcbc_digest_setkey(struct crypto_shash *parent, + const u8 *inkey, unsigned int keylen) { - int bs = crypto_hash_blocksize(parent); + unsigned long alignmask = crypto_shash_alignmask(parent); + struct xcbc_tfm_ctx *ctx = crypto_shash_ctx(parent); + int bs = crypto_shash_blocksize(parent); + u8 *consts = PTR_ALIGN(&ctx->ctx[0], alignmask + 1); int err = 0; u8 key1[bs]; - if ((err = crypto_cipher_setkey(ctx->child, ctx->key, ctx->keylen))) - return err; + if ((err = crypto_cipher_setkey(ctx->child, inkey, keylen))) + return err; - crypto_cipher_encrypt_one(ctx->child, key1, ctx->consts); + crypto_cipher_encrypt_one(ctx->child, consts, (u8 *)ks + bs); + crypto_cipher_encrypt_one(ctx->child, consts + bs, (u8 *)ks + bs * 2); + crypto_cipher_encrypt_one(ctx->child, key1, (u8 *)ks); return crypto_cipher_setkey(ctx->child, key1, bs); -} - -static int crypto_xcbc_digest_setkey(struct crypto_hash *parent, - const u8 *inkey, unsigned int keylen) -{ - struct crypto_xcbc_ctx *ctx = crypto_hash_ctx_aligned(parent); - - if (keylen != crypto_cipher_blocksize(ctx->child)) - return -EINVAL; - ctx->keylen = keylen; - memcpy(ctx->key, inkey, keylen); - ctx->consts = (u8*)ks; - - return _crypto_xcbc_digest_setkey(parent, ctx); } -static int crypto_xcbc_digest_init(struct hash_desc *pdesc) +static int crypto_xcbc_digest_init(struct shash_desc *pdesc) { - struct crypto_xcbc_ctx *ctx = crypto_hash_ctx_aligned(pdesc->tfm); - int bs = crypto_hash_blocksize(pdesc->tfm); + unsigned long alignmask = crypto_shash_alignmask(pdesc->tfm); + struct xcbc_desc_ctx *ctx = shash_desc_ctx(pdesc); + int bs = crypto_shash_blocksize(pdesc->tfm); + u8 *prev = PTR_ALIGN(&ctx->ctx[0], alignmask + 1) + bs; ctx->len = 0; - memset(ctx->odds, 0, bs); - memset(ctx->prev, 0, bs); + memset(prev, 0, bs); return 0; } -static int crypto_xcbc_digest_update2(struct hash_desc *pdesc, - struct scatterlist *sg, - unsigned int nbytes) +static int crypto_xcbc_digest_update(struct shash_desc *pdesc, const u8 *p, + unsigned int len) { - struct crypto_hash *parent = pdesc->tfm; - struct crypto_xcbc_ctx *ctx = crypto_hash_ctx_aligned(parent); - struct crypto_cipher *tfm = ctx->child; - int bs = crypto_hash_blocksize(parent); - - for (;;) { - struct page *pg = sg_page(sg); - unsigned int offset = sg->offset; - unsigned int slen = sg->length; - - if (unlikely(slen > nbytes)) - slen = nbytes; - - nbytes -= slen; - - while (slen > 0) { - unsigned int len = min(slen, ((unsigned int)(PAGE_SIZE)) - offset); - char *p = crypto_kmap(pg, 0) + offset; - - /* checking the data can fill the block */ - if ((ctx->len + len) <= bs) { - memcpy(ctx->odds + ctx->len, p, len); - ctx->len += len; - slen -= len; - - /* checking the rest of the page */ - if (len + offset >= PAGE_SIZE) { - offset = 0; - pg++; - } else - offset += len; - - crypto_kunmap(p, 0); - crypto_yield(pdesc->flags); - continue; - } - - /* filling odds with new data and encrypting it */ - memcpy(ctx->odds + ctx->len, p, bs - ctx->len); - len -= bs - ctx->len; - p += bs - ctx->len; - - ctx->xor(ctx->prev, ctx->odds, bs); - crypto_cipher_encrypt_one(tfm, ctx->prev, ctx->prev); - - /* clearing the length */ - ctx->len = 0; - - /* encrypting the rest of data */ - while (len > bs) { - ctx->xor(ctx->prev, p, bs); - crypto_cipher_encrypt_one(tfm, ctx->prev, - ctx->prev); - p += bs; - len -= bs; - } - - /* keeping the surplus of blocksize */ - if (len) { - memcpy(ctx->odds, p, len); - ctx->len = len; - } - crypto_kunmap(p, 0); - crypto_yield(pdesc->flags); - slen -= min(slen, ((unsigned int)(PAGE_SIZE)) - offset); - offset = 0; - pg++; - } - - if (!nbytes) - break; - sg = scatterwalk_sg_next(sg); + struct crypto_shash *parent = pdesc->tfm; + unsigned long alignmask = crypto_shash_alignmask(parent); + struct xcbc_tfm_ctx *tctx = crypto_shash_ctx(parent); + struct xcbc_desc_ctx *ctx = shash_desc_ctx(pdesc); + struct crypto_cipher *tfm = tctx->child; + int bs = crypto_shash_blocksize(parent); + u8 *odds = PTR_ALIGN(&ctx->ctx[0], alignmask + 1); + u8 *prev = odds + bs; + + /* checking the data can fill the block */ + if ((ctx->len + len) <= bs) { + memcpy(odds + ctx->len, p, len); + ctx->len += len; + return 0; } - return 0; -} + /* filling odds with new data and encrypting it */ + memcpy(odds + ctx->len, p, bs - ctx->len); + len -= bs - ctx->len; + p += bs - ctx->len; -static int crypto_xcbc_digest_update(struct hash_desc *pdesc, - struct scatterlist *sg, - unsigned int nbytes) -{ - if (WARN_ON_ONCE(in_irq())) - return -EDEADLK; - return crypto_xcbc_digest_update2(pdesc, sg, nbytes); -} + crypto_xor(prev, odds, bs); + crypto_cipher_encrypt_one(tfm, prev, prev); -static int crypto_xcbc_digest_final(struct hash_desc *pdesc, u8 *out) -{ - struct crypto_hash *parent = pdesc->tfm; - struct crypto_xcbc_ctx *ctx = crypto_hash_ctx_aligned(parent); - struct crypto_cipher *tfm = ctx->child; - int bs = crypto_hash_blocksize(parent); - int err = 0; - - if (ctx->len == bs) { - u8 key2[bs]; + /* clearing the length */ + ctx->len = 0; - if ((err = crypto_cipher_setkey(tfm, ctx->key, ctx->keylen)) != 0) - return err; + /* encrypting the rest of data */ + while (len > bs) { + crypto_xor(prev, p, bs); + crypto_cipher_encrypt_one(tfm, prev, prev); + p += bs; + len -= bs; + } - crypto_cipher_encrypt_one(tfm, key2, - (u8 *)(ctx->consts + bs)); + /* keeping the surplus of blocksize */ + if (len) { + memcpy(odds, p, len); + ctx->len = len; + } - ctx->xor(ctx->prev, ctx->odds, bs); - ctx->xor(ctx->prev, key2, bs); - _crypto_xcbc_digest_setkey(parent, ctx); + return 0; +} - crypto_cipher_encrypt_one(tfm, out, ctx->prev); - } else { - u8 key3[bs]; +static int crypto_xcbc_digest_final(struct shash_desc *pdesc, u8 *out) +{ + struct crypto_shash *parent = pdesc->tfm; + unsigned long alignmask = crypto_shash_alignmask(parent); + struct xcbc_tfm_ctx *tctx = crypto_shash_ctx(parent); + struct xcbc_desc_ctx *ctx = shash_desc_ctx(pdesc); + struct crypto_cipher *tfm = tctx->child; + int bs = crypto_shash_blocksize(parent); + u8 *consts = PTR_ALIGN(&tctx->ctx[0], alignmask + 1); + u8 *odds = PTR_ALIGN(&ctx->ctx[0], alignmask + 1); + u8 *prev = odds + bs; + unsigned int offset = 0; + + if (ctx->len != bs) { unsigned int rlen; - u8 *p = ctx->odds + ctx->len; + u8 *p = odds + ctx->len; + *p = 0x80; p++; @@ -231,32 +162,15 @@ static int crypto_xcbc_digest_final(struct hash_desc *pdesc, u8 *out) if (rlen) memset(p, 0, rlen); - if ((err = crypto_cipher_setkey(tfm, ctx->key, ctx->keylen)) != 0) - return err; - - crypto_cipher_encrypt_one(tfm, key3, - (u8 *)(ctx->consts + bs * 2)); - - ctx->xor(ctx->prev, ctx->odds, bs); - ctx->xor(ctx->prev, key3, bs); - - _crypto_xcbc_digest_setkey(parent, ctx); - - crypto_cipher_encrypt_one(tfm, out, ctx->prev); + offset += bs; } - return 0; -} + crypto_xor(prev, odds, bs); + crypto_xor(prev, consts + offset, bs); -static int crypto_xcbc_digest(struct hash_desc *pdesc, - struct scatterlist *sg, unsigned int nbytes, u8 *out) -{ - if (WARN_ON_ONCE(in_irq())) - return -EDEADLK; + crypto_cipher_encrypt_one(tfm, out, prev); - crypto_xcbc_digest_init(pdesc); - crypto_xcbc_digest_update2(pdesc, sg, nbytes); - return crypto_xcbc_digest_final(pdesc, out); + return 0; } static int xcbc_init_tfm(struct crypto_tfm *tfm) @@ -264,95 +178,95 @@ static int xcbc_init_tfm(struct crypto_tfm *tfm) struct crypto_cipher *cipher; struct crypto_instance *inst = (void *)tfm->__crt_alg; struct crypto_spawn *spawn = crypto_instance_ctx(inst); - struct crypto_xcbc_ctx *ctx = crypto_hash_ctx_aligned(__crypto_hash_cast(tfm)); - int bs = crypto_hash_blocksize(__crypto_hash_cast(tfm)); + struct xcbc_tfm_ctx *ctx = crypto_tfm_ctx(tfm); cipher = crypto_spawn_cipher(spawn); if (IS_ERR(cipher)) return PTR_ERR(cipher); - switch(bs) { - case 16: - ctx->xor = xor_128; - break; - default: - return -EINVAL; - } - ctx->child = cipher; - ctx->odds = (u8*)(ctx+1); - ctx->prev = ctx->odds + bs; - ctx->key = ctx->prev + bs; return 0; }; static void xcbc_exit_tfm(struct crypto_tfm *tfm) { - struct crypto_xcbc_ctx *ctx = crypto_hash_ctx_aligned(__crypto_hash_cast(tfm)); + struct xcbc_tfm_ctx *ctx = crypto_tfm_ctx(tfm); crypto_free_cipher(ctx->child); } -static struct crypto_instance *xcbc_alloc(struct rtattr **tb) +static int xcbc_create(struct crypto_template *tmpl, struct rtattr **tb) { - struct crypto_instance *inst; + struct shash_instance *inst; struct crypto_alg *alg; + unsigned long alignmask; int err; - err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_HASH); + err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH); if (err) - return ERR_PTR(err); + return err; alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER, CRYPTO_ALG_TYPE_MASK); if (IS_ERR(alg)) - return ERR_CAST(alg); + return PTR_ERR(alg); switch(alg->cra_blocksize) { case 16: break; default: - inst = ERR_PTR(-EINVAL); goto out_put_alg; } - inst = crypto_alloc_instance("xcbc", alg); + inst = shash_alloc_instance("xcbc", alg); + err = PTR_ERR(inst); if (IS_ERR(inst)) goto out_put_alg; - inst->alg.cra_flags = CRYPTO_ALG_TYPE_HASH; - inst->alg.cra_priority = alg->cra_priority; - inst->alg.cra_blocksize = alg->cra_blocksize; - inst->alg.cra_alignmask = alg->cra_alignmask; - inst->alg.cra_type = &crypto_hash_type; - - inst->alg.cra_hash.digestsize = alg->cra_blocksize; - inst->alg.cra_ctxsize = sizeof(struct crypto_xcbc_ctx) + - ALIGN(inst->alg.cra_blocksize * 3, sizeof(void *)); - inst->alg.cra_init = xcbc_init_tfm; - inst->alg.cra_exit = xcbc_exit_tfm; - - inst->alg.cra_hash.init = crypto_xcbc_digest_init; - inst->alg.cra_hash.update = crypto_xcbc_digest_update; - inst->alg.cra_hash.final = crypto_xcbc_digest_final; - inst->alg.cra_hash.digest = crypto_xcbc_digest; - inst->alg.cra_hash.setkey = crypto_xcbc_digest_setkey; + err = crypto_init_spawn(shash_instance_ctx(inst), alg, + shash_crypto_instance(inst), + CRYPTO_ALG_TYPE_MASK); + if (err) + goto out_free_inst; + + alignmask = alg->cra_alignmask | 3; + inst->alg.base.cra_alignmask = alignmask; + inst->alg.base.cra_priority = alg->cra_priority; + inst->alg.base.cra_blocksize = alg->cra_blocksize; + + inst->alg.digestsize = alg->cra_blocksize; + inst->alg.descsize = ALIGN(sizeof(struct xcbc_desc_ctx), + crypto_tfm_ctx_alignment()) + + (alignmask & + ~(crypto_tfm_ctx_alignment() - 1)) + + alg->cra_blocksize * 2; + + inst->alg.base.cra_ctxsize = ALIGN(sizeof(struct xcbc_tfm_ctx), + alignmask + 1) + + alg->cra_blocksize * 2; + inst->alg.base.cra_init = xcbc_init_tfm; + inst->alg.base.cra_exit = xcbc_exit_tfm; + + inst->alg.init = crypto_xcbc_digest_init; + inst->alg.update = crypto_xcbc_digest_update; + inst->alg.final = crypto_xcbc_digest_final; + inst->alg.setkey = crypto_xcbc_digest_setkey; + + err = shash_register_instance(tmpl, inst); + if (err) { +out_free_inst: + shash_free_instance(shash_crypto_instance(inst)); + } out_put_alg: crypto_mod_put(alg); - return inst; -} - -static void xcbc_free(struct crypto_instance *inst) -{ - crypto_drop_spawn(crypto_instance_ctx(inst)); - kfree(inst); + return err; } static struct crypto_template crypto_xcbc_tmpl = { .name = "xcbc", - .alloc = xcbc_alloc, - .free = xcbc_free, + .create = xcbc_create, + .free = shash_free_instance, .module = THIS_MODULE, }; diff --git a/crypto/xor.c b/crypto/xor.c index fc5b836f3430..b75182d8ab14 100644 --- a/crypto/xor.c +++ b/crypto/xor.c @@ -18,6 +18,7 @@ #define BH_TRACE 0 #include <linux/module.h> +#include <linux/gfp.h> #include <linux/raid/xor.h> #include <linux/jiffies.h> #include <asm/xor.h> diff --git a/crypto/xts.c b/crypto/xts.c index d87b0f3102c3..555ecaab1e54 100644 --- a/crypto/xts.c +++ b/crypto/xts.c @@ -224,7 +224,7 @@ static struct crypto_instance *alloc(struct rtattr **tb) alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER, CRYPTO_ALG_TYPE_MASK); if (IS_ERR(alg)) - return ERR_PTR(PTR_ERR(alg)); + return ERR_CAST(alg); inst = crypto_alloc_instance("xts", alg); if (IS_ERR(inst)) |