From fdb2726f4e61c5e3abc052f547d5a5f6c0dc5504 Mon Sep 17 00:00:00 2001 From: Michael Davidson Date: Wed, 15 Mar 2017 15:36:00 -0700 Subject: crypto, x86: aesni - fix token pasting for clang aes_ctrby8_avx-x86_64.S uses the C preprocessor for token pasting of character sequences that are not valid preprocessor tokens. While this is allowed when preprocessing assembler files it exposes an incompatibilty between the clang and gcc preprocessors where clang does not strip leading white space from macro parameters, leading to the CONCAT(%xmm, i) macro expansion on line 96 resulting in a token with a space character embedded in it. While this could be resolved by deleting the offending space character, the assembler is perfectly capable of doing the token pasting correctly for itself so we can just get rid of the preprocessor macros. Signed-off-by: Michael Davidson Signed-off-by: Herbert Xu --- arch/x86/crypto/aes_ctrby8_avx-x86_64.S | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'arch/x86') diff --git a/arch/x86/crypto/aes_ctrby8_avx-x86_64.S b/arch/x86/crypto/aes_ctrby8_avx-x86_64.S index a916c4a61165..5f6a5af9c489 100644 --- a/arch/x86/crypto/aes_ctrby8_avx-x86_64.S +++ b/arch/x86/crypto/aes_ctrby8_avx-x86_64.S @@ -65,7 +65,6 @@ #include #include -#define CONCAT(a,b) a##b #define VMOVDQ vmovdqu #define xdata0 %xmm0 @@ -92,8 +91,6 @@ #define num_bytes %r8 #define tmp %r10 -#define DDQ(i) CONCAT(ddq_add_,i) -#define XMM(i) CONCAT(%xmm, i) #define DDQ_DATA 0 #define XDATA 1 #define KEY_128 1 @@ -131,12 +128,12 @@ ddq_add_8: /* generate a unique variable for ddq_add_x */ .macro setddq n - var_ddq_add = DDQ(\n) + var_ddq_add = ddq_add_\n .endm /* generate a unique variable for xmm register */ .macro setxdata n - var_xdata = XMM(\n) + var_xdata = %xmm\n .endm /* club the numeric 'id' to the symbol 'name' */ -- cgit v1.2.3 From e55318c84f199d6056a0bcd98bc4612d01ccfe80 Mon Sep 17 00:00:00 2001 From: Ondrej Mosnáček Date: Sun, 2 Apr 2017 21:19:14 +0200 Subject: crypto: gf128mul - switch gf128mul_x_ble to le128 Currently, gf128mul_x_ble works with pointers to be128, even though it actually interprets the words as little-endian. Consequently, it uses cpu_to_le64/le64_to_cpu on fields of type __be64, which is incorrect. This patch fixes that by changing the function to accept pointers to le128 and updating all users accordingly. Signed-off-by: Ondrej Mosnacek Reviewd-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/camellia_glue.c | 4 ++-- arch/x86/crypto/serpent_sse2_glue.c | 4 ++-- arch/x86/crypto/twofish_glue_3way.c | 4 ++-- crypto/xts.c | 38 ++++++++++++++++++------------------- include/crypto/gf128mul.h | 8 ++++---- include/crypto/xts.h | 2 +- 6 files changed, 30 insertions(+), 30 deletions(-) (limited to 'arch/x86') diff --git a/arch/x86/crypto/camellia_glue.c b/arch/x86/crypto/camellia_glue.c index aa76cad9d262..af4840ab2a3d 100644 --- a/arch/x86/crypto/camellia_glue.c +++ b/arch/x86/crypto/camellia_glue.c @@ -1522,7 +1522,7 @@ static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - be128 buf[2 * 4]; + le128 buf[2 * 4]; struct xts_crypt_req req = { .tbuf = buf, .tbuflen = sizeof(buf), @@ -1540,7 +1540,7 @@ static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct camellia_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - be128 buf[2 * 4]; + le128 buf[2 * 4]; struct xts_crypt_req req = { .tbuf = buf, .tbuflen = sizeof(buf), diff --git a/arch/x86/crypto/serpent_sse2_glue.c b/arch/x86/crypto/serpent_sse2_glue.c index 644f97ab8cac..ac0e831943f5 100644 --- a/arch/x86/crypto/serpent_sse2_glue.c +++ b/arch/x86/crypto/serpent_sse2_glue.c @@ -328,7 +328,7 @@ static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - be128 buf[SERPENT_PARALLEL_BLOCKS]; + le128 buf[SERPENT_PARALLEL_BLOCKS]; struct crypt_priv crypt_ctx = { .ctx = &ctx->crypt_ctx, .fpu_enabled = false, @@ -355,7 +355,7 @@ static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - be128 buf[SERPENT_PARALLEL_BLOCKS]; + le128 buf[SERPENT_PARALLEL_BLOCKS]; struct crypt_priv crypt_ctx = { .ctx = &ctx->crypt_ctx, .fpu_enabled = false, diff --git a/arch/x86/crypto/twofish_glue_3way.c b/arch/x86/crypto/twofish_glue_3way.c index 2ebb5e9789f3..243e90a4b5d9 100644 --- a/arch/x86/crypto/twofish_glue_3way.c +++ b/arch/x86/crypto/twofish_glue_3way.c @@ -296,7 +296,7 @@ static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - be128 buf[3]; + le128 buf[3]; struct xts_crypt_req req = { .tbuf = buf, .tbuflen = sizeof(buf), @@ -314,7 +314,7 @@ static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - be128 buf[3]; + le128 buf[3]; struct xts_crypt_req req = { .tbuf = buf, .tbuflen = sizeof(buf), diff --git a/crypto/xts.c b/crypto/xts.c index c976bfac29da..e197e64eb45c 100644 --- a/crypto/xts.c +++ b/crypto/xts.c @@ -39,11 +39,11 @@ struct xts_instance_ctx { }; struct rctx { - be128 buf[XTS_BUFFER_SIZE / sizeof(be128)]; + le128 buf[XTS_BUFFER_SIZE / sizeof(le128)]; - be128 t; + le128 t; - be128 *ext; + le128 *ext; struct scatterlist srcbuf[2]; struct scatterlist dstbuf[2]; @@ -99,7 +99,7 @@ static int setkey(struct crypto_skcipher *parent, const u8 *key, static int post_crypt(struct skcipher_request *req) { struct rctx *rctx = skcipher_request_ctx(req); - be128 *buf = rctx->ext ?: rctx->buf; + le128 *buf = rctx->ext ?: rctx->buf; struct skcipher_request *subreq; const int bs = XTS_BLOCK_SIZE; struct skcipher_walk w; @@ -112,12 +112,12 @@ static int post_crypt(struct skcipher_request *req) while (w.nbytes) { unsigned int avail = w.nbytes; - be128 *wdst; + le128 *wdst; wdst = w.dst.virt.addr; do { - be128_xor(wdst, buf++, wdst); + le128_xor(wdst, buf++, wdst); wdst++; } while ((avail -= bs) >= bs); @@ -150,7 +150,7 @@ out: static int pre_crypt(struct skcipher_request *req) { struct rctx *rctx = skcipher_request_ctx(req); - be128 *buf = rctx->ext ?: rctx->buf; + le128 *buf = rctx->ext ?: rctx->buf; struct skcipher_request *subreq; const int bs = XTS_BLOCK_SIZE; struct skcipher_walk w; @@ -174,15 +174,15 @@ static int pre_crypt(struct skcipher_request *req) while (w.nbytes) { unsigned int avail = w.nbytes; - be128 *wsrc; - be128 *wdst; + le128 *wsrc; + le128 *wdst; wsrc = w.src.virt.addr; wdst = w.dst.virt.addr; do { *buf++ = rctx->t; - be128_xor(wdst++, &rctx->t, wsrc++); + le128_xor(wdst++, &rctx->t, wsrc++); gf128mul_x_ble(&rctx->t, &rctx->t); } while ((avail -= bs) >= bs); @@ -353,8 +353,8 @@ int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst, const unsigned int max_blks = req->tbuflen / bsize; struct blkcipher_walk walk; unsigned int nblocks; - be128 *src, *dst, *t; - be128 *t_buf = req->tbuf; + le128 *src, *dst, *t; + le128 *t_buf = req->tbuf; int err, i; BUG_ON(max_blks < 1); @@ -367,8 +367,8 @@ int xts_crypt(struct blkcipher_desc *desc, struct scatterlist *sdst, return err; nblocks = min(nbytes / bsize, max_blks); - src = (be128 *)walk.src.virt.addr; - dst = (be128 *)walk.dst.virt.addr; + src = (le128 *)walk.src.virt.addr; + dst = (le128 *)walk.dst.virt.addr; /* calculate first value of T */ req->tweak_fn(req->tweak_ctx, (u8 *)&t_buf[0], walk.iv); @@ -384,7 +384,7 @@ first: t = &t_buf[i]; /* PP <- T xor P */ - be128_xor(dst + i, t, src + i); + le128_xor(dst + i, t, src + i); } /* CC <- E(Key2,PP) */ @@ -393,7 +393,7 @@ first: /* C <- T xor CC */ for (i = 0; i < nblocks; i++) - be128_xor(dst + i, dst + i, &t_buf[i]); + le128_xor(dst + i, dst + i, &t_buf[i]); src += nblocks; dst += nblocks; @@ -401,7 +401,7 @@ first: nblocks = min(nbytes / bsize, max_blks); } while (nblocks > 0); - *(be128 *)walk.iv = *t; + *(le128 *)walk.iv = *t; err = blkcipher_walk_done(desc, &walk, nbytes); nbytes = walk.nbytes; @@ -409,8 +409,8 @@ first: break; nblocks = min(nbytes / bsize, max_blks); - src = (be128 *)walk.src.virt.addr; - dst = (be128 *)walk.dst.virt.addr; + src = (le128 *)walk.src.virt.addr; + dst = (le128 *)walk.dst.virt.addr; } return err; diff --git a/include/crypto/gf128mul.h b/include/crypto/gf128mul.h index 35ced9db70ea..0977fb18ff68 100644 --- a/include/crypto/gf128mul.h +++ b/include/crypto/gf128mul.h @@ -205,16 +205,16 @@ static inline void gf128mul_x_bbe(be128 *r, const be128 *x) } /* needed by XTS */ -static inline void gf128mul_x_ble(be128 *r, const be128 *x) +static inline void gf128mul_x_ble(le128 *r, const le128 *x) { u64 a = le64_to_cpu(x->a); u64 b = le64_to_cpu(x->b); /* equivalent to gf128mul_table_be[b >> 63] (see crypto/gf128mul.c): */ - u64 _tt = gf128mul_mask_from_bit(b, 63) & 0x87; + u64 _tt = gf128mul_mask_from_bit(a, 63) & 0x87; - r->a = cpu_to_le64((a << 1) ^ _tt); - r->b = cpu_to_le64((b << 1) | (a >> 63)); + r->a = cpu_to_le64((a << 1) | (b >> 63)); + r->b = cpu_to_le64((b << 1) ^ _tt); } /* 4k table optimization */ diff --git a/include/crypto/xts.h b/include/crypto/xts.h index 77b630672b2c..c0bde308b28a 100644 --- a/include/crypto/xts.h +++ b/include/crypto/xts.h @@ -11,7 +11,7 @@ struct blkcipher_desc; #define XTS_BLOCK_SIZE 16 struct xts_crypt_req { - be128 *tbuf; + le128 *tbuf; unsigned int tbuflen; void *tweak_ctx; -- cgit v1.2.3 From 692016bdf7cf33abaaa7f2b080b47f504c98810c Mon Sep 17 00:00:00 2001 From: Ondrej Mosnáček Date: Sun, 2 Apr 2017 21:19:15 +0200 Subject: crypto: glue_helper - remove the le128_gf128mul_x_ble function The le128_gf128mul_x_ble function in glue_helper.h is now obsolete and can be replaced with the gf128mul_x_ble function from gf128mul.h. Signed-off-by: Ondrej Mosnacek Reviewd-by: Eric Biggers Signed-off-by: Herbert Xu --- arch/x86/crypto/glue_helper.c | 3 ++- arch/x86/include/asm/crypto/glue_helper.h | 10 ---------- 2 files changed, 2 insertions(+), 11 deletions(-) (limited to 'arch/x86') diff --git a/arch/x86/crypto/glue_helper.c b/arch/x86/crypto/glue_helper.c index 260a060d7275..24ac9fad832d 100644 --- a/arch/x86/crypto/glue_helper.c +++ b/arch/x86/crypto/glue_helper.c @@ -27,6 +27,7 @@ #include #include +#include #include #include #include @@ -457,7 +458,7 @@ void glue_xts_crypt_128bit_one(void *ctx, u128 *dst, const u128 *src, le128 *iv, le128 ivblk = *iv; /* generate next IV */ - le128_gf128mul_x_ble(iv, &ivblk); + gf128mul_x_ble(iv, &ivblk); /* CC <- T xor C */ u128_xor(dst, src, (u128 *)&ivblk); diff --git a/arch/x86/include/asm/crypto/glue_helper.h b/arch/x86/include/asm/crypto/glue_helper.h index 29e53ea7d764..ed8b66de541f 100644 --- a/arch/x86/include/asm/crypto/glue_helper.h +++ b/arch/x86/include/asm/crypto/glue_helper.h @@ -125,16 +125,6 @@ static inline void le128_inc(le128 *i) i->b = cpu_to_le64(b); } -static inline void le128_gf128mul_x_ble(le128 *dst, const le128 *src) -{ - u64 a = le64_to_cpu(src->a); - u64 b = le64_to_cpu(src->b); - u64 _tt = ((s64)a >> 63) & 0x87; - - dst->a = cpu_to_le64((a << 1) ^ (b >> 63)); - dst->b = cpu_to_le64((b << 1) ^ _tt); -} - extern int glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx, struct blkcipher_desc *desc, struct scatterlist *dst, -- cgit v1.2.3