diff options
168 files changed, 4527 insertions, 3798 deletions
diff --git a/Documentation/crypto/api-samples.rst b/Documentation/crypto/api-samples.rst index f14afaaf2f32..e923f17bc2bd 100644 --- a/Documentation/crypto/api-samples.rst +++ b/Documentation/crypto/api-samples.rst @@ -4,111 +4,89 @@ Code Examples Code Example For Symmetric Key Cipher Operation ----------------------------------------------- -:: - - - /* tie all data structures together */ - struct skcipher_def { - struct scatterlist sg; - struct crypto_skcipher *tfm; - struct skcipher_request *req; - struct crypto_wait wait; - }; - - /* Perform cipher operation */ - static unsigned int test_skcipher_encdec(struct skcipher_def *sk, - int enc) - { - int rc; - - if (enc) - rc = crypto_wait_req(crypto_skcipher_encrypt(sk->req), &sk->wait); - else - rc = crypto_wait_req(crypto_skcipher_decrypt(sk->req), &sk->wait); - - if (rc) - pr_info("skcipher encrypt returned with result %d\n", rc); +This code encrypts some data with AES-256-XTS. For sake of example, +all inputs are random bytes, the encryption is done in-place, and it's +assumed the code is running in a context where it can sleep. - return rc; - } +:: - /* Initialize and trigger cipher operation */ static int test_skcipher(void) { - struct skcipher_def sk; - struct crypto_skcipher *skcipher = NULL; - struct skcipher_request *req = NULL; - char *scratchpad = NULL; - char *ivdata = NULL; - unsigned char key[32]; - int ret = -EFAULT; - - skcipher = crypto_alloc_skcipher("cbc-aes-aesni", 0, 0); - if (IS_ERR(skcipher)) { - pr_info("could not allocate skcipher handle\n"); - return PTR_ERR(skcipher); - } - - req = skcipher_request_alloc(skcipher, GFP_KERNEL); - if (!req) { - pr_info("could not allocate skcipher request\n"); - ret = -ENOMEM; - goto out; - } - - skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, - crypto_req_done, - &sk.wait); - - /* AES 256 with random key */ - get_random_bytes(&key, 32); - if (crypto_skcipher_setkey(skcipher, key, 32)) { - pr_info("key could not be set\n"); - ret = -EAGAIN; - goto out; - } - - /* IV will be random */ - ivdata = kmalloc(16, GFP_KERNEL); - if (!ivdata) { - pr_info("could not allocate ivdata\n"); - goto out; - } - get_random_bytes(ivdata, 16); - - /* Input data will be random */ - scratchpad = kmalloc(16, GFP_KERNEL); - if (!scratchpad) { - pr_info("could not allocate scratchpad\n"); - goto out; - } - get_random_bytes(scratchpad, 16); - - sk.tfm = skcipher; - sk.req = req; - - /* We encrypt one block */ - sg_init_one(&sk.sg, scratchpad, 16); - skcipher_request_set_crypt(req, &sk.sg, &sk.sg, 16, ivdata); - crypto_init_wait(&sk.wait); - - /* encrypt data */ - ret = test_skcipher_encdec(&sk, 1); - if (ret) - goto out; - - pr_info("Encryption triggered successfully\n"); - + struct crypto_skcipher *tfm = NULL; + struct skcipher_request *req = NULL; + u8 *data = NULL; + const size_t datasize = 512; /* data size in bytes */ + struct scatterlist sg; + DECLARE_CRYPTO_WAIT(wait); + u8 iv[16]; /* AES-256-XTS takes a 16-byte IV */ + u8 key[64]; /* AES-256-XTS takes a 64-byte key */ + int err; + + /* + * Allocate a tfm (a transformation object) and set the key. + * + * In real-world use, a tfm and key are typically used for many + * encryption/decryption operations. But in this example, we'll just do a + * single encryption operation with it (which is not very efficient). + */ + + tfm = crypto_alloc_skcipher("xts(aes)", 0, 0); + if (IS_ERR(tfm)) { + pr_err("Error allocating xts(aes) handle: %ld\n", PTR_ERR(tfm)); + return PTR_ERR(tfm); + } + + get_random_bytes(key, sizeof(key)); + err = crypto_skcipher_setkey(tfm, key, sizeof(key)); + if (err) { + pr_err("Error setting key: %d\n", err); + goto out; + } + + /* Allocate a request object */ + req = skcipher_request_alloc(tfm, GFP_KERNEL); + if (!req) { + err = -ENOMEM; + goto out; + } + + /* Prepare the input data */ + data = kmalloc(datasize, GFP_KERNEL); + if (!data) { + err = -ENOMEM; + goto out; + } + get_random_bytes(data, datasize); + + /* Initialize the IV */ + get_random_bytes(iv, sizeof(iv)); + + /* + * Encrypt the data in-place. + * + * For simplicity, in this example we wait for the request to complete + * before proceeding, even if the underlying implementation is asynchronous. + * + * To decrypt instead of encrypt, just change crypto_skcipher_encrypt() to + * crypto_skcipher_decrypt(). + */ + sg_init_one(&sg, data, datasize); + skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG | + CRYPTO_TFM_REQ_MAY_SLEEP, + crypto_req_done, &wait); + skcipher_request_set_crypt(req, &sg, &sg, datasize, iv); + err = crypto_wait_req(crypto_skcipher_encrypt(req), &wait); + if (err) { + pr_err("Error encrypting data: %d\n", err); + goto out; + } + + pr_debug("Encryption was successful\n"); out: - if (skcipher) - crypto_free_skcipher(skcipher); - if (req) + crypto_free_skcipher(tfm); skcipher_request_free(req); - if (ivdata) - kfree(ivdata); - if (scratchpad) - kfree(scratchpad); - return ret; + kfree(data); + return err; } diff --git a/Documentation/crypto/api-skcipher.rst b/Documentation/crypto/api-skcipher.rst index 4eec4a93f7e3..20ba08dddf2e 100644 --- a/Documentation/crypto/api-skcipher.rst +++ b/Documentation/crypto/api-skcipher.rst @@ -5,7 +5,7 @@ Block Cipher Algorithm Definitions :doc: Block Cipher Algorithm Definitions .. kernel-doc:: include/linux/crypto.h - :functions: crypto_alg ablkcipher_alg blkcipher_alg cipher_alg + :functions: crypto_alg ablkcipher_alg blkcipher_alg cipher_alg compress_alg Symmetric Key Cipher API ------------------------ diff --git a/Documentation/crypto/architecture.rst b/Documentation/crypto/architecture.rst index ee8ff0762d7f..3eae1ae7f798 100644 --- a/Documentation/crypto/architecture.rst +++ b/Documentation/crypto/architecture.rst @@ -208,9 +208,7 @@ the aforementioned cipher types: - CRYPTO_ALG_TYPE_KPP Key-agreement Protocol Primitive (KPP) such as an ECDH or DH implementation -- CRYPTO_ALG_TYPE_DIGEST Raw message digest - -- CRYPTO_ALG_TYPE_HASH Alias for CRYPTO_ALG_TYPE_DIGEST +- CRYPTO_ALG_TYPE_HASH Raw message digest - CRYPTO_ALG_TYPE_SHASH Synchronous multi-block hash diff --git a/Documentation/crypto/crypto_engine.rst b/Documentation/crypto/crypto_engine.rst index 1d56221dfe35..236c674d6897 100644 --- a/Documentation/crypto/crypto_engine.rst +++ b/Documentation/crypto/crypto_engine.rst @@ -1,50 +1,85 @@ -============= -CRYPTO ENGINE +.. SPDX-License-Identifier: GPL-2.0 +Crypto Engine ============= Overview -------- -The crypto engine API (CE), is a crypto queue manager. +The crypto engine (CE) API is a crypto queue manager. Requirement ----------- -You have to put at start of your tfm_ctx the struct crypto_engine_ctx:: +You must put, at the start of your transform context your_tfm_ctx, the structure +crypto_engine: + +:: - struct your_tfm_ctx { - struct crypto_engine_ctx enginectx; - ... - }; + struct your_tfm_ctx { + struct crypto_engine engine; + ... + }; -Why: Since CE manage only crypto_async_request, it cannot know the underlying -request_type and so have access only on the TFM. -So using container_of for accessing __ctx is impossible. -Furthermore, the crypto engine cannot know the "struct your_tfm_ctx", -so it must assume that crypto_engine_ctx is at start of it. +The crypto engine only manages asynchronous requests in the form of +crypto_async_request. It cannot know the underlying request type and thus only +has access to the transform structure. It is not possible to access the context +using container_of. In addition, the engine knows nothing about your +structure "``struct your_tfm_ctx``". The engine assumes (requires) the placement +of the known member ``struct crypto_engine`` at the beginning. Order of operations ------------------- -You have to obtain a struct crypto_engine via crypto_engine_alloc_init(). -And start it via crypto_engine_start(). - -Before transferring any request, you have to fill the enginectx. -- prepare_request: (taking a function pointer) If you need to do some processing before doing the request -- unprepare_request: (taking a function pointer) Undoing what's done in prepare_request -- do_one_request: (taking a function pointer) Do encryption for current request - -Note: that those three functions get the crypto_async_request associated with the received request. -So your need to get the original request via container_of(areq, struct yourrequesttype_request, base); - -When your driver receive a crypto_request, you have to transfer it to -the cryptoengine via one of: -- crypto_transfer_ablkcipher_request_to_engine() -- crypto_transfer_aead_request_to_engine() -- crypto_transfer_akcipher_request_to_engine() -- crypto_transfer_hash_request_to_engine() -- crypto_transfer_skcipher_request_to_engine() - -At the end of the request process, a call to one of the following function is needed: -- crypto_finalize_ablkcipher_request -- crypto_finalize_aead_request -- crypto_finalize_akcipher_request -- crypto_finalize_hash_request -- crypto_finalize_skcipher_request +You are required to obtain a struct crypto_engine via ``crypto_engine_alloc_init()``. +Start it via ``crypto_engine_start()``. When finished with your work, shut down the +engine using ``crypto_engine_stop()`` and destroy the engine with +``crypto_engine_exit()``. + +Before transferring any request, you have to fill the context enginectx by +providing functions for the following: + +* ``prepare_crypt_hardware``: Called once before any prepare functions are + called. + +* ``unprepare_crypt_hardware``: Called once after all unprepare functions have + been called. + +* ``prepare_cipher_request``/``prepare_hash_request``: Called before each + corresponding request is performed. If some processing or other preparatory + work is required, do it here. + +* ``unprepare_cipher_request``/``unprepare_hash_request``: Called after each + request is handled. Clean up / undo what was done in the prepare function. + +* ``cipher_one_request``/``hash_one_request``: Handle the current request by + performing the operation. + +Note that these functions access the crypto_async_request structure +associated with the received request. You are able to retrieve the original +request by using: + +:: + + container_of(areq, struct yourrequesttype_request, base); + +When your driver receives a crypto_request, you must to transfer it to +the crypto engine via one of: + +* crypto_transfer_ablkcipher_request_to_engine() + +* crypto_transfer_aead_request_to_engine() + +* crypto_transfer_akcipher_request_to_engine() + +* crypto_transfer_hash_request_to_engine() + +* crypto_transfer_skcipher_request_to_engine() + +At the end of the request process, a call to one of the following functions is needed: + +* crypto_finalize_ablkcipher_request() + +* crypto_finalize_aead_request() + +* crypto_finalize_akcipher_request() + +* crypto_finalize_hash_request() + +* crypto_finalize_skcipher_request() diff --git a/Documentation/devicetree/bindings/crypto/atmel-crypto.txt b/Documentation/devicetree/bindings/crypto/atmel-crypto.txt index 6b458bb2440d..f2aab3dc2b52 100644 --- a/Documentation/devicetree/bindings/crypto/atmel-crypto.txt +++ b/Documentation/devicetree/bindings/crypto/atmel-crypto.txt @@ -66,16 +66,3 @@ sha@f8034000 { dmas = <&dma1 2 17>; dma-names = "tx"; }; - -* Eliptic Curve Cryptography (I2C) - -Required properties: -- compatible : must be "atmel,atecc508a". -- reg: I2C bus address of the device. -- clock-frequency: must be present in the i2c controller node. - -Example: -atecc508a@c0 { - compatible = "atmel,atecc508a"; - reg = <0xC0>; -}; diff --git a/Documentation/devicetree/bindings/rng/brcm,iproc-rng200.txt b/Documentation/devicetree/bindings/rng/brcm,iproc-rng200.txt index 0014da9145af..c223e54452da 100644 --- a/Documentation/devicetree/bindings/rng/brcm,iproc-rng200.txt +++ b/Documentation/devicetree/bindings/rng/brcm,iproc-rng200.txt @@ -2,6 +2,7 @@ HWRNG support for the iproc-rng200 driver Required properties: - compatible : Must be one of: + "brcm,bcm7211-rng200" "brcm,bcm7278-rng200" "brcm,iproc-rng200" - reg : base address and size of control register block diff --git a/Documentation/devicetree/bindings/trivial-devices.yaml b/Documentation/devicetree/bindings/trivial-devices.yaml index 747fd3f689dc..2e742d399e87 100644 --- a/Documentation/devicetree/bindings/trivial-devices.yaml +++ b/Documentation/devicetree/bindings/trivial-devices.yaml @@ -52,6 +52,10 @@ properties: - at,24c08 # i2c trusted platform module (TPM) - atmel,at97sc3204t + # i2c h/w symmetric crypto module + - atmel,atsha204a + # i2c h/w elliptic curve crypto module + - atmel,atecc508a # CM32181: Ambient Light Sensor - capella,cm32181 # CM3232: Ambient Light Sensor diff --git a/MAINTAINERS b/MAINTAINERS index a2b97d38a704..4a9e8e5b2432 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4257,6 +4257,7 @@ F: crypto/ F: drivers/crypto/ F: include/crypto/ F: include/linux/crypto* +F: lib/crypto/ CRYPTOGRAPHIC RANDOM NUMBER GENERATOR M: Neil Horman <nhorman@tuxdriver.com> diff --git a/arch/arm/boot/dts/imx7ulp.dtsi b/arch/arm/boot/dts/imx7ulp.dtsi index d6b711011cba..e20483714be5 100644 --- a/arch/arm/boot/dts/imx7ulp.dtsi +++ b/arch/arm/boot/dts/imx7ulp.dtsi @@ -100,6 +100,29 @@ reg = <0x40000000 0x800000>; ranges; + crypto: crypto@40240000 { + compatible = "fsl,sec-v4.0"; + #address-cells = <1>; + #size-cells = <1>; + reg = <0x40240000 0x10000>; + ranges = <0 0x40240000 0x10000>; + clocks = <&pcc2 IMX7ULP_CLK_CAAM>, + <&scg1 IMX7ULP_CLK_NIC1_BUS_DIV>; + clock-names = "aclk", "ipg"; + + sec_jr0: jr0@1000 { + compatible = "fsl,sec-v4.0-job-ring"; + reg = <0x1000 0x1000>; + interrupts = <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>; + }; + + sec_jr1: jr1@2000 { + compatible = "fsl,sec-v4.0-job-ring"; + reg = <0x2000 0x1000>; + interrupts = <GIC_SPI 54 IRQ_TYPE_LEVEL_HIGH>; + }; + }; + lpuart4: serial@402d0000 { compatible = "fsl,imx7ulp-lpuart"; reg = <0x402d0000 0x1000>; diff --git a/arch/arm/crypto/chacha-neon-glue.c b/arch/arm/crypto/chacha-neon-glue.c index 48a89537b828..a8e9b534c8da 100644 --- a/arch/arm/crypto/chacha-neon-glue.c +++ b/arch/arm/crypto/chacha-neon-glue.c @@ -63,7 +63,7 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, } static int chacha_neon_stream_xor(struct skcipher_request *req, - struct chacha_ctx *ctx, u8 *iv) + const struct chacha_ctx *ctx, const u8 *iv) { struct skcipher_walk walk; u32 state[16]; diff --git a/arch/arm/crypto/sha512-glue.c b/arch/arm/crypto/sha512-glue.c index 232eeab1ec37..8775aa42bbbe 100644 --- a/arch/arm/crypto/sha512-glue.c +++ b/arch/arm/crypto/sha512-glue.c @@ -34,7 +34,7 @@ int sha512_arm_update(struct shash_desc *desc, const u8 *data, (sha512_block_fn *)sha512_block_data_order); } -int sha512_arm_final(struct shash_desc *desc, u8 *out) +static int sha512_arm_final(struct shash_desc *desc, u8 *out) { sha512_base_do_finalize(desc, (sha512_block_fn *)sha512_block_data_order); diff --git a/arch/arm64/crypto/aes-ce.S b/arch/arm64/crypto/aes-ce.S index 3ebfaec97e27..00bd2885feaa 100644 --- a/arch/arm64/crypto/aes-ce.S +++ b/arch/arm64/crypto/aes-ce.S @@ -15,6 +15,8 @@ .arch armv8-a+crypto xtsmask .req v16 + cbciv .req v16 + vctr .req v16 .macro xts_reload_mask, tmp .endm @@ -49,7 +51,7 @@ load_round_keys \rounds, \temp .endm - .macro do_enc_Nx, de, mc, k, i0, i1, i2, i3 + .macro do_enc_Nx, de, mc, k, i0, i1, i2, i3, i4 aes\de \i0\().16b, \k\().16b aes\mc \i0\().16b, \i0\().16b .ifnb \i1 @@ -60,27 +62,34 @@ aes\mc \i2\().16b, \i2\().16b aes\de \i3\().16b, \k\().16b aes\mc \i3\().16b, \i3\().16b + .ifnb \i4 + aes\de \i4\().16b, \k\().16b + aes\mc \i4\().16b, \i4\().16b + .endif .endif .endif .endm - /* up to 4 interleaved encryption rounds with the same round key */ - .macro round_Nx, enc, k, i0, i1, i2, i3 + /* up to 5 interleaved encryption rounds with the same round key */ + .macro round_Nx, enc, k, i0, i1, i2, i3, i4 .ifc \enc, e - do_enc_Nx e, mc, \k, \i0, \i1, \i2, \i3 + do_enc_Nx e, mc, \k, \i0, \i1, \i2, \i3, \i4 .else - do_enc_Nx d, imc, \k, \i0, \i1, \i2, \i3 + do_enc_Nx d, imc, \k, \i0, \i1, \i2, \i3, \i4 .endif .endm - /* up to 4 interleaved final rounds */ - .macro fin_round_Nx, de, k, k2, i0, i1, i2, i3 + /* up to 5 interleaved final rounds */ + .macro fin_round_Nx, de, k, k2, i0, i1, i2, i3, i4 aes\de \i0\().16b, \k\().16b .ifnb \i1 aes\de \i1\().16b, \k\().16b .ifnb \i3 aes\de \i2\().16b, \k\().16b aes\de \i3\().16b, \k\().16b + .ifnb \i4 + aes\de \i4\().16b, \k\().16b + .endif .endif .endif eor \i0\().16b, \i0\().16b, \k2\().16b @@ -89,47 +98,52 @@ .ifnb \i3 eor \i2\().16b, \i2\().16b, \k2\().16b eor \i3\().16b, \i3\().16b, \k2\().16b + .ifnb \i4 + eor \i4\().16b, \i4\().16b, \k2\().16b + .endif .endif .endif .endm - /* up to 4 interleaved blocks */ - .macro do_block_Nx, enc, rounds, i0, i1, i2, i3 + /* up to 5 interleaved blocks */ + .macro do_block_Nx, enc, rounds, i0, i1, i2, i3, i4 cmp \rounds, #12 blo 2222f /* 128 bits */ beq 1111f /* 192 bits */ - round_Nx \enc, v17, \i0, \i1, \i2, \i3 - round_Nx \enc, v18, \i0, \i1, \i2, \i3 -1111: round_Nx \enc, v19, \i0, \i1, \i2, \i3 - round_Nx \enc, v20, \i0, \i1, \i2, \i3 + round_Nx \enc, v17, \i0, \i1, \i2, \i3, \i4 + round_Nx \enc, v18, \i0, \i1, \i2, \i3, \i4 +1111: round_Nx \enc, v19, \i0, \i1, \i2, \i3, \i4 + round_Nx \enc, v20, \i0, \i1, \i2, \i3, \i4 2222: .irp key, v21, v22, v23, v24, v25, v26, v27, v28, v29 - round_Nx \enc, \key, \i0, \i1, \i2, \i3 + round_Nx \enc, \key, \i0, \i1, \i2, \i3, \i4 .endr - fin_round_Nx \enc, v30, v31, \i0, \i1, \i2, \i3 + fin_round_Nx \enc, v30, v31, \i0, \i1, \i2, \i3, \i4 .endm .macro encrypt_block, in, rounds, t0, t1, t2 do_block_Nx e, \rounds, \in .endm - .macro encrypt_block2x, i0, i1, rounds, t0, t1, t2 - do_block_Nx e, \rounds, \i0, \i1 - .endm - .macro encrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2 do_block_Nx e, \rounds, \i0, \i1, \i2, \i3 .endm - .macro decrypt_block, in, rounds, t0, t1, t2 - do_block_Nx d, \rounds, \in + .macro encrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2 + do_block_Nx e, \rounds, \i0, \i1, \i2, \i3, \i4 .endm - .macro decrypt_block2x, i0, i1, rounds, t0, t1, t2 - do_block_Nx d, \rounds, \i0, \i1 + .macro decrypt_block, in, rounds, t0, t1, t2 + do_block_Nx d, \rounds, \in .endm .macro decrypt_block4x, i0, i1, i2, i3, rounds, t0, t1, t2 do_block_Nx d, \rounds, \i0, \i1, \i2, \i3 .endm + .macro decrypt_block5x, i0, i1, i2, i3, i4, rounds, t0, t1, t2 + do_block_Nx d, \rounds, \i0, \i1, \i2, \i3, \i4 + .endm + +#define MAX_STRIDE 5 + #include "aes-modes.S" diff --git a/arch/arm64/crypto/aes-modes.S b/arch/arm64/crypto/aes-modes.S index 2883def14be5..324039b72094 100644 --- a/arch/arm64/crypto/aes-modes.S +++ b/arch/arm64/crypto/aes-modes.S @@ -10,6 +10,18 @@ .text .align 4 +#ifndef MAX_STRIDE +#define MAX_STRIDE 4 +#endif + +#if MAX_STRIDE == 4 +#define ST4(x...) x +#define ST5(x...) +#else +#define ST4(x...) +#define ST5(x...) x +#endif + aes_encrypt_block4x: encrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7 ret @@ -20,6 +32,18 @@ aes_decrypt_block4x: ret ENDPROC(aes_decrypt_block4x) +#if MAX_STRIDE == 5 +aes_encrypt_block5x: + encrypt_block5x v0, v1, v2, v3, v4, w3, x2, x8, w7 + ret +ENDPROC(aes_encrypt_block5x) + +aes_decrypt_block5x: + decrypt_block5x v0, v1, v2, v3, v4, w3, x2, x8, w7 + ret +ENDPROC(aes_decrypt_block5x) +#endif + /* * aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, * int blocks) @@ -34,14 +58,17 @@ AES_ENTRY(aes_ecb_encrypt) enc_prepare w3, x2, x5 .LecbencloopNx: - subs w4, w4, #4 + subs w4, w4, #MAX_STRIDE bmi .Lecbenc1x ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */ - bl aes_encrypt_block4x +ST4( bl aes_encrypt_block4x ) +ST5( ld1 {v4.16b}, [x1], #16 ) +ST5( bl aes_encrypt_block5x ) st1 {v0.16b-v3.16b}, [x0], #64 +ST5( st1 {v4.16b}, [x0], #16 ) b .LecbencloopNx .Lecbenc1x: - adds w4, w4, #4 + adds w4, w4, #MAX_STRIDE beq .Lecbencout .Lecbencloop: ld1 {v0.16b}, [x1], #16 /* get next pt block */ @@ -62,14 +89,17 @@ AES_ENTRY(aes_ecb_decrypt) dec_prepare w3, x2, x5 .LecbdecloopNx: - subs w4, w4, #4 + subs w4, w4, #MAX_STRIDE bmi .Lecbdec1x ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */ - bl aes_decrypt_block4x +ST4( bl aes_decrypt_block4x ) +ST5( ld1 {v4.16b}, [x1], #16 ) +ST5( bl aes_decrypt_block5x ) st1 {v0.16b-v3.16b}, [x0], #64 +ST5( st1 {v4.16b}, [x0], #16 ) b .LecbdecloopNx .Lecbdec1x: - adds w4, w4, #4 + adds w4, w4, #MAX_STRIDE beq .Lecbdecout .Lecbdecloop: ld1 {v0.16b}, [x1], #16 /* get next ct block */ @@ -129,39 +159,56 @@ AES_ENTRY(aes_cbc_decrypt) stp x29, x30, [sp, #-16]! mov x29, sp - ld1 {v7.16b}, [x5] /* get iv */ + ld1 {cbciv.16b}, [x5] /* get iv */ dec_prepare w3, x2, x6 .LcbcdecloopNx: - subs w4, w4, #4 + subs w4, w4, #MAX_STRIDE bmi .Lcbcdec1x ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */ +#if MAX_STRIDE == 5 + ld1 {v4.16b}, [x1], #16 /* get 1 ct block */ + mov v5.16b, v0.16b + mov v6.16b, v1.16b + mov v7.16b, v2.16b + bl aes_decrypt_block5x + sub x1, x1, #32 + eor v0.16b, v0.16b, cbciv.16b + eor v1.16b, v1.16b, v5.16b + ld1 {v5.16b}, [x1], #16 /* reload 1 ct block */ + ld1 {cbciv.16b}, [x1], #16 /* reload 1 ct block */ + eor v2.16b, v2.16b, v6.16b + eor v3.16b, v3.16b, v7.16b + eor v4.16b, v4.16b, v5.16b +#else mov v4.16b, v0.16b mov v5.16b, v1.16b mov v6.16b, v2.16b bl aes_decrypt_block4x sub x1, x1, #16 - eor v0.16b, v0.16b, v7.16b + eor v0.16b, v0.16b, cbciv.16b eor v1.16b, v1.16b, v4.16b - ld1 {v7.16b}, [x1], #16 /* reload 1 ct block */ + ld1 {cbciv.16b}, [x1], #16 /* reload 1 ct block */ eor v2.16b, v2.16b, v5.16b eor v3.16b, v3.16b, v6.16b +#endif st1 {v0.16b-v3.16b}, [x0], #64 +ST5( st1 {v4.16b}, [x0], #16 ) b .LcbcdecloopNx .Lcbcdec1x: - adds w4, w4, #4 + adds w4, w4, #MAX_STRIDE beq .Lcbcdecout .Lcbcdecloop: ld1 {v1.16b}, [x1], #16 /* get next ct block */ mov v0.16b, v1.16b /* ...and copy to v0 */ decrypt_block v0, w3, x2, x6, w7 - eor v0.16b, v0.16b, v7.16b /* xor with iv => pt */ - mov v7.16b, v1.16b /* ct is next iv */ + eor v0.16b, v0.16b, cbciv.16b /* xor with iv => pt */ + mov cbciv.16b, v1.16b /* ct is next iv */ st1 {v0.16b}, [x0], #16 subs w4, w4, #1 bne .Lcbcdecloop .Lcbcdecout: - st1 {v7.16b}, [x5] /* return iv */ + st1 {cbciv.16b}, [x5] /* return iv */ ldp x29, x30, [sp], #16 ret AES_ENDPROC(aes_cbc_decrypt) @@ -255,51 +302,60 @@ AES_ENTRY(aes_ctr_encrypt) mov x29, sp enc_prepare w3, x2, x6 - ld1 {v4.16b}, [x5] + ld1 {vctr.16b}, [x5] - umov x6, v4.d[1] /* keep swabbed ctr in reg */ + umov x6, vctr.d[1] /* keep swabbed ctr in reg */ rev x6, x6 cmn w6, w4 /* 32 bit overflow? */ bcs .Lctrloop .LctrloopNx: - subs w4, w4, #4 + subs w4, w4, #MAX_STRIDE bmi .Lctr1x add w7, w6, #1 - mov v0.16b, v4.16b + mov v0.16b, vctr.16b add w8, w6, #2 - mov v1.16b, v4.16b + mov v1.16b, vctr.16b + add w9, w6, #3 + mov v2.16b, vctr.16b add w9, w6, #3 - mov v2.16b, v4.16b rev w7, w7 - mov v3.16b, v4.16b + mov v3.16b, vctr.16b rev w8, w8 +ST5( mov v4.16b, vctr.16b ) mov v1.s[3], w7 rev w9, w9 +ST5( add w10, w6, #4 ) mov v2.s[3], w8 +ST5( rev w10, w10 ) mov v3.s[3], w9 +ST5( mov v4.s[3], w10 ) ld1 {v5.16b-v7.16b}, [x1], #48 /* get 3 input blocks */ - bl aes_encrypt_block4x +ST4( bl aes_encrypt_block4x ) +ST5( bl aes_encrypt_block5x ) eor v0.16b, v5.16b, v0.16b - ld1 {v5.16b}, [x1], #16 /* get 1 input block */ +ST4( ld1 {v5.16b}, [x1], #16 ) eor v1.16b, v6.16b, v1.16b +ST5( ld1 {v5.16b-v6.16b}, [x1], #32 ) eor v2.16b, v7.16b, v2.16b eor v3.16b, v5.16b, v3.16b +ST5( eor v4.16b, v6.16b, v4.16b ) st1 {v0.16b-v3.16b}, [x0], #64 - add x6, x6, #4 +ST5( st1 {v4.16b}, [x0], #16 ) + add x6, x6, #MAX_STRIDE rev x7, x6 - ins v4.d[1], x7 + ins vctr.d[1], x7 cbz w4, .Lctrout b .LctrloopNx .Lctr1x: - adds w4, w4, #4 + adds w4, w4, #MAX_STRIDE beq .Lctrout .Lctrloop: - mov v0.16b, v4.16b + mov v0.16b, vctr.16b encrypt_block v0, w3, x2, x8, w7 adds x6, x6, #1 /* increment BE ctr */ rev x7, x6 - ins v4.d[1], x7 + ins vctr.d[1], x7 bcs .Lctrcarry /* overflow? */ .Lctrcarrydone: @@ -311,7 +367,7 @@ AES_ENTRY(aes_ctr_encrypt) bne .Lctrloop .Lctrout: - st1 {v4.16b}, [x5] /* return next CTR value */ + st1 {vctr.16b}, [x5] /* return next CTR value */ ldp x29, x30, [sp], #16 ret @@ -320,11 +376,11 @@ AES_ENTRY(aes_ctr_encrypt) b .Lctrout .Lctrcarry: - umov x7, v4.d[0] /* load upper word of ctr */ + umov x7, vctr.d[0] /* load upper word of ctr */ rev x7, x7 /* ... to handle the carry */ add x7, x7, #1 rev x7, x7 - ins v4.d[0], x7 + ins vctr.d[0], x7 b .Lctrcarrydone AES_ENDPROC(aes_ctr_encrypt) diff --git a/arch/arm64/crypto/aes-neon.S b/arch/arm64/crypto/aes-neon.S index d261331747f2..2bebccc73869 100644 --- a/arch/arm64/crypto/aes-neon.S +++ b/arch/arm64/crypto/aes-neon.S @@ -12,6 +12,8 @@ #define AES_ENDPROC(func) ENDPROC(neon_ ## func) xtsmask .req v7 + cbciv .req v7 + vctr .req v4 .macro xts_reload_mask, tmp xts_load_mask \tmp @@ -114,26 +116,9 @@ /* * Interleaved versions: functionally equivalent to the - * ones above, but applied to 2 or 4 AES states in parallel. + * ones above, but applied to AES states in parallel. */ - .macro sub_bytes_2x, in0, in1 - sub v8.16b, \in0\().16b, v15.16b - tbl \in0\().16b, {v16.16b-v19.16b}, \in0\().16b - sub v9.16b, \in1\().16b, v15.16b - tbl \in1\().16b, {v16.16b-v19.16b}, \in1\().16b - sub v10.16b, v8.16b, v15.16b - tbx \in0\().16b, {v20.16b-v23.16b}, v8.16b - sub v11.16b, v9.16b, v15.16b - tbx \in1\().16b, {v20.16b-v23.16b}, v9.16b - sub v8.16b, v10.16b, v15.16b - tbx \in0\().16b, {v24.16b-v27.16b}, v10.16b - sub v9.16b, v11.16b, v15.16b - tbx \in1\().16b, {v24.16b-v27.16b}, v11.16b - tbx \in0\().16b, {v28.16b-v31.16b}, v8.16b - tbx \in1\().16b, {v28.16b-v31.16b}, v9.16b - .endm - .macro sub_bytes_4x, in0, in1, in2, in3 sub v8.16b, \in0\().16b, v15.16b tbl \in0\().16b, {v16.16b-v19.16b}, \in0\().16b @@ -212,25 +197,6 @@ eor \in1\().16b, \in1\().16b, v11.16b .endm - .macro do_block_2x, enc, in0, in1, rounds, rk, rkp, i - ld1 {v15.4s}, [\rk] - add \rkp, \rk, #16 - mov \i, \rounds -1111: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */ - eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */ - movi v15.16b, #0x40 - tbl \in0\().16b, {\in0\().16b}, v13.16b /* ShiftRows */ - tbl \in1\().16b, {\in1\().16b}, v13.16b /* ShiftRows */ - sub_bytes_2x \in0, \in1 - subs \i, \i, #1 - ld1 {v15.4s}, [\rkp], #16 - beq 2222f - mix_columns_2x \in0, \in1, \enc - b 1111b -2222: eor \in0\().16b, \in0\().16b, v15.16b /* ^round key */ - eor \in1\().16b, \in1\().16b, v15.16b /* ^round key */ - .endm - .macro do_block_4x, enc, in0, in1, in2, in3, rounds, rk, rkp, i ld1 {v15.4s}, [\rk] add \rkp, \rk, #16 @@ -257,14 +223,6 @@ eor \in3\().16b, \in3\().16b, v15.16b /* ^round key */ .endm - .macro encrypt_block2x, in0, in1, rounds, rk, rkp, i - do_block_2x 1, \in0, \in1, \rounds, \rk, \rkp, \i - .endm - - .macro decrypt_block2x, in0, in1, rounds, rk, rkp, i - do_block_2x 0, \in0, \in1, \rounds, \rk, \rkp, \i - .endm - .macro encrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i do_block_4x 1, \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i .endm diff --git a/arch/arm64/crypto/chacha-neon-glue.c b/arch/arm64/crypto/chacha-neon-glue.c index 82029cda2e77..1495d2b18518 100644 --- a/arch/arm64/crypto/chacha-neon-glue.c +++ b/arch/arm64/crypto/chacha-neon-glue.c @@ -60,7 +60,7 @@ static void chacha_doneon(u32 *state, u8 *dst, const u8 *src, } static int chacha_neon_stream_xor(struct skcipher_request *req, - struct chacha_ctx *ctx, u8 *iv) + const struct chacha_ctx *ctx, const u8 *iv) { struct skcipher_walk walk; u32 state[16]; diff --git a/arch/arm64/crypto/sha1-ce-glue.c b/arch/arm64/crypto/sha1-ce-glue.c index ecb0f67e5998..bdc1b6d7aff7 100644 --- a/arch/arm64/crypto/sha1-ce-glue.c +++ b/arch/arm64/crypto/sha1-ce-glue.c @@ -52,7 +52,7 @@ static int sha1_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { struct sha1_ce_state *sctx = shash_desc_ctx(desc); - bool finalize = !sctx->sst.count && !(len % SHA1_BLOCK_SIZE); + bool finalize = !sctx->sst.count && !(len % SHA1_BLOCK_SIZE) && len; if (!crypto_simd_usable()) return crypto_sha1_finup(desc, data, len, out); diff --git a/arch/arm64/crypto/sha2-ce-glue.c b/arch/arm64/crypto/sha2-ce-glue.c index 955c3c2d3f5a..604a01a4ede6 100644 --- a/arch/arm64/crypto/sha2-ce-glue.c +++ b/arch/arm64/crypto/sha2-ce-glue.c @@ -57,7 +57,7 @@ static int sha256_ce_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { struct sha256_ce_state *sctx = shash_desc_ctx(desc); - bool finalize = !sctx->sst.count && !(len % SHA256_BLOCK_SIZE); + bool finalize = !sctx->sst.count && !(len % SHA256_BLOCK_SIZE) && len; if (!crypto_simd_usable()) { if (len) diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index e9b866e87d48..73c0ccb009a0 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -371,20 +371,6 @@ static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) } } -static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) -{ - struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); - - aesni_enc(ctx, dst, src); -} - -static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) -{ - struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); - - aesni_dec(ctx, dst, src); -} - static int aesni_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int len) { @@ -920,7 +906,7 @@ static int helper_rfc4106_decrypt(struct aead_request *req) } #endif -static struct crypto_alg aesni_algs[] = { { +static struct crypto_alg aesni_cipher_alg = { .cra_name = "aes", .cra_driver_name = "aes-aesni", .cra_priority = 300, @@ -937,24 +923,7 @@ static struct crypto_alg aesni_algs[] = { { .cia_decrypt = aes_decrypt } } -}, { - .cra_name = "__aes", - .cra_driver_name = "__aes-aesni", - .cra_priority = 300, - .cra_flags = CRYPTO_ALG_TYPE_CIPHER | CRYPTO_ALG_INTERNAL, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = CRYPTO_AES_CTX_SIZE, - .cra_module = THIS_MODULE, - .cra_u = { - .cipher = { - .cia_min_keysize = AES_MIN_KEY_SIZE, - .cia_max_keysize = AES_MAX_KEY_SIZE, - .cia_setkey = aes_set_key, - .cia_encrypt = __aes_encrypt, - .cia_decrypt = __aes_decrypt - } - } -} }; +}; static struct skcipher_alg aesni_skciphers[] = { { @@ -1150,7 +1119,7 @@ static int __init aesni_init(void) #endif #endif - err = crypto_register_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); + err = crypto_register_alg(&aesni_cipher_alg); if (err) return err; @@ -1158,7 +1127,7 @@ static int __init aesni_init(void) ARRAY_SIZE(aesni_skciphers), aesni_simd_skciphers); if (err) - goto unregister_algs; + goto unregister_cipher; err = simd_register_aeads_compat(aesni_aeads, ARRAY_SIZE(aesni_aeads), aesni_simd_aeads); @@ -1170,8 +1139,8 @@ static int __init aesni_init(void) unregister_skciphers: simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers), aesni_simd_skciphers); -unregister_algs: - crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); +unregister_cipher: + crypto_unregister_alg(&aesni_cipher_alg); return err; } @@ -1181,7 +1150,7 @@ static void __exit aesni_exit(void) aesni_simd_aeads); simd_unregister_skciphers(aesni_skciphers, ARRAY_SIZE(aesni_skciphers), aesni_simd_skciphers); - crypto_unregister_algs(aesni_algs, ARRAY_SIZE(aesni_algs)); + crypto_unregister_alg(&aesni_cipher_alg); } late_initcall(aesni_init); diff --git a/arch/x86/crypto/chacha_glue.c b/arch/x86/crypto/chacha_glue.c index 1ce0019c059c..388f95a4ec24 100644 --- a/arch/x86/crypto/chacha_glue.c +++ b/arch/x86/crypto/chacha_glue.c @@ -124,7 +124,7 @@ static void chacha_dosimd(u32 *state, u8 *dst, const u8 *src, } static int chacha_simd_stream_xor(struct skcipher_walk *walk, - struct chacha_ctx *ctx, u8 *iv) + const struct chacha_ctx *ctx, const u8 *iv) { u32 *state, state_buf[16 + 2] __aligned(8); int next_yield = 4096; /* bytes until next FPU yield */ diff --git a/crypto/Kconfig b/crypto/Kconfig index 3d056e7da65f..e801450bcb1c 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -61,7 +61,6 @@ config CRYPTO_BLKCIPHER2 tristate select CRYPTO_ALGAPI2 select CRYPTO_RNG2 - select CRYPTO_WORKQUEUE config CRYPTO_HASH tristate @@ -137,10 +136,11 @@ config CRYPTO_USER Userspace configuration for cryptographic instantiations such as cbc(aes). +if CRYPTO_MANAGER2 + config CRYPTO_MANAGER_DISABLE_TESTS bool "Disable run-time self tests" default y - depends on CRYPTO_MANAGER2 help Disable run-time self tests that normally take place at algorithm registration. @@ -155,14 +155,10 @@ config CRYPTO_MANAGER_EXTRA_TESTS This is intended for developer use only, as these tests take much longer to run than the normal self tests. +endif # if CRYPTO_MANAGER2 + config CRYPTO_GF128MUL - tristate "GF(2^128) multiplication functions" - help - Efficient table driven implementation of multiplications in the - field GF(2^128). This is needed by some cypher modes. This - option will be selected automatically if you select such a - cipher mode. Only select this option by hand if you expect to load - an external module that requires these functions. + tristate config CRYPTO_NULL tristate "Null algorithms" @@ -186,15 +182,11 @@ config CRYPTO_PCRYPT This converts an arbitrary crypto algorithm into a parallel algorithm that executes in kernel threads. -config CRYPTO_WORKQUEUE - tristate - config CRYPTO_CRYPTD tristate "Software async crypto daemon" select CRYPTO_BLKCIPHER select CRYPTO_HASH select CRYPTO_MANAGER - select CRYPTO_WORKQUEUE help This is a generic software asynchronous crypto daemon that converts an arbitrary synchronous software crypto algorithm @@ -279,6 +271,7 @@ config CRYPTO_CCM select CRYPTO_CTR select CRYPTO_HASH select CRYPTO_AEAD + select CRYPTO_MANAGER help Support for Counter with CBC MAC. Required for IPsec. @@ -288,6 +281,7 @@ config CRYPTO_GCM select CRYPTO_AEAD select CRYPTO_GHASH select CRYPTO_NULL + select CRYPTO_MANAGER help Support for Galois/Counter Mode (GCM) and Galois Message Authentication Code (GMAC). Required for IPSec. @@ -297,6 +291,7 @@ config CRYPTO_CHACHA20POLY1305 select CRYPTO_CHACHA20 select CRYPTO_POLY1305 select CRYPTO_AEAD + select CRYPTO_MANAGER help ChaCha20-Poly1305 AEAD support, RFC7539. @@ -411,6 +406,7 @@ config CRYPTO_SEQIV select CRYPTO_BLKCIPHER select CRYPTO_NULL select CRYPTO_RNG_DEFAULT + select CRYPTO_MANAGER help This IV generator generates an IV based on a sequence number by xoring it with a salt. This algorithm is mainly useful for CTR @@ -420,7 +416,7 @@ config CRYPTO_ECHAINIV select CRYPTO_AEAD select CRYPTO_NULL select CRYPTO_RNG_DEFAULT - default m + select CRYPTO_MANAGER help This IV generator generates an IV based on the encryption of a sequence number xored with a salt. This is the default @@ -456,6 +452,7 @@ config CRYPTO_CTR config CRYPTO_CTS tristate "CTS support" select CRYPTO_BLKCIPHER + select CRYPTO_MANAGER help CTS: Cipher Text Stealing This is the Cipher Text Stealing mode as described by @@ -521,6 +518,7 @@ config CRYPTO_XTS config CRYPTO_KEYWRAP tristate "Key wrapping support" select CRYPTO_BLKCIPHER + select CRYPTO_MANAGER help Support for key wrapping (NIST SP800-38F / RFC3394) without padding. @@ -551,6 +549,7 @@ config CRYPTO_ADIANTUM select CRYPTO_CHACHA20 select CRYPTO_POLY1305 select CRYPTO_NHPOLY1305 + select CRYPTO_MANAGER help Adiantum is a tweakable, length-preserving encryption mode designed for fast and secure disk encryption, especially on @@ -684,6 +683,14 @@ config CRYPTO_CRC32_MIPS instructions, when available. +config CRYPTO_XXHASH + tristate "xxHash hash algorithm" + select CRYPTO_HASH + select XXHASH + help + xxHash non-cryptographic hash algorithm. Extremely fast, working at + speeds close to RAM limits. + config CRYPTO_CRCT10DIF tristate "CRCT10DIF algorithm" select CRYPTO_HASH @@ -1230,9 +1237,13 @@ config CRYPTO_ANUBIS <https://www.cosic.esat.kuleuven.be/nessie/reports/> <http://www.larc.usp.br/~pbarreto/AnubisPage.html> +config CRYPTO_LIB_ARC4 + tristate + config CRYPTO_ARC4 tristate "ARC4 cipher algorithm" select CRYPTO_BLKCIPHER + select CRYPTO_LIB_ARC4 help ARC4 cipher algorithm. diff --git a/crypto/Makefile b/crypto/Makefile index 266a4cdbb9e2..9479e1a45d8c 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -6,8 +6,6 @@ obj-$(CONFIG_CRYPTO) += crypto.o crypto-y := api.o cipher.o compress.o memneq.o -obj-$(CONFIG_CRYPTO_WORKQUEUE) += crypto_wq.o - obj-$(CONFIG_CRYPTO_ENGINE) += crypto_engine.o obj-$(CONFIG_CRYPTO_FIPS) += fips.o @@ -131,6 +129,7 @@ obj-$(CONFIG_CRYPTO_AUTHENC) += authenc.o authencesn.o obj-$(CONFIG_CRYPTO_LZO) += lzo.o lzo-rle.o obj-$(CONFIG_CRYPTO_LZ4) += lz4.o obj-$(CONFIG_CRYPTO_LZ4HC) += lz4hc.o +obj-$(CONFIG_CRYPTO_XXHASH) += xxhash_generic.o obj-$(CONFIG_CRYPTO_842) += 842.o obj-$(CONFIG_CRYPTO_RNG2) += rng.o obj-$(CONFIG_CRYPTO_ANSI_CPRNG) += ansi_cprng.o diff --git a/crypto/aead.c b/crypto/aead.c index c3c158ba9883..fbf0ec93bc8e 100644 --- a/crypto/aead.c +++ b/crypto/aead.c @@ -84,6 +84,42 @@ int crypto_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) } EXPORT_SYMBOL_GPL(crypto_aead_setauthsize); +int crypto_aead_encrypt(struct aead_request *req) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct crypto_alg *alg = aead->base.__crt_alg; + unsigned int cryptlen = req->cryptlen; + int ret; + + crypto_stats_get(alg); + if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY) + ret = -ENOKEY; + else + ret = crypto_aead_alg(aead)->encrypt(req); + crypto_stats_aead_encrypt(cryptlen, alg, ret); + return ret; +} +EXPORT_SYMBOL_GPL(crypto_aead_encrypt); + +int crypto_aead_decrypt(struct aead_request *req) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct crypto_alg *alg = aead->base.__crt_alg; + unsigned int cryptlen = req->cryptlen; + int ret; + + crypto_stats_get(alg); + if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY) + ret = -ENOKEY; + else if (req->cryptlen < crypto_aead_authsize(aead)) + ret = -EINVAL; + else + ret = crypto_aead_alg(aead)->decrypt(req); + crypto_stats_aead_decrypt(cryptlen, alg, ret); + return ret; +} +EXPORT_SYMBOL_GPL(crypto_aead_decrypt); + static void crypto_aead_exit_tfm(struct crypto_tfm *tfm) { struct crypto_aead *aead = __crypto_aead_cast(tfm); diff --git a/crypto/algapi.c b/crypto/algapi.c index 313a7682cef1..de30ddc952d8 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -21,23 +21,6 @@ static LIST_HEAD(crypto_template_list); -static inline int crypto_set_driver_name(struct crypto_alg *alg) -{ - static const char suffix[] = "-generic"; - char *driver_name = alg->cra_driver_name; - int len; - - if (*driver_name) - return 0; - - len = strlcpy(driver_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); - if (len + sizeof(suffix) > CRYPTO_MAX_ALG_NAME) - return -ENAMETOOLONG; - - memcpy(driver_name + len, suffix, sizeof(suffix)); - return 0; -} - static inline void crypto_check_module_sig(struct module *mod) { if (fips_enabled && mod && !module_sig_ok(mod)) @@ -49,6 +32,9 @@ static int crypto_check_alg(struct crypto_alg *alg) { crypto_check_module_sig(alg->cra_module); + if (!alg->cra_name[0] || !alg->cra_driver_name[0]) + return -EINVAL; + if (alg->cra_alignmask & (alg->cra_alignmask + 1)) return -EINVAL; @@ -74,7 +60,7 @@ static int crypto_check_alg(struct crypto_alg *alg) refcount_set(&alg->cra_refcnt, 1); - return crypto_set_driver_name(alg); + return 0; } static void crypto_free_instance(struct crypto_instance *inst) @@ -947,19 +933,6 @@ struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue) } EXPORT_SYMBOL_GPL(crypto_dequeue_request); -int crypto_tfm_in_queue(struct crypto_queue *queue, struct crypto_tfm *tfm) -{ - struct crypto_async_request *req; - - list_for_each_entry(req, &queue->list, list) { - if (req->tfm == tfm) - return 1; - } - - return 0; -} -EXPORT_SYMBOL_GPL(crypto_tfm_in_queue); - static inline void crypto_inc_byte(u8 *a, unsigned int size) { u8 *b = (a + size); diff --git a/crypto/anubis.c b/crypto/anubis.c index 673927de0eb9..f9ce78fde6ee 100644 --- a/crypto/anubis.c +++ b/crypto/anubis.c @@ -673,6 +673,7 @@ static void anubis_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) static struct crypto_alg anubis_alg = { .cra_name = "anubis", + .cra_driver_name = "anubis-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = ANUBIS_BLOCK_SIZE, .cra_ctxsize = sizeof (struct anubis_ctx), diff --git a/crypto/arc4.c b/crypto/arc4.c index a2120e06bf84..aa79571dbd49 100644 --- a/crypto/arc4.c +++ b/crypto/arc4.c @@ -13,84 +13,15 @@ #include <linux/init.h> #include <linux/module.h> -struct arc4_ctx { - u32 S[256]; - u32 x, y; -}; - -static int arc4_set_key(struct crypto_tfm *tfm, const u8 *in_key, - unsigned int key_len) -{ - struct arc4_ctx *ctx = crypto_tfm_ctx(tfm); - int i, j = 0, k = 0; - - ctx->x = 1; - ctx->y = 0; - - for (i = 0; i < 256; i++) - ctx->S[i] = i; - - for (i = 0; i < 256; i++) { - u32 a = ctx->S[i]; - j = (j + in_key[k] + a) & 0xff; - ctx->S[i] = ctx->S[j]; - ctx->S[j] = a; - if (++k >= key_len) - k = 0; - } - - return 0; -} - -static int arc4_set_key_skcipher(struct crypto_skcipher *tfm, const u8 *in_key, - unsigned int key_len) +static int crypto_arc4_setkey(struct crypto_skcipher *tfm, const u8 *in_key, + unsigned int key_len) { - return arc4_set_key(&tfm->base, in_key, key_len); -} - -static void arc4_crypt(struct arc4_ctx *ctx, u8 *out, const u8 *in, - unsigned int len) -{ - u32 *const S = ctx->S; - u32 x, y, a, b; - u32 ty, ta, tb; - - if (len == 0) - return; - - x = ctx->x; - y = ctx->y; - - a = S[x]; - y = (y + a) & 0xff; - b = S[y]; - - do { - S[y] = a; - a = (a + b) & 0xff; - S[x] = b; - x = (x + 1) & 0xff; - ta = S[x]; - ty = (y + ta) & 0xff; - tb = S[ty]; - *out++ = *in++ ^ S[a]; - if (--len == 0) - break; - y = ty; - a = ta; - b = tb; - } while (true); - - ctx->x = x; - ctx->y = y; -} + struct arc4_ctx *ctx = crypto_skcipher_ctx(tfm); -static void arc4_crypt_one(struct crypto_tfm *tfm, u8 *out, const u8 *in) -{ - arc4_crypt(crypto_tfm_ctx(tfm), out, in, 1); + return arc4_setkey(ctx, in_key, key_len); } -static int ecb_arc4_crypt(struct skcipher_request *req) +static int crypto_arc4_crypt(struct skcipher_request *req) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); struct arc4_ctx *ctx = crypto_skcipher_ctx(tfm); @@ -108,54 +39,32 @@ static int ecb_arc4_crypt(struct skcipher_request *req) return err; } -static struct crypto_alg arc4_cipher = { - .cra_name = "arc4", - .cra_flags = CRYPTO_ALG_TYPE_CIPHER, - .cra_blocksize = ARC4_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct arc4_ctx), - .cra_module = THIS_MODULE, - .cra_u = { - .cipher = { - .cia_min_keysize = ARC4_MIN_KEY_SIZE, - .cia_max_keysize = ARC4_MAX_KEY_SIZE, - .cia_setkey = arc4_set_key, - .cia_encrypt = arc4_crypt_one, - .cia_decrypt = arc4_crypt_one, - }, - }, -}; - -static struct skcipher_alg arc4_skcipher = { +static struct skcipher_alg arc4_alg = { + /* + * For legacy reasons, this is named "ecb(arc4)", not "arc4". + * Nevertheless it's actually a stream cipher, not a block cipher. + */ .base.cra_name = "ecb(arc4)", + .base.cra_driver_name = "ecb(arc4)-generic", .base.cra_priority = 100, .base.cra_blocksize = ARC4_BLOCK_SIZE, .base.cra_ctxsize = sizeof(struct arc4_ctx), .base.cra_module = THIS_MODULE, .min_keysize = ARC4_MIN_KEY_SIZE, .max_keysize = ARC4_MAX_KEY_SIZE, - .setkey = arc4_set_key_skcipher, - .encrypt = ecb_arc4_crypt, - .decrypt = ecb_arc4_crypt, + .setkey = crypto_arc4_setkey, + .encrypt = crypto_arc4_crypt, + .decrypt = crypto_arc4_crypt, }; static int __init arc4_init(void) { - int err; - - err = crypto_register_alg(&arc4_cipher); - if (err) - return err; - - err = crypto_register_skcipher(&arc4_skcipher); - if (err) - crypto_unregister_alg(&arc4_cipher); - return err; + return crypto_register_skcipher(&arc4_alg); } static void __exit arc4_exit(void) { - crypto_unregister_alg(&arc4_cipher); - crypto_unregister_skcipher(&arc4_skcipher); + crypto_unregister_skcipher(&arc4_alg); } subsys_initcall(arc4_init); @@ -164,4 +73,4 @@ module_exit(arc4_exit); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("ARC4 Cipher Algorithm"); MODULE_AUTHOR("Jon Oberheide <jon@oberheide.org>"); -MODULE_ALIAS_CRYPTO("arc4"); +MODULE_ALIAS_CRYPTO("ecb(arc4)"); diff --git a/crypto/asymmetric_keys/Kconfig b/crypto/asymmetric_keys/Kconfig index be70ca6c85d3..1f1f004dc757 100644 --- a/crypto/asymmetric_keys/Kconfig +++ b/crypto/asymmetric_keys/Kconfig @@ -15,6 +15,7 @@ config ASYMMETRIC_PUBLIC_KEY_SUBTYPE select MPILIB select CRYPTO_HASH_INFO select CRYPTO_AKCIPHER + select CRYPTO_HASH help This option provides support for asymmetric public key type handling. If signature generation and/or verification are to be used, @@ -65,6 +66,7 @@ config TPM_KEY_PARSER config PKCS7_MESSAGE_PARSER tristate "PKCS#7 message parser" depends on X509_CERTIFICATE_PARSER + select CRYPTO_HASH select ASN1 select OID_REGISTRY help @@ -87,6 +89,7 @@ config SIGNED_PE_FILE_VERIFICATION bool "Support for PE file signature verification" depends on PKCS7_MESSAGE_PARSER=y depends on SYSTEM_DATA_VERIFICATION + select CRYPTO_HASH select ASN1 select OID_REGISTRY help diff --git a/crypto/chacha20poly1305.c b/crypto/chacha20poly1305.c index 2db7eac4bf3b..74e824e537e6 100644 --- a/crypto/chacha20poly1305.c +++ b/crypto/chacha20poly1305.c @@ -61,6 +61,8 @@ struct chachapoly_req_ctx { unsigned int cryptlen; /* Actual AD, excluding IV */ unsigned int assoclen; + /* request flags, with MAY_SLEEP cleared if needed */ + u32 flags; union { struct poly_req poly; struct chacha_req chacha; @@ -70,8 +72,12 @@ struct chachapoly_req_ctx { static inline void async_done_continue(struct aead_request *req, int err, int (*cont)(struct aead_request *)) { - if (!err) + if (!err) { + struct chachapoly_req_ctx *rctx = aead_request_ctx(req); + + rctx->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; err = cont(req); + } if (err != -EINPROGRESS && err != -EBUSY) aead_request_complete(req, err); @@ -129,16 +135,12 @@ static int chacha_decrypt(struct aead_request *req) chacha_iv(creq->iv, req, 1); - sg_init_table(rctx->src, 2); src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen); dst = src; - - if (req->src != req->dst) { - sg_init_table(rctx->dst, 2); + if (req->src != req->dst) dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen); - } - skcipher_request_set_callback(&creq->req, aead_request_flags(req), + skcipher_request_set_callback(&creq->req, rctx->flags, chacha_decrypt_done, req); skcipher_request_set_tfm(&creq->req, ctx->chacha); skcipher_request_set_crypt(&creq->req, src, dst, @@ -172,17 +174,13 @@ static int poly_tail(struct aead_request *req) struct chachapoly_ctx *ctx = crypto_aead_ctx(tfm); struct chachapoly_req_ctx *rctx = aead_request_ctx(req); struct poly_req *preq = &rctx->u.poly; - __le64 len; int err; - sg_init_table(preq->src, 1); - len = cpu_to_le64(rctx->assoclen); - memcpy(&preq->tail.assoclen, &len, sizeof(len)); - len = cpu_to_le64(rctx->cryptlen); - memcpy(&preq->tail.cryptlen, &len, sizeof(len)); - sg_set_buf(preq->src, &preq->tail, sizeof(preq->tail)); + preq->tail.assoclen = cpu_to_le64(rctx->assoclen); + preq->tail.cryptlen = cpu_to_le64(rctx->cryptlen); + sg_init_one(preq->src, &preq->tail, sizeof(preq->tail)); - ahash_request_set_callback(&preq->req, aead_request_flags(req), + ahash_request_set_callback(&preq->req, rctx->flags, poly_tail_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); ahash_request_set_crypt(&preq->req, preq->src, @@ -205,15 +203,14 @@ static int poly_cipherpad(struct aead_request *req) struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); struct chachapoly_req_ctx *rctx = aead_request_ctx(req); struct poly_req *preq = &rctx->u.poly; - unsigned int padlen, bs = POLY1305_BLOCK_SIZE; + unsigned int padlen; int err; - padlen = (bs - (rctx->cryptlen % bs)) % bs; + padlen = -rctx->cryptlen % POLY1305_BLOCK_SIZE; memset(preq->pad, 0, sizeof(preq->pad)); - sg_init_table(preq->src, 1); - sg_set_buf(preq->src, &preq->pad, padlen); + sg_init_one(preq->src, preq->pad, padlen); - ahash_request_set_callback(&preq->req, aead_request_flags(req), + ahash_request_set_callback(&preq->req, rctx->flags, poly_cipherpad_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen); @@ -241,10 +238,9 @@ static int poly_cipher(struct aead_request *req) if (rctx->cryptlen == req->cryptlen) /* encrypting */ crypt = req->dst; - sg_init_table(rctx->src, 2); crypt = scatterwalk_ffwd(rctx->src, crypt, req->assoclen); - ahash_request_set_callback(&preq->req, aead_request_flags(req), + ahash_request_set_callback(&preq->req, rctx->flags, poly_cipher_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); ahash_request_set_crypt(&preq->req, crypt, NULL, rctx->cryptlen); @@ -266,15 +262,14 @@ static int poly_adpad(struct aead_request *req) struct chachapoly_ctx *ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); struct chachapoly_req_ctx *rctx = aead_request_ctx(req); struct poly_req *preq = &rctx->u.poly; - unsigned int padlen, bs = POLY1305_BLOCK_SIZE; + unsigned int padlen; int err; - padlen = (bs - (rctx->assoclen % bs)) % bs; + padlen = -rctx->assoclen % POLY1305_BLOCK_SIZE; memset(preq->pad, 0, sizeof(preq->pad)); - sg_init_table(preq->src, 1); - sg_set_buf(preq->src, preq->pad, padlen); + sg_init_one(preq->src, preq->pad, padlen); - ahash_request_set_callback(&preq->req, aead_request_flags(req), + ahash_request_set_callback(&preq->req, rctx->flags, poly_adpad_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); ahash_request_set_crypt(&preq->req, preq->src, NULL, padlen); @@ -298,7 +293,7 @@ static int poly_ad(struct aead_request *req) struct poly_req *preq = &rctx->u.poly; int err; - ahash_request_set_callback(&preq->req, aead_request_flags(req), + ahash_request_set_callback(&preq->req, rctx->flags, poly_ad_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); ahash_request_set_crypt(&preq->req, req->src, NULL, rctx->assoclen); @@ -322,10 +317,9 @@ static int poly_setkey(struct aead_request *req) struct poly_req *preq = &rctx->u.poly; int err; - sg_init_table(preq->src, 1); - sg_set_buf(preq->src, rctx->key, sizeof(rctx->key)); + sg_init_one(preq->src, rctx->key, sizeof(rctx->key)); - ahash_request_set_callback(&preq->req, aead_request_flags(req), + ahash_request_set_callback(&preq->req, rctx->flags, poly_setkey_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); ahash_request_set_crypt(&preq->req, preq->src, NULL, sizeof(rctx->key)); @@ -349,7 +343,7 @@ static int poly_init(struct aead_request *req) struct poly_req *preq = &rctx->u.poly; int err; - ahash_request_set_callback(&preq->req, aead_request_flags(req), + ahash_request_set_callback(&preq->req, rctx->flags, poly_init_done, req); ahash_request_set_tfm(&preq->req, ctx->poly); @@ -381,13 +375,12 @@ static int poly_genkey(struct aead_request *req) rctx->assoclen -= 8; } - sg_init_table(creq->src, 1); memset(rctx->key, 0, sizeof(rctx->key)); - sg_set_buf(creq->src, rctx->key, sizeof(rctx->key)); + sg_init_one(creq->src, rctx->key, sizeof(rctx->key)); chacha_iv(creq->iv, req, 0); - skcipher_request_set_callback(&creq->req, aead_request_flags(req), + skcipher_request_set_callback(&creq->req, rctx->flags, poly_genkey_done, req); skcipher_request_set_tfm(&creq->req, ctx->chacha); skcipher_request_set_crypt(&creq->req, creq->src, creq->src, @@ -418,16 +411,12 @@ static int chacha_encrypt(struct aead_request *req) chacha_iv(creq->iv, req, 1); - sg_init_table(rctx->src, 2); src = scatterwalk_ffwd(rctx->src, req->src, req->assoclen); dst = src; - - if (req->src != req->dst) { - sg_init_table(rctx->dst, 2); + if (req->src != req->dst) dst = scatterwalk_ffwd(rctx->dst, req->dst, req->assoclen); - } - skcipher_request_set_callback(&creq->req, aead_request_flags(req), + skcipher_request_set_callback(&creq->req, rctx->flags, chacha_encrypt_done, req); skcipher_request_set_tfm(&creq->req, ctx->chacha); skcipher_request_set_crypt(&creq->req, src, dst, @@ -445,6 +434,7 @@ static int chachapoly_encrypt(struct aead_request *req) struct chachapoly_req_ctx *rctx = aead_request_ctx(req); rctx->cryptlen = req->cryptlen; + rctx->flags = aead_request_flags(req); /* encrypt call chain: * - chacha_encrypt/done() @@ -466,6 +456,7 @@ static int chachapoly_decrypt(struct aead_request *req) struct chachapoly_req_ctx *rctx = aead_request_ctx(req); rctx->cryptlen = req->cryptlen - POLY1305_DIGEST_SIZE; + rctx->flags = aead_request_flags(req); /* decrypt call chain: * - poly_genkey/done() diff --git a/crypto/chacha_generic.c b/crypto/chacha_generic.c index 04404c479e68..085d8d219987 100644 --- a/crypto/chacha_generic.c +++ b/crypto/chacha_generic.c @@ -32,7 +32,7 @@ static void chacha_docrypt(u32 *state, u8 *dst, const u8 *src, } static int chacha_stream_xor(struct skcipher_request *req, - struct chacha_ctx *ctx, u8 *iv) + const struct chacha_ctx *ctx, const u8 *iv) { struct skcipher_walk walk; u32 state[16]; @@ -56,7 +56,7 @@ static int chacha_stream_xor(struct skcipher_request *req, return err; } -void crypto_chacha_init(u32 *state, struct chacha_ctx *ctx, u8 *iv) +void crypto_chacha_init(u32 *state, const struct chacha_ctx *ctx, const u8 *iv) { state[0] = 0x61707865; /* "expa" */ state[1] = 0x3320646e; /* "nd 3" */ diff --git a/crypto/cryptd.c b/crypto/cryptd.c index 5f76c6e222c6..3748f9b4516d 100644 --- a/crypto/cryptd.c +++ b/crypto/cryptd.c @@ -16,7 +16,6 @@ #include <crypto/internal/aead.h> #include <crypto/internal/skcipher.h> #include <crypto/cryptd.h> -#include <crypto/crypto_wq.h> #include <linux/atomic.h> #include <linux/err.h> #include <linux/init.h> @@ -26,11 +25,14 @@ #include <linux/scatterlist.h> #include <linux/sched.h> #include <linux/slab.h> +#include <linux/workqueue.h> static unsigned int cryptd_max_cpu_qlen = 1000; module_param(cryptd_max_cpu_qlen, uint, 0); MODULE_PARM_DESC(cryptd_max_cpu_qlen, "Set cryptd Max queue depth"); +static struct workqueue_struct *cryptd_wq; + struct cryptd_cpu_queue { struct crypto_queue queue; struct work_struct work; @@ -136,7 +138,7 @@ static int cryptd_enqueue_request(struct cryptd_queue *queue, if (err == -ENOSPC) goto out_put_cpu; - queue_work_on(cpu, kcrypto_wq, &cpu_queue->work); + queue_work_on(cpu, cryptd_wq, &cpu_queue->work); if (!atomic_read(refcnt)) goto out_put_cpu; @@ -179,7 +181,7 @@ static void cryptd_queue_worker(struct work_struct *work) req->complete(req, 0); if (cpu_queue->queue.qlen) - queue_work(kcrypto_wq, &cpu_queue->work); + queue_work(cryptd_wq, &cpu_queue->work); } static inline struct cryptd_queue *cryptd_get_queue(struct crypto_tfm *tfm) @@ -919,7 +921,7 @@ static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb) switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { case CRYPTO_ALG_TYPE_BLKCIPHER: return cryptd_create_skcipher(tmpl, tb, &queue); - case CRYPTO_ALG_TYPE_DIGEST: + case CRYPTO_ALG_TYPE_HASH: return cryptd_create_hash(tmpl, tb, &queue); case CRYPTO_ALG_TYPE_AEAD: return cryptd_create_aead(tmpl, tb, &queue); @@ -1119,19 +1121,31 @@ static int __init cryptd_init(void) { int err; + cryptd_wq = alloc_workqueue("cryptd", WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, + 1); + if (!cryptd_wq) + return -ENOMEM; + err = cryptd_init_queue(&queue, cryptd_max_cpu_qlen); if (err) - return err; + goto err_destroy_wq; err = crypto_register_template(&cryptd_tmpl); if (err) - cryptd_fini_queue(&queue); + goto err_fini_queue; + return 0; + +err_fini_queue: + cryptd_fini_queue(&queue); +err_destroy_wq: + destroy_workqueue(cryptd_wq); return err; } static void __exit cryptd_exit(void) { + destroy_workqueue(cryptd_wq); cryptd_fini_queue(&queue); crypto_unregister_template(&cryptd_tmpl); } diff --git a/crypto/crypto_null.c b/crypto/crypto_null.c index 0d341ddecd54..5b84b0f7cc17 100644 --- a/crypto/crypto_null.c +++ b/crypto/crypto_null.c @@ -100,6 +100,7 @@ static struct shash_alg digest_null = { .final = null_final, .base = { .cra_name = "digest_null", + .cra_driver_name = "digest_null-generic", .cra_blocksize = NULL_BLOCK_SIZE, .cra_module = THIS_MODULE, } @@ -122,6 +123,7 @@ static struct skcipher_alg skcipher_null = { static struct crypto_alg null_algs[] = { { .cra_name = "cipher_null", + .cra_driver_name = "cipher_null-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = NULL_BLOCK_SIZE, .cra_ctxsize = 0, @@ -134,6 +136,7 @@ static struct crypto_alg null_algs[] = { { .cia_decrypt = null_crypt } } }, { .cra_name = "compress_null", + .cra_driver_name = "compress_null-generic", .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, .cra_blocksize = NULL_BLOCK_SIZE, .cra_ctxsize = 0, diff --git a/crypto/crypto_wq.c b/crypto/crypto_wq.c deleted file mode 100644 index 80501928e0bb..000000000000 --- a/crypto/crypto_wq.c +++ /dev/null @@ -1,35 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Workqueue for crypto subsystem - * - * Copyright (c) 2009 Intel Corp. - * Author: Huang Ying <ying.huang@intel.com> - */ - -#include <linux/workqueue.h> -#include <linux/module.h> -#include <crypto/algapi.h> -#include <crypto/crypto_wq.h> - -struct workqueue_struct *kcrypto_wq; -EXPORT_SYMBOL_GPL(kcrypto_wq); - -static int __init crypto_wq_init(void) -{ - kcrypto_wq = alloc_workqueue("crypto", - WQ_MEM_RECLAIM | WQ_CPU_INTENSIVE, 1); - if (unlikely(!kcrypto_wq)) - return -ENOMEM; - return 0; -} - -static void __exit crypto_wq_exit(void) -{ - destroy_workqueue(kcrypto_wq); -} - -subsys_initcall(crypto_wq_init); -module_exit(crypto_wq_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Workqueue for crypto subsystem"); diff --git a/crypto/deflate.c b/crypto/deflate.c index 65be51456398..4c0e6c9d942a 100644 --- a/crypto/deflate.c +++ b/crypto/deflate.c @@ -275,6 +275,7 @@ static int deflate_sdecompress(struct crypto_scomp *tfm, const u8 *src, static struct crypto_alg alg = { .cra_name = "deflate", + .cra_driver_name = "deflate-generic", .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, .cra_ctxsize = sizeof(struct deflate_ctx), .cra_module = THIS_MODULE, diff --git a/crypto/drbg.c b/crypto/drbg.c index 2a5b16bb000c..b6929eb5f565 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -220,6 +220,57 @@ static inline unsigned short drbg_sec_strength(drbg_flag_t flags) } /* + * FIPS 140-2 continuous self test for the noise source + * The test is performed on the noise source input data. Thus, the function + * implicitly knows the size of the buffer to be equal to the security + * strength. + * + * Note, this function disregards the nonce trailing the entropy data during + * initial seeding. + * + * drbg->drbg_mutex must have been taken. + * + * @drbg DRBG handle + * @entropy buffer of seed data to be checked + * + * return: + * 0 on success + * -EAGAIN on when the CTRNG is not yet primed + * < 0 on error + */ +static int drbg_fips_continuous_test(struct drbg_state *drbg, + const unsigned char *entropy) +{ + unsigned short entropylen = drbg_sec_strength(drbg->core->flags); + int ret = 0; + + if (!IS_ENABLED(CONFIG_CRYPTO_FIPS)) + return 0; + + /* skip test if we test the overall system */ + if (list_empty(&drbg->test_data.list)) + return 0; + /* only perform test in FIPS mode */ + if (!fips_enabled) + return 0; + + if (!drbg->fips_primed) { + /* Priming of FIPS test */ + memcpy(drbg->prev, entropy, entropylen); + drbg->fips_primed = true; + /* priming: another round is needed */ + return -EAGAIN; + } + ret = memcmp(drbg->prev, entropy, entropylen); + if (!ret) + panic("DRBG continuous self test failed\n"); + memcpy(drbg->prev, entropy, entropylen); + + /* the test shall pass when the two values are not equal */ + return 0; +} + +/* * Convert an integer into a byte representation of this integer. * The byte representation is big-endian * @@ -998,6 +1049,22 @@ static inline int __drbg_seed(struct drbg_state *drbg, struct list_head *seed, return ret; } +static inline int drbg_get_random_bytes(struct drbg_state *drbg, + unsigned char *entropy, + unsigned int entropylen) +{ + int ret; + + do { + get_random_bytes(entropy, entropylen); + ret = drbg_fips_continuous_test(drbg, entropy); + if (ret && ret != -EAGAIN) + return ret; + } while (ret); + + return 0; +} + static void drbg_async_seed(struct work_struct *work) { struct drbg_string data; @@ -1006,16 +1073,20 @@ static void drbg_async_seed(struct work_struct *work) seed_work); unsigned int entropylen = drbg_sec_strength(drbg->core->flags); unsigned char entropy[32]; + int ret; BUG_ON(!entropylen); BUG_ON(entropylen > sizeof(entropy)); - get_random_bytes(entropy, entropylen); drbg_string_fill(&data, entropy, entropylen); list_add_tail(&data.list, &seedlist); mutex_lock(&drbg->drbg_mutex); + ret = drbg_get_random_bytes(drbg, entropy, entropylen); + if (ret) + goto unlock; + /* If nonblocking pool is initialized, deactivate Jitter RNG */ crypto_free_rng(drbg->jent); drbg->jent = NULL; @@ -1030,6 +1101,7 @@ static void drbg_async_seed(struct work_struct *work) if (drbg->seeded) drbg->reseed_threshold = drbg_max_requests(drbg); +unlock: mutex_unlock(&drbg->drbg_mutex); memzero_explicit(entropy, entropylen); @@ -1081,7 +1153,9 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, BUG_ON((entropylen * 2) > sizeof(entropy)); /* Get seed from in-kernel /dev/urandom */ - get_random_bytes(entropy, entropylen); + ret = drbg_get_random_bytes(drbg, entropy, entropylen); + if (ret) + goto out; if (!drbg->jent) { drbg_string_fill(&data1, entropy, entropylen); @@ -1094,7 +1168,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, entropylen); if (ret) { pr_devel("DRBG: jent failed with %d\n", ret); - return ret; + goto out; } drbg_string_fill(&data1, entropy, entropylen * 2); @@ -1121,6 +1195,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers, ret = __drbg_seed(drbg, &seedlist, reseed); +out: memzero_explicit(entropy, entropylen * 2); return ret; @@ -1142,6 +1217,11 @@ static inline void drbg_dealloc_state(struct drbg_state *drbg) drbg->reseed_ctr = 0; drbg->d_ops = NULL; drbg->core = NULL; + if (IS_ENABLED(CONFIG_CRYPTO_FIPS)) { + kzfree(drbg->prev); + drbg->prev = NULL; + drbg->fips_primed = false; + } } /* @@ -1211,6 +1291,14 @@ static inline int drbg_alloc_state(struct drbg_state *drbg) drbg->scratchpad = PTR_ALIGN(drbg->scratchpadbuf, ret + 1); } + if (IS_ENABLED(CONFIG_CRYPTO_FIPS)) { + drbg->prev = kzalloc(drbg_sec_strength(drbg->core->flags), + GFP_KERNEL); + if (!drbg->prev) + goto fini; + drbg->fips_primed = false; + } + return 0; fini: diff --git a/crypto/fcrypt.c b/crypto/fcrypt.c index 4e8704405a3b..58f935315cf8 100644 --- a/crypto/fcrypt.c +++ b/crypto/fcrypt.c @@ -391,6 +391,7 @@ static int fcrypt_setkey(struct crypto_tfm *tfm, const u8 *key, unsigned int key static struct crypto_alg fcrypt_alg = { .cra_name = "fcrypt", + .cra_driver_name = "fcrypt-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = 8, .cra_ctxsize = sizeof(struct fcrypt_ctx), diff --git a/crypto/ghash-generic.c b/crypto/ghash-generic.c index 6425b9cd718e..dad9e1f91a78 100644 --- a/crypto/ghash-generic.c +++ b/crypto/ghash-generic.c @@ -31,6 +31,7 @@ static int ghash_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen) { struct ghash_ctx *ctx = crypto_shash_ctx(tfm); + be128 k; if (keylen != GHASH_BLOCK_SIZE) { crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); @@ -39,7 +40,12 @@ static int ghash_setkey(struct crypto_shash *tfm, if (ctx->gf128) gf128mul_free_4k(ctx->gf128); - ctx->gf128 = gf128mul_init_4k_lle((be128 *)key); + + BUILD_BUG_ON(sizeof(k) != GHASH_BLOCK_SIZE); + memcpy(&k, key, GHASH_BLOCK_SIZE); /* avoid violating alignment rules */ + ctx->gf128 = gf128mul_init_4k_lle(&k); + memzero_explicit(&k, GHASH_BLOCK_SIZE); + if (!ctx->gf128) return -ENOMEM; diff --git a/crypto/jitterentropy-kcapi.c b/crypto/jitterentropy-kcapi.c index 787dccca3715..701b8d86ab49 100644 --- a/crypto/jitterentropy-kcapi.c +++ b/crypto/jitterentropy-kcapi.c @@ -56,11 +56,6 @@ void jent_entropy_collector_free(struct rand_data *entropy_collector); * Helper function ***************************************************************************/ -__u64 jent_rol64(__u64 word, unsigned int shift) -{ - return rol64(word, shift); -} - void *jent_zalloc(unsigned int len) { return kzalloc(len, GFP_KERNEL); diff --git a/crypto/jitterentropy.c b/crypto/jitterentropy.c index acf44b2d2d1d..77fa2120fe0c 100644 --- a/crypto/jitterentropy.c +++ b/crypto/jitterentropy.c @@ -2,7 +2,7 @@ * Non-physical true random number generator based on timing jitter -- * Jitter RNG standalone code. * - * Copyright Stephan Mueller <smueller@chronox.de>, 2015 + * Copyright Stephan Mueller <smueller@chronox.de>, 2015 - 2019 * * Design * ====== @@ -47,7 +47,7 @@ /* * This Jitterentropy RNG is based on the jitterentropy library - * version 1.1.0 provided at http://www.chronox.de/jent.html + * version 2.1.2 provided at http://www.chronox.de/jent.html */ #ifdef __OPTIMIZE__ @@ -71,10 +71,7 @@ struct rand_data { #define DATA_SIZE_BITS ((sizeof(__u64)) * 8) __u64 last_delta; /* SENSITIVE stuck test */ __s64 last_delta2; /* SENSITIVE stuck test */ - unsigned int stuck:1; /* Time measurement stuck */ unsigned int osr; /* Oversample rate */ - unsigned int stir:1; /* Post-processing stirring */ - unsigned int disable_unbias:1; /* Deactivate Von-Neuman unbias */ #define JENT_MEMORY_BLOCKS 64 #define JENT_MEMORY_BLOCKSIZE 32 #define JENT_MEMORY_ACCESSLOOPS 128 @@ -89,8 +86,6 @@ struct rand_data { }; /* Flags that can be used to initialize the RNG */ -#define JENT_DISABLE_STIR (1<<0) /* Disable stirring the entropy pool */ -#define JENT_DISABLE_UNBIAS (1<<1) /* Disable the Von-Neuman Unbiaser */ #define JENT_DISABLE_MEMORY_ACCESS (1<<2) /* Disable memory access for more * entropy, saves MEMORY_SIZE RAM for * entropy collector */ @@ -99,19 +94,16 @@ struct rand_data { #define JENT_ENOTIME 1 /* Timer service not available */ #define JENT_ECOARSETIME 2 /* Timer too coarse for RNG */ #define JENT_ENOMONOTONIC 3 /* Timer is not monotonic increasing */ -#define JENT_EMINVARIATION 4 /* Timer variations too small for RNG */ #define JENT_EVARVAR 5 /* Timer does not produce variations of * variations (2nd derivation of time is * zero). */ -#define JENT_EMINVARVAR 6 /* Timer variations of variations is tooi - * small. */ +#define JENT_ESTUCK 8 /* Too many stuck results during init. */ /*************************************************************************** * Helper functions ***************************************************************************/ void jent_get_nstime(__u64 *out); -__u64 jent_rol64(__u64 word, unsigned int shift); void *jent_zalloc(unsigned int len); void jent_zfree(void *ptr); int jent_fips_enabled(void); @@ -140,16 +132,16 @@ static __u64 jent_loop_shuffle(struct rand_data *ec, jent_get_nstime(&time); /* - * mix the current state of the random number into the shuffle - * calculation to balance that shuffle a bit more + * Mix the current state of the random number into the shuffle + * calculation to balance that shuffle a bit more. */ if (ec) time ^= ec->data; /* - * we fold the time value as much as possible to ensure that as many - * bits of the time stamp are included as possible + * We fold the time value as much as possible to ensure that as many + * bits of the time stamp are included as possible. */ - for (i = 0; (DATA_SIZE_BITS / bits) > i; i++) { + for (i = 0; ((DATA_SIZE_BITS + bits - 1) / bits) > i; i++) { shuffle ^= time & mask; time = time >> bits; } @@ -169,38 +161,28 @@ static __u64 jent_loop_shuffle(struct rand_data *ec, * CPU Jitter noise source -- this is the noise source based on the CPU * execution time jitter * - * This function folds the time into one bit units by iterating - * through the DATA_SIZE_BITS bit time value as follows: assume our time value - * is 0xabcd - * 1st loop, 1st shift generates 0xd000 - * 1st loop, 2nd shift generates 0x000d - * 2nd loop, 1st shift generates 0xcd00 - * 2nd loop, 2nd shift generates 0x000c - * 3rd loop, 1st shift generates 0xbcd0 - * 3rd loop, 2nd shift generates 0x000b - * 4th loop, 1st shift generates 0xabcd - * 4th loop, 2nd shift generates 0x000a - * Now, the values at the end of the 2nd shifts are XORed together. + * This function injects the individual bits of the time value into the + * entropy pool using an LFSR. * - * The code is deliberately inefficient and shall stay that way. This function - * is the root cause why the code shall be compiled without optimization. This - * function not only acts as folding operation, but this function's execution - * is used to measure the CPU execution time jitter. Any change to the loop in - * this function implies that careful retesting must be done. + * The code is deliberately inefficient with respect to the bit shifting + * and shall stay that way. This function is the root cause why the code + * shall be compiled without optimization. This function not only acts as + * folding operation, but this function's execution is used to measure + * the CPU execution time jitter. Any change to the loop in this function + * implies that careful retesting must be done. * * Input: * @ec entropy collector struct -- may be NULL - * @time time stamp to be folded + * @time time stamp to be injected * @loop_cnt if a value not equal to 0 is set, use the given value as number of * loops to perform the folding * * Output: - * @folded result of folding operation + * updated ec->data * * @return Number of loops the folding operation is performed */ -static __u64 jent_fold_time(struct rand_data *ec, __u64 time, - __u64 *folded, __u64 loop_cnt) +static __u64 jent_lfsr_time(struct rand_data *ec, __u64 time, __u64 loop_cnt) { unsigned int i; __u64 j = 0; @@ -217,15 +199,34 @@ static __u64 jent_fold_time(struct rand_data *ec, __u64 time, if (loop_cnt) fold_loop_cnt = loop_cnt; for (j = 0; j < fold_loop_cnt; j++) { - new = 0; + new = ec->data; for (i = 1; (DATA_SIZE_BITS) >= i; i++) { __u64 tmp = time << (DATA_SIZE_BITS - i); tmp = tmp >> (DATA_SIZE_BITS - 1); + + /* + * Fibonacci LSFR with polynomial of + * x^64 + x^61 + x^56 + x^31 + x^28 + x^23 + 1 which is + * primitive according to + * http://poincare.matf.bg.ac.rs/~ezivkovm/publications/primpol1.pdf + * (the shift values are the polynomial values minus one + * due to counting bits from 0 to 63). As the current + * position is always the LSB, the polynomial only needs + * to shift data in from the left without wrap. + */ + tmp ^= ((new >> 63) & 1); + tmp ^= ((new >> 60) & 1); + tmp ^= ((new >> 55) & 1); + tmp ^= ((new >> 30) & 1); + tmp ^= ((new >> 27) & 1); + tmp ^= ((new >> 22) & 1); + new <<= 1; new ^= tmp; } } - *folded = new; + ec->data = new; + return fold_loop_cnt; } @@ -258,7 +259,6 @@ static __u64 jent_fold_time(struct rand_data *ec, __u64 time, */ static unsigned int jent_memaccess(struct rand_data *ec, __u64 loop_cnt) { - unsigned char *tmpval = NULL; unsigned int wrap = 0; __u64 i = 0; #define MAX_ACC_LOOP_BIT 7 @@ -278,7 +278,7 @@ static unsigned int jent_memaccess(struct rand_data *ec, __u64 loop_cnt) acc_loop_cnt = loop_cnt; for (i = 0; i < (ec->memaccessloops + acc_loop_cnt); i++) { - tmpval = ec->mem + ec->memlocation; + unsigned char *tmpval = ec->mem + ec->memlocation; /* * memory access: just add 1 to one byte, * wrap at 255 -- memory access implies read @@ -316,7 +316,7 @@ static unsigned int jent_memaccess(struct rand_data *ec, __u64 loop_cnt) * 0 jitter measurement not stuck (good bit) * 1 jitter measurement stuck (reject bit) */ -static void jent_stuck(struct rand_data *ec, __u64 current_delta) +static int jent_stuck(struct rand_data *ec, __u64 current_delta) { __s64 delta2 = ec->last_delta - current_delta; __s64 delta3 = delta2 - ec->last_delta2; @@ -325,14 +325,15 @@ static void jent_stuck(struct rand_data *ec, __u64 current_delta) ec->last_delta2 = delta2; if (!current_delta || !delta2 || !delta3) - ec->stuck = 1; + return 1; + + return 0; } /** * This is the heart of the entropy generation: calculate time deltas and - * use the CPU jitter in the time deltas. The jitter is folded into one - * bit. You can call this function the "random bit generator" as it - * produces one random bit per invocation. + * use the CPU jitter in the time deltas. The jitter is injected into the + * entropy pool. * * WARNING: ensure that ->prev_time is primed before using the output * of this function! This can be done by calling this function @@ -341,12 +342,11 @@ static void jent_stuck(struct rand_data *ec, __u64 current_delta) * Input: * @entropy_collector Reference to entropy collector * - * @return One random bit + * @return result of stuck test */ -static __u64 jent_measure_jitter(struct rand_data *ec) +static int jent_measure_jitter(struct rand_data *ec) { __u64 time = 0; - __u64 data = 0; __u64 current_delta = 0; /* Invoke one noise source before time measurement to add variations */ @@ -360,109 +360,11 @@ static __u64 jent_measure_jitter(struct rand_data *ec) current_delta = time - ec->prev_time; ec->prev_time = time; - /* Now call the next noise sources which also folds the data */ - jent_fold_time(ec, current_delta, &data, 0); - - /* - * Check whether we have a stuck measurement. The enforcement - * is performed after the stuck value has been mixed into the - * entropy pool. - */ - jent_stuck(ec, current_delta); - - return data; -} - -/** - * Von Neuman unbias as explained in RFC 4086 section 4.2. As shown in the - * documentation of that RNG, the bits from jent_measure_jitter are considered - * independent which implies that the Von Neuman unbias operation is applicable. - * A proof of the Von-Neumann unbias operation to remove skews is given in the - * document "A proposal for: Functionality classes for random number - * generators", version 2.0 by Werner Schindler, section 5.4.1. - * - * Input: - * @entropy_collector Reference to entropy collector - * - * @return One random bit - */ -static __u64 jent_unbiased_bit(struct rand_data *entropy_collector) -{ - do { - __u64 a = jent_measure_jitter(entropy_collector); - __u64 b = jent_measure_jitter(entropy_collector); - - if (a == b) - continue; - if (1 == a) - return 1; - else - return 0; - } while (1); -} - -/** - * Shuffle the pool a bit by mixing some value with a bijective function (XOR) - * into the pool. - * - * The function generates a mixer value that depends on the bits set and the - * location of the set bits in the random number generated by the entropy - * source. Therefore, based on the generated random number, this mixer value - * can have 2**64 different values. That mixer value is initialized with the - * first two SHA-1 constants. After obtaining the mixer value, it is XORed into - * the random number. - * - * The mixer value is not assumed to contain any entropy. But due to the XOR - * operation, it can also not destroy any entropy present in the entropy pool. - * - * Input: - * @entropy_collector Reference to entropy collector - */ -static void jent_stir_pool(struct rand_data *entropy_collector) -{ - /* - * to shut up GCC on 32 bit, we have to initialize the 64 variable - * with two 32 bit variables - */ - union c { - __u64 u64; - __u32 u32[2]; - }; - /* - * This constant is derived from the first two 32 bit initialization - * vectors of SHA-1 as defined in FIPS 180-4 section 5.3.1 - */ - union c constant; - /* - * The start value of the mixer variable is derived from the third - * and fourth 32 bit initialization vector of SHA-1 as defined in - * FIPS 180-4 section 5.3.1 - */ - union c mixer; - unsigned int i = 0; - - /* - * Store the SHA-1 constants in reverse order to make up the 64 bit - * value -- this applies to a little endian system, on a big endian - * system, it reverses as expected. But this really does not matter - * as we do not rely on the specific numbers. We just pick the SHA-1 - * constants as they have a good mix of bit set and unset. - */ - constant.u32[1] = 0x67452301; - constant.u32[0] = 0xefcdab89; - mixer.u32[1] = 0x98badcfe; - mixer.u32[0] = 0x10325476; + /* Now call the next noise sources which also injects the data */ + jent_lfsr_time(ec, current_delta, 0); - for (i = 0; i < DATA_SIZE_BITS; i++) { - /* - * get the i-th bit of the input random number and only XOR - * the constant into the mixer value when that bit is set - */ - if ((entropy_collector->data >> i) & 1) - mixer.u64 ^= constant.u64; - mixer.u64 = jent_rol64(mixer.u64, 1); - } - entropy_collector->data ^= mixer.u64; + /* Check whether we have a stuck measurement. */ + return jent_stuck(ec, current_delta); } /** @@ -480,48 +382,9 @@ static void jent_gen_entropy(struct rand_data *ec) jent_measure_jitter(ec); while (1) { - __u64 data = 0; - - if (ec->disable_unbias == 1) - data = jent_measure_jitter(ec); - else - data = jent_unbiased_bit(ec); - - /* enforcement of the jent_stuck test */ - if (ec->stuck) { - /* - * We only mix in the bit considered not appropriate - * without the LSFR. The reason is that if we apply - * the LSFR and we do not rotate, the 2nd bit with LSFR - * will cancel out the first LSFR application on the - * bad bit. - * - * And we do not rotate as we apply the next bit to the - * current bit location again. - */ - ec->data ^= data; - ec->stuck = 0; + /* If a stuck measurement is received, repeat measurement */ + if (jent_measure_jitter(ec)) continue; - } - - /* - * Fibonacci LSFR with polynom of - * x^64 + x^61 + x^56 + x^31 + x^28 + x^23 + 1 which is - * primitive according to - * http://poincare.matf.bg.ac.rs/~ezivkovm/publications/primpol1.pdf - * (the shift values are the polynom values minus one - * due to counting bits from 0 to 63). As the current - * position is always the LSB, the polynom only needs - * to shift data in from the left without wrap. - */ - ec->data ^= data; - ec->data ^= ((ec->data >> 63) & 1); - ec->data ^= ((ec->data >> 60) & 1); - ec->data ^= ((ec->data >> 55) & 1); - ec->data ^= ((ec->data >> 30) & 1); - ec->data ^= ((ec->data >> 27) & 1); - ec->data ^= ((ec->data >> 22) & 1); - ec->data = jent_rol64(ec->data, 1); /* * We multiply the loop value with ->osr to obtain the @@ -530,8 +393,6 @@ static void jent_gen_entropy(struct rand_data *ec) if (++k >= (DATA_SIZE_BITS * ec->osr)) break; } - if (ec->stir) - jent_stir_pool(ec); } /** @@ -639,12 +500,6 @@ struct rand_data *jent_entropy_collector_alloc(unsigned int osr, osr = 1; /* minimum sampling rate is 1 */ entropy_collector->osr = osr; - entropy_collector->stir = 1; - if (flags & JENT_DISABLE_STIR) - entropy_collector->stir = 0; - if (flags & JENT_DISABLE_UNBIAS) - entropy_collector->disable_unbias = 1; - /* fill the data pad with non-zero values */ jent_gen_entropy(entropy_collector); @@ -656,7 +511,6 @@ void jent_entropy_collector_free(struct rand_data *entropy_collector) jent_zfree(entropy_collector->mem); entropy_collector->mem = NULL; jent_zfree(entropy_collector); - entropy_collector = NULL; } int jent_entropy_init(void) @@ -665,8 +519,9 @@ int jent_entropy_init(void) __u64 delta_sum = 0; __u64 old_delta = 0; int time_backwards = 0; - int count_var = 0; int count_mod = 0; + int count_stuck = 0; + struct rand_data ec = { 0 }; /* We could perform statistical tests here, but the problem is * that we only have a few loop counts to do testing. These @@ -695,12 +550,14 @@ int jent_entropy_init(void) for (i = 0; (TESTLOOPCOUNT + CLEARCACHE) > i; i++) { __u64 time = 0; __u64 time2 = 0; - __u64 folded = 0; __u64 delta = 0; unsigned int lowdelta = 0; + int stuck; + /* Invoke core entropy collection logic */ jent_get_nstime(&time); - jent_fold_time(NULL, time, &folded, 1<<MIN_FOLD_LOOP_BIT); + ec.prev_time = time; + jent_lfsr_time(&ec, time, 0); jent_get_nstime(&time2); /* test whether timer works */ @@ -715,6 +572,8 @@ int jent_entropy_init(void) if (!delta) return JENT_ECOARSETIME; + stuck = jent_stuck(&ec, delta); + /* * up to here we did not modify any variable that will be * evaluated later, but we already performed some work. Thus we @@ -725,14 +584,14 @@ int jent_entropy_init(void) if (CLEARCACHE > i) continue; + if (stuck) + count_stuck++; + /* test whether we have an increasing timer */ if (!(time2 > time)) time_backwards++; - /* - * Avoid modulo of 64 bit integer to allow code to compile - * on 32 bit architectures. - */ + /* use 32 bit value to ensure compilation on 32 bit arches */ lowdelta = time2 - time; if (!(lowdelta % 100)) count_mod++; @@ -743,14 +602,10 @@ int jent_entropy_init(void) * only after the first loop is executed as we need to prime * the old_data value */ - if (i) { - if (delta != old_delta) - count_var++; - if (delta > old_delta) - delta_sum += (delta - old_delta); - else - delta_sum += (old_delta - delta); - } + if (delta > old_delta) + delta_sum += (delta - old_delta); + else + delta_sum += (old_delta - delta); old_delta = delta; } @@ -763,25 +618,29 @@ int jent_entropy_init(void) */ if (3 < time_backwards) return JENT_ENOMONOTONIC; - /* Error if the time variances are always identical */ - if (!delta_sum) - return JENT_EVARVAR; /* * Variations of deltas of time must on average be larger * than 1 to ensure the entropy estimation * implied with 1 is preserved */ - if (delta_sum <= 1) - return JENT_EMINVARVAR; + if ((delta_sum) <= 1) + return JENT_EVARVAR; /* * Ensure that we have variations in the time stamp below 10 for at - * least 10% of all checks -- on some platforms, the counter - * increments in multiples of 100, but not always + * least 10% of all checks -- on some platforms, the counter increments + * in multiples of 100, but not always */ if ((TESTLOOPCOUNT/10 * 9) < count_mod) return JENT_ECOARSETIME; + /* + * If we have more than 90% stuck results, then this Jitter RNG is + * likely to not work well. + */ + if ((TESTLOOPCOUNT/10 * 9) < count_stuck) + return JENT_ESTUCK; + return 0; } diff --git a/crypto/khazad.c b/crypto/khazad.c index b50aa8a3ab4c..14ca7f1631c7 100644 --- a/crypto/khazad.c +++ b/crypto/khazad.c @@ -848,6 +848,7 @@ static void khazad_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) static struct crypto_alg khazad_alg = { .cra_name = "khazad", + .cra_driver_name = "khazad-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = KHAZAD_BLOCK_SIZE, .cra_ctxsize = sizeof (struct khazad_ctx), diff --git a/crypto/lrw.c b/crypto/lrw.c index 58009cf63a6e..be829f6afc8e 100644 --- a/crypto/lrw.c +++ b/crypto/lrw.c @@ -384,7 +384,7 @@ static int create(struct crypto_template *tmpl, struct rtattr **tb) inst->alg.base.cra_priority = alg->base.cra_priority; inst->alg.base.cra_blocksize = LRW_BLOCK_SIZE; inst->alg.base.cra_alignmask = alg->base.cra_alignmask | - (__alignof__(__be32) - 1); + (__alignof__(be128) - 1); inst->alg.ivsize = LRW_BLOCK_SIZE; inst->alg.min_keysize = crypto_skcipher_alg_min_keysize(alg) + diff --git a/crypto/lz4.c b/crypto/lz4.c index 54673cf88385..0606f8862e78 100644 --- a/crypto/lz4.c +++ b/crypto/lz4.c @@ -106,6 +106,7 @@ static int lz4_decompress_crypto(struct crypto_tfm *tfm, const u8 *src, static struct crypto_alg alg_lz4 = { .cra_name = "lz4", + .cra_driver_name = "lz4-generic", .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, .cra_ctxsize = sizeof(struct lz4_ctx), .cra_module = THIS_MODULE, diff --git a/crypto/lz4hc.c b/crypto/lz4hc.c index daae7bfb385d..d7cc94aa2fcf 100644 --- a/crypto/lz4hc.c +++ b/crypto/lz4hc.c @@ -107,6 +107,7 @@ static int lz4hc_decompress_crypto(struct crypto_tfm *tfm, const u8 *src, static struct crypto_alg alg_lz4hc = { .cra_name = "lz4hc", + .cra_driver_name = "lz4hc-generic", .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, .cra_ctxsize = sizeof(struct lz4hc_ctx), .cra_module = THIS_MODULE, diff --git a/crypto/lzo-rle.c b/crypto/lzo-rle.c index c4303e96f2b1..0631d975bfac 100644 --- a/crypto/lzo-rle.c +++ b/crypto/lzo-rle.c @@ -109,6 +109,7 @@ static int lzorle_sdecompress(struct crypto_scomp *tfm, const u8 *src, static struct crypto_alg alg = { .cra_name = "lzo-rle", + .cra_driver_name = "lzo-rle-generic", .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, .cra_ctxsize = sizeof(struct lzorle_ctx), .cra_module = THIS_MODULE, diff --git a/crypto/lzo.c b/crypto/lzo.c index 97051a2ca08e..ebda132dd22b 100644 --- a/crypto/lzo.c +++ b/crypto/lzo.c @@ -109,6 +109,7 @@ static int lzo_sdecompress(struct crypto_scomp *tfm, const u8 *src, static struct crypto_alg alg = { .cra_name = "lzo", + .cra_driver_name = "lzo-generic", .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, .cra_ctxsize = sizeof(struct lzo_ctx), .cra_module = THIS_MODULE, diff --git a/crypto/md4.c b/crypto/md4.c index 9a1a228a0c69..2e7f2f319f95 100644 --- a/crypto/md4.c +++ b/crypto/md4.c @@ -216,9 +216,10 @@ static struct shash_alg alg = { .final = md4_final, .descsize = sizeof(struct md4_ctx), .base = { - .cra_name = "md4", - .cra_blocksize = MD4_HMAC_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "md4", + .cra_driver_name = "md4-generic", + .cra_blocksize = MD4_HMAC_BLOCK_SIZE, + .cra_module = THIS_MODULE, } }; diff --git a/crypto/md5.c b/crypto/md5.c index 221c2c0932f8..22dc60bc0437 100644 --- a/crypto/md5.c +++ b/crypto/md5.c @@ -228,9 +228,10 @@ static struct shash_alg alg = { .descsize = sizeof(struct md5_state), .statesize = sizeof(struct md5_state), .base = { - .cra_name = "md5", - .cra_blocksize = MD5_HMAC_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "md5", + .cra_driver_name = "md5-generic", + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_module = THIS_MODULE, } }; diff --git a/crypto/michael_mic.c b/crypto/michael_mic.c index b3d83ff709d3..20e6220f46f6 100644 --- a/crypto/michael_mic.c +++ b/crypto/michael_mic.c @@ -156,6 +156,7 @@ static struct shash_alg alg = { .descsize = sizeof(struct michael_mic_desc_ctx), .base = { .cra_name = "michael_mic", + .cra_driver_name = "michael_mic-generic", .cra_blocksize = 8, .cra_alignmask = 3, .cra_ctxsize = sizeof(struct michael_mic_ctx), diff --git a/crypto/rmd128.c b/crypto/rmd128.c index d6c031a9fd14..29308fb97e7e 100644 --- a/crypto/rmd128.c +++ b/crypto/rmd128.c @@ -298,6 +298,7 @@ static struct shash_alg alg = { .descsize = sizeof(struct rmd128_ctx), .base = { .cra_name = "rmd128", + .cra_driver_name = "rmd128-generic", .cra_blocksize = RMD128_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/crypto/rmd160.c b/crypto/rmd160.c index f3add4d54a22..c5fe4034b153 100644 --- a/crypto/rmd160.c +++ b/crypto/rmd160.c @@ -342,6 +342,7 @@ static struct shash_alg alg = { .descsize = sizeof(struct rmd160_ctx), .base = { .cra_name = "rmd160", + .cra_driver_name = "rmd160-generic", .cra_blocksize = RMD160_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/crypto/rmd256.c b/crypto/rmd256.c index 79ca3029848f..3c730e9de5fd 100644 --- a/crypto/rmd256.c +++ b/crypto/rmd256.c @@ -317,6 +317,7 @@ static struct shash_alg alg = { .descsize = sizeof(struct rmd256_ctx), .base = { .cra_name = "rmd256", + .cra_driver_name = "rmd256-generic", .cra_blocksize = RMD256_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/crypto/rmd320.c b/crypto/rmd320.c index b2392ef7467b..c919ad6c4705 100644 --- a/crypto/rmd320.c +++ b/crypto/rmd320.c @@ -366,6 +366,7 @@ static struct shash_alg alg = { .descsize = sizeof(struct rmd320_ctx), .base = { .cra_name = "rmd320", + .cra_driver_name = "rmd320-generic", .cra_blocksize = RMD320_BLOCK_SIZE, .cra_module = THIS_MODULE, } diff --git a/crypto/serpent_generic.c b/crypto/serpent_generic.c index 16f612b6dbca..56fa665a4f01 100644 --- a/crypto/serpent_generic.c +++ b/crypto/serpent_generic.c @@ -225,7 +225,13 @@ x4 ^= x2; \ }) -static void __serpent_setkey_sbox(u32 r0, u32 r1, u32 r2, u32 r3, u32 r4, u32 *k) +/* + * both gcc and clang have misoptimized this function in the past, + * producing horrible object code from spilling temporary variables + * on the stack. Forcing this part out of line avoids that. + */ +static noinline void __serpent_setkey_sbox(u32 r0, u32 r1, u32 r2, + u32 r3, u32 r4, u32 *k) { k += 100; S3(r3, r4, r0, r1, r2); store_and_load_keys(r1, r2, r4, r3, 28, 24); @@ -637,6 +643,7 @@ static struct crypto_alg srp_algs[2] = { { .cia_decrypt = serpent_decrypt } } }, { .cra_name = "tnepres", + .cra_driver_name = "tnepres-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = SERPENT_BLOCK_SIZE, .cra_ctxsize = sizeof(struct serpent_ctx), diff --git a/crypto/skcipher.c b/crypto/skcipher.c index df735148000f..5d836fc3df3e 100644 --- a/crypto/skcipher.c +++ b/crypto/skcipher.c @@ -837,6 +837,40 @@ static int skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, return 0; } +int crypto_skcipher_encrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct crypto_alg *alg = tfm->base.__crt_alg; + unsigned int cryptlen = req->cryptlen; + int ret; + + crypto_stats_get(alg); + if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) + ret = -ENOKEY; + else + ret = tfm->encrypt(req); + crypto_stats_skcipher_encrypt(cryptlen, ret, alg); + return ret; +} +EXPORT_SYMBOL_GPL(crypto_skcipher_encrypt); + +int crypto_skcipher_decrypt(struct skcipher_request *req) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + struct crypto_alg *alg = tfm->base.__crt_alg; + unsigned int cryptlen = req->cryptlen; + int ret; + + crypto_stats_get(alg); + if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) + ret = -ENOKEY; + else + ret = tfm->decrypt(req); + crypto_stats_skcipher_decrypt(cryptlen, ret, alg); + return ret; +} +EXPORT_SYMBOL_GPL(crypto_skcipher_decrypt); + static void crypto_skcipher_exit_tfm(struct crypto_tfm *tfm) { struct crypto_skcipher *skcipher = __crypto_skcipher_cast(tfm); diff --git a/crypto/tea.c b/crypto/tea.c index 37a18a9be2f4..02efc5d81690 100644 --- a/crypto/tea.c +++ b/crypto/tea.c @@ -216,6 +216,7 @@ static void xeta_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) static struct crypto_alg tea_algs[3] = { { .cra_name = "tea", + .cra_driver_name = "tea-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = TEA_BLOCK_SIZE, .cra_ctxsize = sizeof (struct tea_ctx), @@ -229,6 +230,7 @@ static struct crypto_alg tea_algs[3] = { { .cia_decrypt = tea_decrypt } } }, { .cra_name = "xtea", + .cra_driver_name = "xtea-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = XTEA_BLOCK_SIZE, .cra_ctxsize = sizeof (struct xtea_ctx), @@ -242,6 +244,7 @@ static struct crypto_alg tea_algs[3] = { { .cia_decrypt = xtea_decrypt } } }, { .cra_name = "xeta", + .cra_driver_name = "xeta-generic", .cra_flags = CRYPTO_ALG_TYPE_CIPHER, .cra_blocksize = XTEA_BLOCK_SIZE, .cra_ctxsize = sizeof (struct xtea_ctx), diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 658a7eeebab2..d0b5b33806a6 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -1032,6 +1032,205 @@ static void crypto_reenable_simd_for_test(void) } #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ +static int build_hash_sglist(struct test_sglist *tsgl, + const struct hash_testvec *vec, + const struct testvec_config *cfg, + unsigned int alignmask, + const struct test_sg_division *divs[XBUFSIZE]) +{ + struct kvec kv; + struct iov_iter input; + + kv.iov_base = (void *)vec->plaintext; + kv.iov_len = vec->psize; + iov_iter_kvec(&input, WRITE, &kv, 1, vec->psize); + return build_test_sglist(tsgl, cfg->src_divs, alignmask, vec->psize, + &input, divs); +} + +static int check_hash_result(const char *type, + const u8 *result, unsigned int digestsize, + const struct hash_testvec *vec, + const char *vec_name, + const char *driver, + const struct testvec_config *cfg) +{ + if (memcmp(result, vec->digest, digestsize) != 0) { + pr_err("alg: %s: %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n", + type, driver, vec_name, cfg->name); + return -EINVAL; + } + if (!testmgr_is_poison(&result[digestsize], TESTMGR_POISON_LEN)) { + pr_err("alg: %s: %s overran result buffer on test vector %s, cfg=\"%s\"\n", + type, driver, vec_name, cfg->name); + return -EOVERFLOW; + } + return 0; +} + +static inline int check_shash_op(const char *op, int err, + const char *driver, const char *vec_name, + const struct testvec_config *cfg) +{ + if (err) + pr_err("alg: shash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n", + driver, op, err, vec_name, cfg->name); + return err; +} + +static inline const void *sg_data(struct scatterlist *sg) +{ + return page_address(sg_page(sg)) + sg->offset; +} + +/* Test one hash test vector in one configuration, using the shash API */ +static int test_shash_vec_cfg(const char *driver, + const struct hash_testvec *vec, + const char *vec_name, + const struct testvec_config *cfg, + struct shash_desc *desc, + struct test_sglist *tsgl, + u8 *hashstate) +{ + struct crypto_shash *tfm = desc->tfm; + const unsigned int alignmask = crypto_shash_alignmask(tfm); + const unsigned int digestsize = crypto_shash_digestsize(tfm); + const unsigned int statesize = crypto_shash_statesize(tfm); + const struct test_sg_division *divs[XBUFSIZE]; + unsigned int i; + u8 result[HASH_MAX_DIGESTSIZE + TESTMGR_POISON_LEN]; + int err; + + /* Set the key, if specified */ + if (vec->ksize) { + err = crypto_shash_setkey(tfm, vec->key, vec->ksize); + if (err) { + if (err == vec->setkey_error) + return 0; + pr_err("alg: shash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n", + driver, vec_name, vec->setkey_error, err, + crypto_shash_get_flags(tfm)); + return err; + } + if (vec->setkey_error) { + pr_err("alg: shash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n", + driver, vec_name, vec->setkey_error); + return -EINVAL; + } + } + + /* Build the scatterlist for the source data */ + err = build_hash_sglist(tsgl, vec, cfg, alignmask, divs); + if (err) { + pr_err("alg: shash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n", + driver, vec_name, cfg->name); + return err; + } + + /* Do the actual hashing */ + + testmgr_poison(desc->__ctx, crypto_shash_descsize(tfm)); + testmgr_poison(result, digestsize + TESTMGR_POISON_LEN); + + if (cfg->finalization_type == FINALIZATION_TYPE_DIGEST || + vec->digest_error) { + /* Just using digest() */ + if (tsgl->nents != 1) + return 0; + if (cfg->nosimd) + crypto_disable_simd_for_test(); + err = crypto_shash_digest(desc, sg_data(&tsgl->sgl[0]), + tsgl->sgl[0].length, result); + if (cfg->nosimd) + crypto_reenable_simd_for_test(); + if (err) { + if (err == vec->digest_error) + return 0; + pr_err("alg: shash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n", + driver, vec_name, vec->digest_error, err, + cfg->name); + return err; + } + if (vec->digest_error) { + pr_err("alg: shash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n", + driver, vec_name, vec->digest_error, cfg->name); + return -EINVAL; + } + goto result_ready; + } + + /* Using init(), zero or more update(), then final() or finup() */ + + if (cfg->nosimd) + crypto_disable_simd_for_test(); + err = crypto_shash_init(desc); + if (cfg->nosimd) + crypto_reenable_simd_for_test(); + err = check_shash_op("init", err, driver, vec_name, cfg); + if (err) + return err; + + for (i = 0; i < tsgl->nents; i++) { + if (i + 1 == tsgl->nents && + cfg->finalization_type == FINALIZATION_TYPE_FINUP) { + if (divs[i]->nosimd) + crypto_disable_simd_for_test(); + err = crypto_shash_finup(desc, sg_data(&tsgl->sgl[i]), + tsgl->sgl[i].length, result); + if (divs[i]->nosimd) + crypto_reenable_simd_for_test(); + err = check_shash_op("finup", err, driver, vec_name, + cfg); + if (err) + return err; + goto result_ready; + } + if (divs[i]->nosimd) + crypto_disable_simd_for_test(); + err = crypto_shash_update(desc, sg_data(&tsgl->sgl[i]), + tsgl->sgl[i].length); + if (divs[i]->nosimd) + crypto_reenable_simd_for_test(); + err = check_shash_op("update", err, driver, vec_name, cfg); + if (err) + return err; + if (divs[i]->flush_type == FLUSH_TYPE_REIMPORT) { + /* Test ->export() and ->import() */ + testmgr_poison(hashstate + statesize, + TESTMGR_POISON_LEN); + err = crypto_shash_export(desc, hashstate); + err = check_shash_op("export", err, driver, vec_name, + cfg); + if (err) + return err; + if (!testmgr_is_poison(hashstate + statesize, + TESTMGR_POISON_LEN)) { + pr_err("alg: shash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n", + driver, vec_name, cfg->name); + return -EOVERFLOW; + } + testmgr_poison(desc->__ctx, crypto_shash_descsize(tfm)); + err = crypto_shash_import(desc, hashstate); + err = check_shash_op("import", err, driver, vec_name, + cfg); + if (err) + return err; + } + } + + if (cfg->nosimd) + crypto_disable_simd_for_test(); + err = crypto_shash_final(desc, result); + if (cfg->nosimd) + crypto_reenable_simd_for_test(); + err = check_shash_op("final", err, driver, vec_name, cfg); + if (err) + return err; +result_ready: + return check_hash_result("shash", result, digestsize, vec, vec_name, + driver, cfg); +} + static int do_ahash_op(int (*op)(struct ahash_request *req), struct ahash_request *req, struct crypto_wait *wait, bool nosimd) @@ -1049,31 +1248,32 @@ static int do_ahash_op(int (*op)(struct ahash_request *req), return crypto_wait_req(err, wait); } -static int check_nonfinal_hash_op(const char *op, int err, - u8 *result, unsigned int digestsize, - const char *driver, const char *vec_name, - const struct testvec_config *cfg) +static int check_nonfinal_ahash_op(const char *op, int err, + u8 *result, unsigned int digestsize, + const char *driver, const char *vec_name, + const struct testvec_config *cfg) { if (err) { - pr_err("alg: hash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n", + pr_err("alg: ahash: %s %s() failed with err %d on test vector %s, cfg=\"%s\"\n", driver, op, err, vec_name, cfg->name); return err; } if (!testmgr_is_poison(result, digestsize)) { - pr_err("alg: hash: %s %s() used result buffer on test vector %s, cfg=\"%s\"\n", + pr_err("alg: ahash: %s %s() used result buffer on test vector %s, cfg=\"%s\"\n", driver, op, vec_name, cfg->name); return -EINVAL; } return 0; } -static int test_hash_vec_cfg(const char *driver, - const struct hash_testvec *vec, - const char *vec_name, - const struct testvec_config *cfg, - struct ahash_request *req, - struct test_sglist *tsgl, - u8 *hashstate) +/* Test one hash test vector in one configuration, using the ahash API */ +static int test_ahash_vec_cfg(const char *driver, + const struct hash_testvec *vec, + const char *vec_name, + const struct testvec_config *cfg, + struct ahash_request *req, + struct test_sglist *tsgl, + u8 *hashstate) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); const unsigned int alignmask = crypto_ahash_alignmask(tfm); @@ -1082,8 +1282,6 @@ static int test_hash_vec_cfg(const char *driver, const u32 req_flags = CRYPTO_TFM_REQ_MAY_BACKLOG | cfg->req_flags; const struct test_sg_division *divs[XBUFSIZE]; DECLARE_CRYPTO_WAIT(wait); - struct kvec _input; - struct iov_iter input; unsigned int i; struct scatterlist *pending_sgl; unsigned int pending_len; @@ -1096,26 +1294,22 @@ static int test_hash_vec_cfg(const char *driver, if (err) { if (err == vec->setkey_error) return 0; - pr_err("alg: hash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n", + pr_err("alg: ahash: %s setkey failed on test vector %s; expected_error=%d, actual_error=%d, flags=%#x\n", driver, vec_name, vec->setkey_error, err, crypto_ahash_get_flags(tfm)); return err; } if (vec->setkey_error) { - pr_err("alg: hash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n", + pr_err("alg: ahash: %s setkey unexpectedly succeeded on test vector %s; expected_error=%d\n", driver, vec_name, vec->setkey_error); return -EINVAL; } } /* Build the scatterlist for the source data */ - _input.iov_base = (void *)vec->plaintext; - _input.iov_len = vec->psize; - iov_iter_kvec(&input, WRITE, &_input, 1, vec->psize); - err = build_test_sglist(tsgl, cfg->src_divs, alignmask, vec->psize, - &input, divs); + err = build_hash_sglist(tsgl, vec, cfg, alignmask, divs); if (err) { - pr_err("alg: hash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n", + pr_err("alg: ahash: %s: error preparing scatterlist for test vector %s, cfg=\"%s\"\n", driver, vec_name, cfg->name); return err; } @@ -1135,13 +1329,13 @@ static int test_hash_vec_cfg(const char *driver, if (err) { if (err == vec->digest_error) return 0; - pr_err("alg: hash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n", + pr_err("alg: ahash: %s digest() failed on test vector %s; expected_error=%d, actual_error=%d, cfg=\"%s\"\n", driver, vec_name, vec->digest_error, err, cfg->name); return err; } if (vec->digest_error) { - pr_err("alg: hash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n", + pr_err("alg: ahash: %s digest() unexpectedly succeeded on test vector %s; expected_error=%d, cfg=\"%s\"\n", driver, vec_name, vec->digest_error, cfg->name); return -EINVAL; } @@ -1153,8 +1347,8 @@ static int test_hash_vec_cfg(const char *driver, ahash_request_set_callback(req, req_flags, crypto_req_done, &wait); ahash_request_set_crypt(req, NULL, result, 0); err = do_ahash_op(crypto_ahash_init, req, &wait, cfg->nosimd); - err = check_nonfinal_hash_op("init", err, result, digestsize, - driver, vec_name, cfg); + err = check_nonfinal_ahash_op("init", err, result, digestsize, + driver, vec_name, cfg); if (err) return err; @@ -1170,9 +1364,9 @@ static int test_hash_vec_cfg(const char *driver, pending_len); err = do_ahash_op(crypto_ahash_update, req, &wait, divs[i]->nosimd); - err = check_nonfinal_hash_op("update", err, - result, digestsize, - driver, vec_name, cfg); + err = check_nonfinal_ahash_op("update", err, + result, digestsize, + driver, vec_name, cfg); if (err) return err; pending_sgl = NULL; @@ -1183,23 +1377,23 @@ static int test_hash_vec_cfg(const char *driver, testmgr_poison(hashstate + statesize, TESTMGR_POISON_LEN); err = crypto_ahash_export(req, hashstate); - err = check_nonfinal_hash_op("export", err, - result, digestsize, - driver, vec_name, cfg); + err = check_nonfinal_ahash_op("export", err, + result, digestsize, + driver, vec_name, cfg); if (err) return err; if (!testmgr_is_poison(hashstate + statesize, TESTMGR_POISON_LEN)) { - pr_err("alg: hash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n", + pr_err("alg: ahash: %s export() overran state buffer on test vector %s, cfg=\"%s\"\n", driver, vec_name, cfg->name); return -EOVERFLOW; } testmgr_poison(req->__ctx, crypto_ahash_reqsize(tfm)); err = crypto_ahash_import(req, hashstate); - err = check_nonfinal_hash_op("import", err, - result, digestsize, - driver, vec_name, cfg); + err = check_nonfinal_ahash_op("import", err, + result, digestsize, + driver, vec_name, cfg); if (err) return err; } @@ -1213,13 +1407,13 @@ static int test_hash_vec_cfg(const char *driver, if (cfg->finalization_type == FINALIZATION_TYPE_FINAL) { /* finish with update() and final() */ err = do_ahash_op(crypto_ahash_update, req, &wait, cfg->nosimd); - err = check_nonfinal_hash_op("update", err, result, digestsize, - driver, vec_name, cfg); + err = check_nonfinal_ahash_op("update", err, result, digestsize, + driver, vec_name, cfg); if (err) return err; err = do_ahash_op(crypto_ahash_final, req, &wait, cfg->nosimd); if (err) { - pr_err("alg: hash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n", + pr_err("alg: ahash: %s final() failed with err %d on test vector %s, cfg=\"%s\"\n", driver, err, vec_name, cfg->name); return err; } @@ -1227,31 +1421,49 @@ static int test_hash_vec_cfg(const char *driver, /* finish with finup() */ err = do_ahash_op(crypto_ahash_finup, req, &wait, cfg->nosimd); if (err) { - pr_err("alg: hash: %s finup() failed with err %d on test vector %s, cfg=\"%s\"\n", + pr_err("alg: ahash: %s finup() failed with err %d on test vector %s, cfg=\"%s\"\n", driver, err, vec_name, cfg->name); return err; } } result_ready: - /* Check that the algorithm produced the correct digest */ - if (memcmp(result, vec->digest, digestsize) != 0) { - pr_err("alg: hash: %s test failed (wrong result) on test vector %s, cfg=\"%s\"\n", - driver, vec_name, cfg->name); - return -EINVAL; - } - if (!testmgr_is_poison(&result[digestsize], TESTMGR_POISON_LEN)) { - pr_err("alg: hash: %s overran result buffer on test vector %s, cfg=\"%s\"\n", - driver, vec_name, cfg->name); - return -EOVERFLOW; + return check_hash_result("ahash", result, digestsize, vec, vec_name, + driver, cfg); +} + +static int test_hash_vec_cfg(const char *driver, + const struct hash_testvec *vec, + const char *vec_name, + const struct testvec_config *cfg, + struct ahash_request *req, + struct shash_desc *desc, + struct test_sglist *tsgl, + u8 *hashstate) +{ + int err; + + /* + * For algorithms implemented as "shash", most bugs will be detected by + * both the shash and ahash tests. Test the shash API first so that the + * failures involve less indirection, so are easier to debug. + */ + + if (desc) { + err = test_shash_vec_cfg(driver, vec, vec_name, cfg, desc, tsgl, + hashstate); + if (err) + return err; } - return 0; + return test_ahash_vec_cfg(driver, vec, vec_name, cfg, req, tsgl, + hashstate); } static int test_hash_vec(const char *driver, const struct hash_testvec *vec, unsigned int vec_num, struct ahash_request *req, - struct test_sglist *tsgl, u8 *hashstate) + struct shash_desc *desc, struct test_sglist *tsgl, + u8 *hashstate) { char vec_name[16]; unsigned int i; @@ -1262,7 +1474,7 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, for (i = 0; i < ARRAY_SIZE(default_hash_testvec_configs); i++) { err = test_hash_vec_cfg(driver, vec, vec_name, &default_hash_testvec_configs[i], - req, tsgl, hashstate); + req, desc, tsgl, hashstate); if (err) return err; } @@ -1276,9 +1488,10 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); err = test_hash_vec_cfg(driver, vec, vec_name, &cfg, - req, tsgl, hashstate); + req, desc, tsgl, hashstate); if (err) return err; + cond_resched(); } } #endif @@ -1290,14 +1503,12 @@ static int test_hash_vec(const char *driver, const struct hash_testvec *vec, * Generate a hash test vector from the given implementation. * Assumes the buffers in 'vec' were already allocated. */ -static void generate_random_hash_testvec(struct crypto_shash *tfm, +static void generate_random_hash_testvec(struct shash_desc *desc, struct hash_testvec *vec, unsigned int maxkeysize, unsigned int maxdatasize, char *name, size_t max_namelen) { - SHASH_DESC_ON_STACK(desc, tfm); - /* Data */ vec->psize = generate_random_length(maxdatasize); generate_random_bytes((u8 *)vec->plaintext, vec->psize); @@ -1314,7 +1525,7 @@ static void generate_random_hash_testvec(struct crypto_shash *tfm, vec->ksize = 1 + (prandom_u32() % maxkeysize); generate_random_bytes((u8 *)vec->key, vec->ksize); - vec->setkey_error = crypto_shash_setkey(tfm, vec->key, + vec->setkey_error = crypto_shash_setkey(desc->tfm, vec->key, vec->ksize); /* If the key couldn't be set, no need to continue to digest. */ if (vec->setkey_error) @@ -1322,7 +1533,6 @@ static void generate_random_hash_testvec(struct crypto_shash *tfm, } /* Digest */ - desc->tfm = tfm; vec->digest_error = crypto_shash_digest(desc, vec->plaintext, vec->psize, (u8 *)vec->digest); done: @@ -1338,6 +1548,7 @@ static int test_hash_vs_generic_impl(const char *driver, const char *generic_driver, unsigned int maxkeysize, struct ahash_request *req, + struct shash_desc *desc, struct test_sglist *tsgl, u8 *hashstate) { @@ -1348,10 +1559,11 @@ static int test_hash_vs_generic_impl(const char *driver, const char *algname = crypto_hash_alg_common(tfm)->base.cra_name; char _generic_driver[CRYPTO_MAX_ALG_NAME]; struct crypto_shash *generic_tfm = NULL; + struct shash_desc *generic_desc = NULL; unsigned int i; struct hash_testvec vec = { 0 }; char vec_name[64]; - struct testvec_config cfg; + struct testvec_config *cfg; char cfgname[TESTVEC_CONFIG_NAMELEN]; int err; @@ -1381,6 +1593,20 @@ static int test_hash_vs_generic_impl(const char *driver, return err; } + cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); + if (!cfg) { + err = -ENOMEM; + goto out; + } + + generic_desc = kzalloc(sizeof(*desc) + + crypto_shash_descsize(generic_tfm), GFP_KERNEL); + if (!generic_desc) { + err = -ENOMEM; + goto out; + } + generic_desc->tfm = generic_tfm; + /* Check the algorithm properties for consistency. */ if (digestsize != crypto_shash_digestsize(generic_tfm)) { @@ -1412,23 +1638,25 @@ static int test_hash_vs_generic_impl(const char *driver, } for (i = 0; i < fuzz_iterations * 8; i++) { - generate_random_hash_testvec(generic_tfm, &vec, + generate_random_hash_testvec(generic_desc, &vec, maxkeysize, maxdatasize, vec_name, sizeof(vec_name)); - generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); + generate_random_testvec_config(cfg, cfgname, sizeof(cfgname)); - err = test_hash_vec_cfg(driver, &vec, vec_name, &cfg, - req, tsgl, hashstate); + err = test_hash_vec_cfg(driver, &vec, vec_name, cfg, + req, desc, tsgl, hashstate); if (err) goto out; cond_resched(); } err = 0; out: + kfree(cfg); kfree(vec.key); kfree(vec.plaintext); kfree(vec.digest); crypto_free_shash(generic_tfm); + kzfree(generic_desc); return err; } #else /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ @@ -1436,6 +1664,7 @@ static int test_hash_vs_generic_impl(const char *driver, const char *generic_driver, unsigned int maxkeysize, struct ahash_request *req, + struct shash_desc *desc, struct test_sglist *tsgl, u8 *hashstate) { @@ -1443,26 +1672,67 @@ static int test_hash_vs_generic_impl(const char *driver, } #endif /* !CONFIG_CRYPTO_MANAGER_EXTRA_TESTS */ +static int alloc_shash(const char *driver, u32 type, u32 mask, + struct crypto_shash **tfm_ret, + struct shash_desc **desc_ret) +{ + struct crypto_shash *tfm; + struct shash_desc *desc; + + tfm = crypto_alloc_shash(driver, type, mask); + if (IS_ERR(tfm)) { + if (PTR_ERR(tfm) == -ENOENT) { + /* + * This algorithm is only available through the ahash + * API, not the shash API, so skip the shash tests. + */ + return 0; + } + pr_err("alg: hash: failed to allocate shash transform for %s: %ld\n", + driver, PTR_ERR(tfm)); + return PTR_ERR(tfm); + } + + desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(tfm), GFP_KERNEL); + if (!desc) { + crypto_free_shash(tfm); + return -ENOMEM; + } + desc->tfm = tfm; + + *tfm_ret = tfm; + *desc_ret = desc; + return 0; +} + static int __alg_test_hash(const struct hash_testvec *vecs, unsigned int num_vecs, const char *driver, u32 type, u32 mask, const char *generic_driver, unsigned int maxkeysize) { - struct crypto_ahash *tfm; + struct crypto_ahash *atfm = NULL; struct ahash_request *req = NULL; + struct crypto_shash *stfm = NULL; + struct shash_desc *desc = NULL; struct test_sglist *tsgl = NULL; u8 *hashstate = NULL; + unsigned int statesize; unsigned int i; int err; - tfm = crypto_alloc_ahash(driver, type, mask); - if (IS_ERR(tfm)) { + /* + * Always test the ahash API. This works regardless of whether the + * algorithm is implemented as ahash or shash. + */ + + atfm = crypto_alloc_ahash(driver, type, mask); + if (IS_ERR(atfm)) { pr_err("alg: hash: failed to allocate transform for %s: %ld\n", - driver, PTR_ERR(tfm)); - return PTR_ERR(tfm); + driver, PTR_ERR(atfm)); + return PTR_ERR(atfm); } - req = ahash_request_alloc(tfm, GFP_KERNEL); + req = ahash_request_alloc(atfm, GFP_KERNEL); if (!req) { pr_err("alg: hash: failed to allocate request for %s\n", driver); @@ -1470,6 +1740,14 @@ static int __alg_test_hash(const struct hash_testvec *vecs, goto out; } + /* + * If available also test the shash API, to cover corner cases that may + * be missed by testing the ahash API only. + */ + err = alloc_shash(driver, type, mask, &stfm, &desc); + if (err) + goto out; + tsgl = kmalloc(sizeof(*tsgl), GFP_KERNEL); if (!tsgl || init_test_sglist(tsgl) != 0) { pr_err("alg: hash: failed to allocate test buffers for %s\n", @@ -1480,8 +1758,10 @@ static int __alg_test_hash(const struct hash_testvec *vecs, goto out; } - hashstate = kmalloc(crypto_ahash_statesize(tfm) + TESTMGR_POISON_LEN, - GFP_KERNEL); + statesize = crypto_ahash_statesize(atfm); + if (stfm) + statesize = max(statesize, crypto_shash_statesize(stfm)); + hashstate = kmalloc(statesize + TESTMGR_POISON_LEN, GFP_KERNEL); if (!hashstate) { pr_err("alg: hash: failed to allocate hash state buffer for %s\n", driver); @@ -1490,20 +1770,24 @@ static int __alg_test_hash(const struct hash_testvec *vecs, } for (i = 0; i < num_vecs; i++) { - err = test_hash_vec(driver, &vecs[i], i, req, tsgl, hashstate); + err = test_hash_vec(driver, &vecs[i], i, req, desc, tsgl, + hashstate); if (err) goto out; + cond_resched(); } err = test_hash_vs_generic_impl(driver, generic_driver, maxkeysize, req, - tsgl, hashstate); + desc, tsgl, hashstate); out: kfree(hashstate); if (tsgl) { destroy_test_sglist(tsgl); kfree(tsgl); } + kfree(desc); + crypto_free_shash(stfm); ahash_request_free(req); - crypto_free_ahash(tfm); + crypto_free_ahash(atfm); return err; } @@ -1755,6 +2039,7 @@ static int test_aead_vec(const char *driver, int enc, &cfg, req, tsgls); if (err) return err; + cond_resched(); } } #endif @@ -1864,7 +2149,7 @@ static int test_aead_vs_generic_impl(const char *driver, unsigned int i; struct aead_testvec vec = { 0 }; char vec_name[64]; - struct testvec_config cfg; + struct testvec_config *cfg; char cfgname[TESTVEC_CONFIG_NAMELEN]; int err; @@ -1894,6 +2179,12 @@ static int test_aead_vs_generic_impl(const char *driver, return err; } + cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); + if (!cfg) { + err = -ENOMEM; + goto out; + } + generic_req = aead_request_alloc(generic_tfm, GFP_KERNEL); if (!generic_req) { err = -ENOMEM; @@ -1948,13 +2239,13 @@ static int test_aead_vs_generic_impl(const char *driver, generate_random_aead_testvec(generic_req, &vec, maxkeysize, maxdatasize, vec_name, sizeof(vec_name)); - generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); + generate_random_testvec_config(cfg, cfgname, sizeof(cfgname)); - err = test_aead_vec_cfg(driver, ENCRYPT, &vec, vec_name, &cfg, + err = test_aead_vec_cfg(driver, ENCRYPT, &vec, vec_name, cfg, req, tsgls); if (err) goto out; - err = test_aead_vec_cfg(driver, DECRYPT, &vec, vec_name, &cfg, + err = test_aead_vec_cfg(driver, DECRYPT, &vec, vec_name, cfg, req, tsgls); if (err) goto out; @@ -1962,6 +2253,7 @@ static int test_aead_vs_generic_impl(const char *driver, } err = 0; out: + kfree(cfg); kfree(vec.key); kfree(vec.iv); kfree(vec.assoc); @@ -1994,6 +2286,7 @@ static int test_aead(const char *driver, int enc, tsgls); if (err) return err; + cond_resched(); } return 0; } @@ -2336,6 +2629,7 @@ static int test_skcipher_vec(const char *driver, int enc, &cfg, req, tsgls); if (err) return err; + cond_resched(); } } #endif @@ -2409,7 +2703,7 @@ static int test_skcipher_vs_generic_impl(const char *driver, unsigned int i; struct cipher_testvec vec = { 0 }; char vec_name[64]; - struct testvec_config cfg; + struct testvec_config *cfg; char cfgname[TESTVEC_CONFIG_NAMELEN]; int err; @@ -2443,6 +2737,12 @@ static int test_skcipher_vs_generic_impl(const char *driver, return err; } + cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); + if (!cfg) { + err = -ENOMEM; + goto out; + } + generic_req = skcipher_request_alloc(generic_tfm, GFP_KERNEL); if (!generic_req) { err = -ENOMEM; @@ -2490,20 +2790,21 @@ static int test_skcipher_vs_generic_impl(const char *driver, for (i = 0; i < fuzz_iterations * 8; i++) { generate_random_cipher_testvec(generic_req, &vec, maxdatasize, vec_name, sizeof(vec_name)); - generate_random_testvec_config(&cfg, cfgname, sizeof(cfgname)); + generate_random_testvec_config(cfg, cfgname, sizeof(cfgname)); err = test_skcipher_vec_cfg(driver, ENCRYPT, &vec, vec_name, - &cfg, req, tsgls); + cfg, req, tsgls); if (err) goto out; err = test_skcipher_vec_cfg(driver, DECRYPT, &vec, vec_name, - &cfg, req, tsgls); + cfg, req, tsgls); if (err) goto out; cond_resched(); } err = 0; out: + kfree(cfg); kfree(vec.key); kfree(vec.iv); kfree(vec.ptext); @@ -2535,6 +2836,7 @@ static int test_skcipher(const char *driver, int enc, tsgls); if (err) return err; + cond_resched(); } return 0; } @@ -4125,6 +4427,7 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "ecb(arc4)", + .generic_driver = "ecb(arc4)-generic", .test = alg_test_skcipher, .suite = { .cipher = __VECS(arc4_tv_template) @@ -4790,6 +5093,13 @@ static const struct alg_test_desc alg_test_descs[] = { .test = alg_test_null, .fips_allowed = 1, }, { + .alg = "xxhash64", + .test = alg_test_hash, + .fips_allowed = 1, + .suite = { + .hash = __VECS(xxhash64_tv_template) + } + }, { .alg = "zlib-deflate", .test = alg_test_comp, .fips_allowed = 1, diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 1fdae5993bc3..073bd2efafca 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -38,7 +38,7 @@ struct hash_testvec { const char *key; const char *plaintext; const char *digest; - unsigned short psize; + unsigned int psize; unsigned short ksize; int setkey_error; int digest_error; @@ -69,7 +69,7 @@ struct cipher_testvec { const char *ctext; unsigned char wk; /* weak key flag */ unsigned short klen; - unsigned short len; + unsigned int len; bool fips_skip; bool generates_iv; int setkey_error; @@ -105,9 +105,9 @@ struct aead_testvec { unsigned char novrfy; unsigned char wk; unsigned char klen; - unsigned short plen; - unsigned short clen; - unsigned short alen; + unsigned int plen; + unsigned int clen; + unsigned int alen; int setkey_error; int setauthsize_error; int crypt_error; @@ -33382,6 +33382,112 @@ static const struct hash_testvec crc32c_tv_template[] = { } }; +static const struct hash_testvec xxhash64_tv_template[] = { + { + .psize = 0, + .digest = "\x99\xe9\xd8\x51\x37\xdb\x46\xef", + }, + { + .plaintext = "\x40", + .psize = 1, + .digest = "\x20\x5c\x91\xaa\x88\xeb\x59\xd0", + }, + { + .plaintext = "\x40\x8b\xb8\x41\xe4\x42\x15\x2d" + "\x88\xc7\x9a\x09\x1a\x9b", + .psize = 14, + .digest = "\xa8\xe8\x2b\xa9\x92\xa1\x37\x4a", + }, + { + .plaintext = "\x40\x8b\xb8\x41\xe4\x42\x15\x2d" + "\x88\xc7\x9a\x09\x1a\x9b\x42\xe0" + "\xd4\x38\xa5\x2a\x26\xa5\x19\x4b" + "\x57\x65\x7f\xad\xc3\x7d\xca\x40" + "\x31\x65\x05\xbb\x31\xae\x51\x11" + "\xa8\xc0\xb3\x28\x42\xeb\x3c\x46" + "\xc8\xed\xed\x0f\x8d\x0b\xfa\x6e" + "\xbc\xe3\x88\x53\xca\x8f\xc8\xd9" + "\x41\x26\x7a\x3d\x21\xdb\x1a\x3c" + "\x01\x1d\xc9\xe9\xb7\x3a\x78\x67" + "\x57\x20\x94\xf1\x1e\xfd\xce\x39" + "\x99\x57\x69\x39\xa5\xd0\x8d\xd9" + "\x43\xfe\x1d\x66\x04\x3c\x27\x6a" + "\xe1\x0d\xe7\xc9\xfa\xc9\x07\x56" + "\xa5\xb3\xec\xd9\x1f\x42\x65\x66" + "\xaa\xbf\x87\x9b\xc5\x41\x9c\x27" + "\x3f\x2f\xa9\x55\x93\x01\x27\x33" + "\x43\x99\x4d\x81\x85\xae\x82\x00" + "\x6c\xd0\xd1\xa3\x57\x18\x06\xcc" + "\xec\x72\xf7\x8e\x87\x2d\x1f\x5e" + "\xd7\x5b\x1f\x36\x4c\xfa\xfd\x18" + "\x89\x76\xd3\x5e\xb5\x5a\xc0\x01" + "\xd2\xa1\x9a\x50\xe6\x08\xb4\x76" + "\x56\x4f\x0e\xbc\x54\xfc\x67\xe6" + "\xb9\xc0\x28\x4b\xb5\xc3\xff\x79" + "\x52\xea\xa1\x90\xc3\xaf\x08\x70" + "\x12\x02\x0c\xdb\x94\x00\x38\x95" + "\xed\xfd\x08\xf7\xe8\x04", + .psize = 222, + .digest = "\x41\xfc\xd4\x29\xfe\xe7\x85\x17", + }, + { + .psize = 0, + .key = "\xb1\x79\x37\x9e\x00\x00\x00\x00", + .ksize = 8, + .digest = "\xef\x17\x9b\x92\xa2\xfd\x75\xac", + }, + + { + .plaintext = "\x40", + .psize = 1, + .key = "\xb1\x79\x37\x9e\x00\x00\x00\x00", + .ksize = 8, + .digest = "\xd1\x70\x4f\x14\x02\xc4\x9e\x71", + }, + { + .plaintext = "\x40\x8b\xb8\x41\xe4\x42\x15\x2d" + "\x88\xc7\x9a\x09\x1a\x9b", + .psize = 14, + .key = "\xb1\x79\x37\x9e\x00\x00\x00\x00", + .ksize = 8, + .digest = "\xa4\xcd\xfe\x8e\x37\xe2\x1c\x64" + }, + { + .plaintext = "\x40\x8b\xb8\x41\xe4\x42\x15\x2d" + "\x88\xc7\x9a\x09\x1a\x9b\x42\xe0" + "\xd4\x38\xa5\x2a\x26\xa5\x19\x4b" + "\x57\x65\x7f\xad\xc3\x7d\xca\x40" + "\x31\x65\x05\xbb\x31\xae\x51\x11" + "\xa8\xc0\xb3\x28\x42\xeb\x3c\x46" + "\xc8\xed\xed\x0f\x8d\x0b\xfa\x6e" + "\xbc\xe3\x88\x53\xca\x8f\xc8\xd9" + "\x41\x26\x7a\x3d\x21\xdb\x1a\x3c" + "\x01\x1d\xc9\xe9\xb7\x3a\x78\x67" + "\x57\x20\x94\xf1\x1e\xfd\xce\x39" + "\x99\x57\x69\x39\xa5\xd0\x8d\xd9" + "\x43\xfe\x1d\x66\x04\x3c\x27\x6a" + "\xe1\x0d\xe7\xc9\xfa\xc9\x07\x56" + "\xa5\xb3\xec\xd9\x1f\x42\x65\x66" + "\xaa\xbf\x87\x9b\xc5\x41\x9c\x27" + "\x3f\x2f\xa9\x55\x93\x01\x27\x33" + "\x43\x99\x4d\x81\x85\xae\x82\x00" + "\x6c\xd0\xd1\xa3\x57\x18\x06\xcc" + "\xec\x72\xf7\x8e\x87\x2d\x1f\x5e" + "\xd7\x5b\x1f\x36\x4c\xfa\xfd\x18" + "\x89\x76\xd3\x5e\xb5\x5a\xc0\x01" + "\xd2\xa1\x9a\x50\xe6\x08\xb4\x76" + "\x56\x4f\x0e\xbc\x54\xfc\x67\xe6" + "\xb9\xc0\x28\x4b\xb5\xc3\xff\x79" + "\x52\xea\xa1\x90\xc3\xaf\x08\x70" + "\x12\x02\x0c\xdb\x94\x00\x38\x95" + "\xed\xfd\x08\xf7\xe8\x04", + .psize = 222, + .key = "\xb1\x79\x37\x9e\x00\x00\x00\x00", + .ksize = 8, + .digest = "\x58\xbc\x55\xf2\x42\x81\x5c\xf0" + }, +}; + static const struct comp_testvec lz4_comp_tv_template[] = { { .inlen = 255, diff --git a/crypto/tgr192.c b/crypto/tgr192.c index 702c2c89c7a1..052648e24909 100644 --- a/crypto/tgr192.c +++ b/crypto/tgr192.c @@ -630,9 +630,10 @@ static struct shash_alg tgr_algs[3] = { { .final = tgr192_final, .descsize = sizeof(struct tgr192_ctx), .base = { - .cra_name = "tgr192", - .cra_blocksize = TGR192_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "tgr192", + .cra_driver_name = "tgr192-generic", + .cra_blocksize = TGR192_BLOCK_SIZE, + .cra_module = THIS_MODULE, } }, { .digestsize = TGR160_DIGEST_SIZE, @@ -641,9 +642,10 @@ static struct shash_alg tgr_algs[3] = { { .final = tgr160_final, .descsize = sizeof(struct tgr192_ctx), .base = { - .cra_name = "tgr160", - .cra_blocksize = TGR192_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "tgr160", + .cra_driver_name = "tgr160-generic", + .cra_blocksize = TGR192_BLOCK_SIZE, + .cra_module = THIS_MODULE, } }, { .digestsize = TGR128_DIGEST_SIZE, @@ -652,9 +654,10 @@ static struct shash_alg tgr_algs[3] = { { .final = tgr128_final, .descsize = sizeof(struct tgr192_ctx), .base = { - .cra_name = "tgr128", - .cra_blocksize = TGR192_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "tgr128", + .cra_driver_name = "tgr128-generic", + .cra_blocksize = TGR192_BLOCK_SIZE, + .cra_module = THIS_MODULE, } } }; diff --git a/crypto/wp512.c b/crypto/wp512.c index 1b8e502d999f..feadc13ccae0 100644 --- a/crypto/wp512.c +++ b/crypto/wp512.c @@ -1126,9 +1126,10 @@ static struct shash_alg wp_algs[3] = { { .final = wp512_final, .descsize = sizeof(struct wp512_ctx), .base = { - .cra_name = "wp512", - .cra_blocksize = WP512_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "wp512", + .cra_driver_name = "wp512-generic", + .cra_blocksize = WP512_BLOCK_SIZE, + .cra_module = THIS_MODULE, } }, { .digestsize = WP384_DIGEST_SIZE, @@ -1137,9 +1138,10 @@ static struct shash_alg wp_algs[3] = { { .final = wp384_final, .descsize = sizeof(struct wp512_ctx), .base = { - .cra_name = "wp384", - .cra_blocksize = WP512_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "wp384", + .cra_driver_name = "wp384-generic", + .cra_blocksize = WP512_BLOCK_SIZE, + .cra_module = THIS_MODULE, } }, { .digestsize = WP256_DIGEST_SIZE, @@ -1148,9 +1150,10 @@ static struct shash_alg wp_algs[3] = { { .final = wp256_final, .descsize = sizeof(struct wp512_ctx), .base = { - .cra_name = "wp256", - .cra_blocksize = WP512_BLOCK_SIZE, - .cra_module = THIS_MODULE, + .cra_name = "wp256", + .cra_driver_name = "wp256-generic", + .cra_blocksize = WP512_BLOCK_SIZE, + .cra_module = THIS_MODULE, } } }; diff --git a/crypto/xxhash_generic.c b/crypto/xxhash_generic.c new file mode 100644 index 000000000000..4aad2c0f40a9 --- /dev/null +++ b/crypto/xxhash_generic.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <crypto/internal/hash.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/xxhash.h> +#include <asm/unaligned.h> + +#define XXHASH64_BLOCK_SIZE 32 +#define XXHASH64_DIGEST_SIZE 8 + +struct xxhash64_tfm_ctx { + u64 seed; +}; + +struct xxhash64_desc_ctx { + struct xxh64_state xxhstate; +}; + +static int xxhash64_setkey(struct crypto_shash *tfm, const u8 *key, + unsigned int keylen) +{ + struct xxhash64_tfm_ctx *tctx = crypto_shash_ctx(tfm); + + if (keylen != sizeof(tctx->seed)) { + crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + tctx->seed = get_unaligned_le64(key); + return 0; +} + +static int xxhash64_init(struct shash_desc *desc) +{ + struct xxhash64_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + struct xxhash64_desc_ctx *dctx = shash_desc_ctx(desc); + + xxh64_reset(&dctx->xxhstate, tctx->seed); + + return 0; +} + +static int xxhash64_update(struct shash_desc *desc, const u8 *data, + unsigned int length) +{ + struct xxhash64_desc_ctx *dctx = shash_desc_ctx(desc); + + xxh64_update(&dctx->xxhstate, data, length); + + return 0; +} + +static int xxhash64_final(struct shash_desc *desc, u8 *out) +{ + struct xxhash64_desc_ctx *dctx = shash_desc_ctx(desc); + + put_unaligned_le64(xxh64_digest(&dctx->xxhstate), out); + + return 0; +} + +static int xxhash64_digest(struct shash_desc *desc, const u8 *data, + unsigned int length, u8 *out) +{ + struct xxhash64_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm); + + put_unaligned_le64(xxh64(data, length, tctx->seed), out); + + return 0; +} + +static struct shash_alg alg = { + .digestsize = XXHASH64_DIGEST_SIZE, + .setkey = xxhash64_setkey, + .init = xxhash64_init, + .update = xxhash64_update, + .final = xxhash64_final, + .digest = xxhash64_digest, + .descsize = sizeof(struct xxhash64_desc_ctx), + .base = { + .cra_name = "xxhash64", + .cra_driver_name = "xxhash64-generic", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, + .cra_blocksize = XXHASH64_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct xxhash64_tfm_ctx), + .cra_module = THIS_MODULE, + } +}; + +static int __init xxhash_mod_init(void) +{ + return crypto_register_shash(&alg); +} + +static void __exit xxhash_mod_fini(void) +{ + crypto_unregister_shash(&alg); +} + +subsys_initcall(xxhash_mod_init); +module_exit(xxhash_mod_fini); + +MODULE_AUTHOR("Nikolay Borisov <nborisov@suse.com>"); +MODULE_DESCRIPTION("xxhash calculations wrapper for lib/xxhash.c"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS_CRYPTO("xxhash64"); +MODULE_ALIAS_CRYPTO("xxhash64-generic"); diff --git a/crypto/zstd.c b/crypto/zstd.c index f1e4c70c9d24..5a3ff258d8f7 100644 --- a/crypto/zstd.c +++ b/crypto/zstd.c @@ -206,6 +206,7 @@ static int zstd_sdecompress(struct crypto_scomp *tfm, const u8 *src, static struct crypto_alg alg = { .cra_name = "zstd", + .cra_driver_name = "zstd-generic", .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, .cra_ctxsize = sizeof(struct zstd_ctx), .cra_module = THIS_MODULE, diff --git a/drivers/char/hw_random/iproc-rng200.c b/drivers/char/hw_random/iproc-rng200.c index 8b5a20b35293..92be1c0ab99f 100644 --- a/drivers/char/hw_random/iproc-rng200.c +++ b/drivers/char/hw_random/iproc-rng200.c @@ -220,6 +220,7 @@ static int iproc_rng200_probe(struct platform_device *pdev) } static const struct of_device_id iproc_rng200_of_match[] = { + { .compatible = "brcm,bcm7211-rng200", }, { .compatible = "brcm,bcm7278-rng200", }, { .compatible = "brcm,iproc-rng200", }, {}, diff --git a/drivers/char/hw_random/meson-rng.c b/drivers/char/hw_random/meson-rng.c index 2e23be802a62..76e693da5dde 100644 --- a/drivers/char/hw_random/meson-rng.c +++ b/drivers/char/hw_random/meson-rng.c @@ -1,58 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * This file is provided under a dual BSD/GPLv2 license. When using or - * redistributing this file, you may do so under either license. - * - * GPL LICENSE SUMMARY - * * Copyright (c) 2016 BayLibre, SAS. * Author: Neil Armstrong <narmstrong@baylibre.com> * Copyright (C) 2014 Amlogic, Inc. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, see <http://www.gnu.org/licenses/>. - * The full GNU General Public License is included in this distribution - * in the file called COPYING. - * - * BSD LICENSE - * - * Copyright (c) 2016 BayLibre, SAS. - * Author: Neil Armstrong <narmstrong@baylibre.com> - * Copyright (C) 2014 Amlogic, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * * Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include <linux/err.h> #include <linux/module.h> diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 0af08081e305..603413f28fa3 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -520,10 +520,13 @@ config CRYPTO_DEV_ATMEL_SHA To compile this driver as a module, choose M here: the module will be called atmel-sha. +config CRYPTO_DEV_ATMEL_I2C + tristate + config CRYPTO_DEV_ATMEL_ECC tristate "Support for Microchip / Atmel ECC hw accelerator" - depends on ARCH_AT91 || COMPILE_TEST depends on I2C + select CRYPTO_DEV_ATMEL_I2C select CRYPTO_ECDH select CRC16 help @@ -534,6 +537,21 @@ config CRYPTO_DEV_ATMEL_ECC To compile this driver as a module, choose M here: the module will be called atmel-ecc. +config CRYPTO_DEV_ATMEL_SHA204A + tristate "Support for Microchip / Atmel SHA accelerator and RNG" + depends on I2C + select CRYPTO_DEV_ATMEL_I2C + select HW_RANDOM + select CRC16 + help + Microhip / Atmel SHA accelerator and RNG. + Select this if you want to use the Microchip / Atmel SHA204A + module as a random number generator. (Other functions of the + chip are currently not exposed by this driver) + + To compile this driver as a module, choose M here: the module + will be called atmel-sha204a. + config CRYPTO_DEV_CCP bool "Support for AMD Secure Processor" depends on ((X86 && PCI) || (ARM64 && (OF_ADDRESS || ACPI))) && HAS_IOMEM diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index a23a7197fcd7..afc4753b5d28 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -2,7 +2,9 @@ obj-$(CONFIG_CRYPTO_DEV_ATMEL_AES) += atmel-aes.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA) += atmel-sha.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_TDES) += atmel-tdes.o +obj-$(CONFIG_CRYPTO_DEV_ATMEL_I2C) += atmel-i2c.o obj-$(CONFIG_CRYPTO_DEV_ATMEL_ECC) += atmel-ecc.o +obj-$(CONFIG_CRYPTO_DEV_ATMEL_SHA204A) += atmel-sha204a.o obj-$(CONFIG_CRYPTO_DEV_CAVIUM_ZIP) += cavium/ obj-$(CONFIG_CRYPTO_DEV_CCP) += ccp/ obj-$(CONFIG_CRYPTO_DEV_CCREE) += ccree/ diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index 49f3e0ce242c..cbfc607282f4 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -67,12 +67,16 @@ static void set_dynamic_sa_command_1(struct dynamic_sa_ctl *sa, u32 cm, } static inline int crypto4xx_crypt(struct skcipher_request *req, - const unsigned int ivlen, bool decrypt) + const unsigned int ivlen, bool decrypt, + bool check_blocksize) { struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(req); struct crypto4xx_ctx *ctx = crypto_skcipher_ctx(cipher); __le32 iv[AES_IV_SIZE]; + if (check_blocksize && !IS_ALIGNED(req->cryptlen, AES_BLOCK_SIZE)) + return -EINVAL; + if (ivlen) crypto4xx_memcpy_to_le32(iv, req->iv, ivlen); @@ -81,24 +85,34 @@ static inline int crypto4xx_crypt(struct skcipher_request *req, ctx->sa_len, 0, NULL); } -int crypto4xx_encrypt_noiv(struct skcipher_request *req) +int crypto4xx_encrypt_noiv_block(struct skcipher_request *req) +{ + return crypto4xx_crypt(req, 0, false, true); +} + +int crypto4xx_encrypt_iv_stream(struct skcipher_request *req) +{ + return crypto4xx_crypt(req, AES_IV_SIZE, false, false); +} + +int crypto4xx_decrypt_noiv_block(struct skcipher_request *req) { - return crypto4xx_crypt(req, 0, false); + return crypto4xx_crypt(req, 0, true, true); } -int crypto4xx_encrypt_iv(struct skcipher_request *req) +int crypto4xx_decrypt_iv_stream(struct skcipher_request *req) { - return crypto4xx_crypt(req, AES_IV_SIZE, false); + return crypto4xx_crypt(req, AES_IV_SIZE, true, false); } -int crypto4xx_decrypt_noiv(struct skcipher_request *req) +int crypto4xx_encrypt_iv_block(struct skcipher_request *req) { - return crypto4xx_crypt(req, 0, true); + return crypto4xx_crypt(req, AES_IV_SIZE, false, true); } -int crypto4xx_decrypt_iv(struct skcipher_request *req) +int crypto4xx_decrypt_iv_block(struct skcipher_request *req) { - return crypto4xx_crypt(req, AES_IV_SIZE, true); + return crypto4xx_crypt(req, AES_IV_SIZE, true, true); } /** @@ -269,8 +283,8 @@ crypto4xx_ctr_crypt(struct skcipher_request *req, bool encrypt) return ret; } - return encrypt ? crypto4xx_encrypt_iv(req) - : crypto4xx_decrypt_iv(req); + return encrypt ? crypto4xx_encrypt_iv_stream(req) + : crypto4xx_decrypt_iv_stream(req); } static int crypto4xx_sk_setup_fallback(struct crypto4xx_ctx *ctx, diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 16d911aaa508..de5e9352e920 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -182,7 +182,6 @@ static u32 crypto4xx_build_pdr(struct crypto4xx_device *dev) dev->pdr_pa); return -ENOMEM; } - memset(dev->pdr, 0, sizeof(struct ce_pd) * PPC4XX_NUM_PD); dev->shadow_sa_pool = dma_alloc_coherent(dev->core_dev->device, sizeof(union shadow_sa_buf) * PPC4XX_NUM_PD, &dev->shadow_sa_pool_pa, @@ -1210,8 +1209,8 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_IV_SIZE, .setkey = crypto4xx_setkey_aes_cbc, - .encrypt = crypto4xx_encrypt_iv, - .decrypt = crypto4xx_decrypt_iv, + .encrypt = crypto4xx_encrypt_iv_block, + .decrypt = crypto4xx_decrypt_iv_block, .init = crypto4xx_sk_init, .exit = crypto4xx_sk_exit, } }, @@ -1222,7 +1221,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = AES_BLOCK_SIZE, + .cra_blocksize = 1, .cra_ctxsize = sizeof(struct crypto4xx_ctx), .cra_module = THIS_MODULE, }, @@ -1230,8 +1229,8 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_IV_SIZE, .setkey = crypto4xx_setkey_aes_cfb, - .encrypt = crypto4xx_encrypt_iv, - .decrypt = crypto4xx_decrypt_iv, + .encrypt = crypto4xx_encrypt_iv_stream, + .decrypt = crypto4xx_decrypt_iv_stream, .init = crypto4xx_sk_init, .exit = crypto4xx_sk_exit, } }, @@ -1243,7 +1242,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { .cra_flags = CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = AES_BLOCK_SIZE, + .cra_blocksize = 1, .cra_ctxsize = sizeof(struct crypto4xx_ctx), .cra_module = THIS_MODULE, }, @@ -1263,7 +1262,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = AES_BLOCK_SIZE, + .cra_blocksize = 1, .cra_ctxsize = sizeof(struct crypto4xx_ctx), .cra_module = THIS_MODULE, }, @@ -1290,8 +1289,8 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .setkey = crypto4xx_setkey_aes_ecb, - .encrypt = crypto4xx_encrypt_noiv, - .decrypt = crypto4xx_decrypt_noiv, + .encrypt = crypto4xx_encrypt_noiv_block, + .decrypt = crypto4xx_decrypt_noiv_block, .init = crypto4xx_sk_init, .exit = crypto4xx_sk_exit, } }, @@ -1302,7 +1301,7 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_KERN_DRIVER_ONLY, - .cra_blocksize = AES_BLOCK_SIZE, + .cra_blocksize = 1, .cra_ctxsize = sizeof(struct crypto4xx_ctx), .cra_module = THIS_MODULE, }, @@ -1310,8 +1309,8 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_IV_SIZE, .setkey = crypto4xx_setkey_aes_ofb, - .encrypt = crypto4xx_encrypt_iv, - .decrypt = crypto4xx_decrypt_iv, + .encrypt = crypto4xx_encrypt_iv_stream, + .decrypt = crypto4xx_decrypt_iv_stream, .init = crypto4xx_sk_init, .exit = crypto4xx_sk_exit, } }, diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index ca1c25c40c23..6b6841359190 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -173,10 +173,12 @@ int crypto4xx_setkey_rfc3686(struct crypto_skcipher *cipher, const u8 *key, unsigned int keylen); int crypto4xx_encrypt_ctr(struct skcipher_request *req); int crypto4xx_decrypt_ctr(struct skcipher_request *req); -int crypto4xx_encrypt_iv(struct skcipher_request *req); -int crypto4xx_decrypt_iv(struct skcipher_request *req); -int crypto4xx_encrypt_noiv(struct skcipher_request *req); -int crypto4xx_decrypt_noiv(struct skcipher_request *req); +int crypto4xx_encrypt_iv_stream(struct skcipher_request *req); +int crypto4xx_decrypt_iv_stream(struct skcipher_request *req); +int crypto4xx_encrypt_iv_block(struct skcipher_request *req); +int crypto4xx_decrypt_iv_block(struct skcipher_request *req); +int crypto4xx_encrypt_noiv_block(struct skcipher_request *req); +int crypto4xx_decrypt_noiv_block(struct skcipher_request *req); int crypto4xx_rfc3686_encrypt(struct skcipher_request *req); int crypto4xx_rfc3686_decrypt(struct skcipher_request *req); int crypto4xx_sha1_alg_init(struct crypto_tfm *tfm); diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c index ba00e4563ca0..ff02cc05affb 100644 --- a/drivers/crypto/atmel-ecc.c +++ b/drivers/crypto/atmel-ecc.c @@ -6,8 +6,6 @@ * Author: Tudor Ambarus <tudor.ambarus@microchip.com> */ -#include <linux/bitrev.h> -#include <linux/crc16.h> #include <linux/delay.h> #include <linux/device.h> #include <linux/err.h> @@ -23,42 +21,11 @@ #include <crypto/internal/kpp.h> #include <crypto/ecdh.h> #include <crypto/kpp.h> -#include "atmel-ecc.h" - -/* Used for binding tfm objects to i2c clients. */ -struct atmel_ecc_driver_data { - struct list_head i2c_client_list; - spinlock_t i2c_list_lock; -} ____cacheline_aligned; +#include "atmel-i2c.h" static struct atmel_ecc_driver_data driver_data; /** - * atmel_ecc_i2c_client_priv - i2c_client private data - * @client : pointer to i2c client device - * @i2c_client_list_node: part of i2c_client_list - * @lock : lock for sending i2c commands - * @wake_token : wake token array of zeros - * @wake_token_sz : size in bytes of the wake_token - * @tfm_count : number of active crypto transformations on i2c client - * - * Reads and writes from/to the i2c client are sequential. The first byte - * transmitted to the device is treated as the byte size. Any attempt to send - * more than this number of bytes will cause the device to not ACK those bytes. - * After the host writes a single command byte to the input buffer, reads are - * prohibited until after the device completes command execution. Use a mutex - * when sending i2c commands. - */ -struct atmel_ecc_i2c_client_priv { - struct i2c_client *client; - struct list_head i2c_client_list_node; - struct mutex lock; - u8 wake_token[WAKE_TOKEN_MAX_SIZE]; - size_t wake_token_sz; - atomic_t tfm_count ____cacheline_aligned; -}; - -/** * atmel_ecdh_ctx - transformation context * @client : pointer to i2c client device * @fallback : used for unsupported curves or when user wants to use its own @@ -80,188 +47,12 @@ struct atmel_ecdh_ctx { bool do_fallback; }; -/** - * atmel_ecc_work_data - data structure representing the work - * @ctx : transformation context. - * @cbk : pointer to a callback function to be invoked upon completion of this - * request. This has the form: - * callback(struct atmel_ecc_work_data *work_data, void *areq, u8 status) - * where: - * @work_data: data structure representing the work - * @areq : optional pointer to an argument passed with the original - * request. - * @status : status returned from the i2c client device or i2c error. - * @areq: optional pointer to a user argument for use at callback time. - * @work: describes the task to be executed. - * @cmd : structure used for communicating with the device. - */ -struct atmel_ecc_work_data { - struct atmel_ecdh_ctx *ctx; - void (*cbk)(struct atmel_ecc_work_data *work_data, void *areq, - int status); - void *areq; - struct work_struct work; - struct atmel_ecc_cmd cmd; -}; - -static u16 atmel_ecc_crc16(u16 crc, const u8 *buffer, size_t len) -{ - return cpu_to_le16(bitrev16(crc16(crc, buffer, len))); -} - -/** - * atmel_ecc_checksum() - Generate 16-bit CRC as required by ATMEL ECC. - * CRC16 verification of the count, opcode, param1, param2 and data bytes. - * The checksum is saved in little-endian format in the least significant - * two bytes of the command. CRC polynomial is 0x8005 and the initial register - * value should be zero. - * - * @cmd : structure used for communicating with the device. - */ -static void atmel_ecc_checksum(struct atmel_ecc_cmd *cmd) -{ - u8 *data = &cmd->count; - size_t len = cmd->count - CRC_SIZE; - u16 *crc16 = (u16 *)(data + len); - - *crc16 = atmel_ecc_crc16(0, data, len); -} - -static void atmel_ecc_init_read_cmd(struct atmel_ecc_cmd *cmd) -{ - cmd->word_addr = COMMAND; - cmd->opcode = OPCODE_READ; - /* - * Read the word from Configuration zone that contains the lock bytes - * (UserExtra, Selector, LockValue, LockConfig). - */ - cmd->param1 = CONFIG_ZONE; - cmd->param2 = DEVICE_LOCK_ADDR; - cmd->count = READ_COUNT; - - atmel_ecc_checksum(cmd); - - cmd->msecs = MAX_EXEC_TIME_READ; - cmd->rxsize = READ_RSP_SIZE; -} - -static void atmel_ecc_init_genkey_cmd(struct atmel_ecc_cmd *cmd, u16 keyid) -{ - cmd->word_addr = COMMAND; - cmd->count = GENKEY_COUNT; - cmd->opcode = OPCODE_GENKEY; - cmd->param1 = GENKEY_MODE_PRIVATE; - /* a random private key will be generated and stored in slot keyID */ - cmd->param2 = cpu_to_le16(keyid); - - atmel_ecc_checksum(cmd); - - cmd->msecs = MAX_EXEC_TIME_GENKEY; - cmd->rxsize = GENKEY_RSP_SIZE; -} - -static int atmel_ecc_init_ecdh_cmd(struct atmel_ecc_cmd *cmd, - struct scatterlist *pubkey) -{ - size_t copied; - - cmd->word_addr = COMMAND; - cmd->count = ECDH_COUNT; - cmd->opcode = OPCODE_ECDH; - cmd->param1 = ECDH_PREFIX_MODE; - /* private key slot */ - cmd->param2 = cpu_to_le16(DATA_SLOT_2); - - /* - * The device only supports NIST P256 ECC keys. The public key size will - * always be the same. Use a macro for the key size to avoid unnecessary - * computations. - */ - copied = sg_copy_to_buffer(pubkey, - sg_nents_for_len(pubkey, - ATMEL_ECC_PUBKEY_SIZE), - cmd->data, ATMEL_ECC_PUBKEY_SIZE); - if (copied != ATMEL_ECC_PUBKEY_SIZE) - return -EINVAL; - - atmel_ecc_checksum(cmd); - - cmd->msecs = MAX_EXEC_TIME_ECDH; - cmd->rxsize = ECDH_RSP_SIZE; - - return 0; -} - -/* - * After wake and after execution of a command, there will be error, status, or - * result bytes in the device's output register that can be retrieved by the - * system. When the length of that group is four bytes, the codes returned are - * detailed in error_list. - */ -static int atmel_ecc_status(struct device *dev, u8 *status) -{ - size_t err_list_len = ARRAY_SIZE(error_list); - int i; - u8 err_id = status[1]; - - if (*status != STATUS_SIZE) - return 0; - - if (err_id == STATUS_WAKE_SUCCESSFUL || err_id == STATUS_NOERR) - return 0; - - for (i = 0; i < err_list_len; i++) - if (error_list[i].value == err_id) - break; - - /* if err_id is not in the error_list then ignore it */ - if (i != err_list_len) { - dev_err(dev, "%02x: %s:\n", err_id, error_list[i].error_text); - return err_id; - } - - return 0; -} - -static int atmel_ecc_wakeup(struct i2c_client *client) -{ - struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); - u8 status[STATUS_RSP_SIZE]; - int ret; - - /* - * The device ignores any levels or transitions on the SCL pin when the - * device is idle, asleep or during waking up. Don't check for error - * when waking up the device. - */ - i2c_master_send(client, i2c_priv->wake_token, i2c_priv->wake_token_sz); - - /* - * Wait to wake the device. Typical execution times for ecdh and genkey - * are around tens of milliseconds. Delta is chosen to 50 microseconds. - */ - usleep_range(TWHI_MIN, TWHI_MAX); - - ret = i2c_master_recv(client, status, STATUS_SIZE); - if (ret < 0) - return ret; - - return atmel_ecc_status(&client->dev, status); -} - -static int atmel_ecc_sleep(struct i2c_client *client) -{ - u8 sleep = SLEEP_TOKEN; - - return i2c_master_send(client, &sleep, 1); -} - -static void atmel_ecdh_done(struct atmel_ecc_work_data *work_data, void *areq, +static void atmel_ecdh_done(struct atmel_i2c_work_data *work_data, void *areq, int status) { struct kpp_request *req = areq; struct atmel_ecdh_ctx *ctx = work_data->ctx; - struct atmel_ecc_cmd *cmd = &work_data->cmd; + struct atmel_i2c_cmd *cmd = &work_data->cmd; size_t copied, n_sz; if (status) @@ -282,82 +73,6 @@ free_work_data: kpp_request_complete(req, status); } -/* - * atmel_ecc_send_receive() - send a command to the device and receive its - * response. - * @client: i2c client device - * @cmd : structure used to communicate with the device - * - * After the device receives a Wake token, a watchdog counter starts within the - * device. After the watchdog timer expires, the device enters sleep mode - * regardless of whether some I/O transmission or command execution is in - * progress. If a command is attempted when insufficient time remains prior to - * watchdog timer execution, the device will return the watchdog timeout error - * code without attempting to execute the command. There is no way to reset the - * counter other than to put the device into sleep or idle mode and then - * wake it up again. - */ -static int atmel_ecc_send_receive(struct i2c_client *client, - struct atmel_ecc_cmd *cmd) -{ - struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); - int ret; - - mutex_lock(&i2c_priv->lock); - - ret = atmel_ecc_wakeup(client); - if (ret) - goto err; - - /* send the command */ - ret = i2c_master_send(client, (u8 *)cmd, cmd->count + WORD_ADDR_SIZE); - if (ret < 0) - goto err; - - /* delay the appropriate amount of time for command to execute */ - msleep(cmd->msecs); - - /* receive the response */ - ret = i2c_master_recv(client, cmd->data, cmd->rxsize); - if (ret < 0) - goto err; - - /* put the device into low-power mode */ - ret = atmel_ecc_sleep(client); - if (ret < 0) - goto err; - - mutex_unlock(&i2c_priv->lock); - return atmel_ecc_status(&client->dev, cmd->data); -err: - mutex_unlock(&i2c_priv->lock); - return ret; -} - -static void atmel_ecc_work_handler(struct work_struct *work) -{ - struct atmel_ecc_work_data *work_data = - container_of(work, struct atmel_ecc_work_data, work); - struct atmel_ecc_cmd *cmd = &work_data->cmd; - struct i2c_client *client = work_data->ctx->client; - int status; - - status = atmel_ecc_send_receive(client, cmd); - work_data->cbk(work_data, work_data->areq, status); -} - -static void atmel_ecc_enqueue(struct atmel_ecc_work_data *work_data, - void (*cbk)(struct atmel_ecc_work_data *work_data, - void *areq, int status), - void *areq) -{ - work_data->cbk = (void *)cbk; - work_data->areq = areq; - - INIT_WORK(&work_data->work, atmel_ecc_work_handler); - schedule_work(&work_data->work); -} - static unsigned int atmel_ecdh_supported_curve(unsigned int curve_id) { if (curve_id == ECC_CURVE_NIST_P256) @@ -374,7 +89,7 @@ static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, unsigned int len) { struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm); - struct atmel_ecc_cmd *cmd; + struct atmel_i2c_cmd *cmd; void *public_key; struct ecdh params; int ret = -ENOMEM; @@ -412,9 +127,9 @@ static int atmel_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, ctx->do_fallback = false; ctx->curve_id = params.curve_id; - atmel_ecc_init_genkey_cmd(cmd, DATA_SLOT_2); + atmel_i2c_init_genkey_cmd(cmd, DATA_SLOT_2); - ret = atmel_ecc_send_receive(ctx->client, cmd); + ret = atmel_i2c_send_receive(ctx->client, cmd); if (ret) goto free_public_key; @@ -444,6 +159,9 @@ static int atmel_ecdh_generate_public_key(struct kpp_request *req) return crypto_kpp_generate_public_key(req); } + if (!ctx->public_key) + return -EINVAL; + /* might want less than we've got */ nbytes = min_t(size_t, ATMEL_ECC_PUBKEY_SIZE, req->dst_len); @@ -461,7 +179,7 @@ static int atmel_ecdh_compute_shared_secret(struct kpp_request *req) { struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); struct atmel_ecdh_ctx *ctx = kpp_tfm_ctx(tfm); - struct atmel_ecc_work_data *work_data; + struct atmel_i2c_work_data *work_data; gfp_t gfp; int ret; @@ -482,12 +200,13 @@ static int atmel_ecdh_compute_shared_secret(struct kpp_request *req) return -ENOMEM; work_data->ctx = ctx; + work_data->client = ctx->client; - ret = atmel_ecc_init_ecdh_cmd(&work_data->cmd, req->src); + ret = atmel_i2c_init_ecdh_cmd(&work_data->cmd, req->src); if (ret) goto free_work_data; - atmel_ecc_enqueue(work_data, atmel_ecdh_done, req); + atmel_i2c_enqueue(work_data, atmel_ecdh_done, req); return -EINPROGRESS; @@ -498,7 +217,7 @@ free_work_data: static struct i2c_client *atmel_ecc_i2c_client_alloc(void) { - struct atmel_ecc_i2c_client_priv *i2c_priv, *min_i2c_priv = NULL; + struct atmel_i2c_client_priv *i2c_priv, *min_i2c_priv = NULL; struct i2c_client *client = ERR_PTR(-ENODEV); int min_tfm_cnt = INT_MAX; int tfm_cnt; @@ -533,7 +252,7 @@ static struct i2c_client *atmel_ecc_i2c_client_alloc(void) static void atmel_ecc_i2c_client_free(struct i2c_client *client) { - struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); + struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); atomic_dec(&i2c_priv->tfm_count); } @@ -604,96 +323,18 @@ static struct kpp_alg atmel_ecdh = { }, }; -static inline size_t atmel_ecc_wake_token_sz(u32 bus_clk_rate) -{ - u32 no_of_bits = DIV_ROUND_UP(TWLO_USEC * bus_clk_rate, USEC_PER_SEC); - - /* return the size of the wake_token in bytes */ - return DIV_ROUND_UP(no_of_bits, 8); -} - -static int device_sanity_check(struct i2c_client *client) -{ - struct atmel_ecc_cmd *cmd; - int ret; - - cmd = kmalloc(sizeof(*cmd), GFP_KERNEL); - if (!cmd) - return -ENOMEM; - - atmel_ecc_init_read_cmd(cmd); - - ret = atmel_ecc_send_receive(client, cmd); - if (ret) - goto free_cmd; - - /* - * It is vital that the Configuration, Data and OTP zones be locked - * prior to release into the field of the system containing the device. - * Failure to lock these zones may permit modification of any secret - * keys and may lead to other security problems. - */ - if (cmd->data[LOCK_CONFIG_IDX] || cmd->data[LOCK_VALUE_IDX]) { - dev_err(&client->dev, "Configuration or Data and OTP zones are unlocked!\n"); - ret = -ENOTSUPP; - } - - /* fall through */ -free_cmd: - kfree(cmd); - return ret; -} - static int atmel_ecc_probe(struct i2c_client *client, const struct i2c_device_id *id) { - struct atmel_ecc_i2c_client_priv *i2c_priv; - struct device *dev = &client->dev; + struct atmel_i2c_client_priv *i2c_priv; int ret; - u32 bus_clk_rate; - - if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) { - dev_err(dev, "I2C_FUNC_I2C not supported\n"); - return -ENODEV; - } - ret = of_property_read_u32(client->adapter->dev.of_node, - "clock-frequency", &bus_clk_rate); - if (ret) { - dev_err(dev, "of: failed to read clock-frequency property\n"); - return ret; - } - - if (bus_clk_rate > 1000000L) { - dev_err(dev, "%d exceeds maximum supported clock frequency (1MHz)\n", - bus_clk_rate); - return -EINVAL; - } - - i2c_priv = devm_kmalloc(dev, sizeof(*i2c_priv), GFP_KERNEL); - if (!i2c_priv) - return -ENOMEM; - - i2c_priv->client = client; - mutex_init(&i2c_priv->lock); - - /* - * WAKE_TOKEN_MAX_SIZE was calculated for the maximum bus_clk_rate - - * 1MHz. The previous bus_clk_rate check ensures us that wake_token_sz - * will always be smaller than or equal to WAKE_TOKEN_MAX_SIZE. - */ - i2c_priv->wake_token_sz = atmel_ecc_wake_token_sz(bus_clk_rate); - - memset(i2c_priv->wake_token, 0, sizeof(i2c_priv->wake_token)); - - atomic_set(&i2c_priv->tfm_count, 0); - - i2c_set_clientdata(client, i2c_priv); - - ret = device_sanity_check(client); + ret = atmel_i2c_probe(client, id); if (ret) return ret; + i2c_priv = i2c_get_clientdata(client); + spin_lock(&driver_data.i2c_list_lock); list_add_tail(&i2c_priv->i2c_client_list_node, &driver_data.i2c_client_list); @@ -705,10 +346,10 @@ static int atmel_ecc_probe(struct i2c_client *client, list_del(&i2c_priv->i2c_client_list_node); spin_unlock(&driver_data.i2c_list_lock); - dev_err(dev, "%s alg registration failed\n", + dev_err(&client->dev, "%s alg registration failed\n", atmel_ecdh.base.cra_driver_name); } else { - dev_info(dev, "atmel ecc algorithms registered in /proc/crypto\n"); + dev_info(&client->dev, "atmel ecc algorithms registered in /proc/crypto\n"); } return ret; @@ -716,7 +357,7 @@ static int atmel_ecc_probe(struct i2c_client *client, static int atmel_ecc_remove(struct i2c_client *client) { - struct atmel_ecc_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); + struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); /* Return EBUSY if i2c client already allocated. */ if (atomic_read(&i2c_priv->tfm_count)) { diff --git a/drivers/crypto/atmel-ecc.h b/drivers/crypto/atmel-ecc.h deleted file mode 100644 index 643a3b947338..000000000000 --- a/drivers/crypto/atmel-ecc.h +++ /dev/null @@ -1,116 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (c) 2017, Microchip Technology Inc. - * Author: Tudor Ambarus <tudor.ambarus@microchip.com> - */ - -#ifndef __ATMEL_ECC_H__ -#define __ATMEL_ECC_H__ - -#define ATMEL_ECC_PRIORITY 300 - -#define COMMAND 0x03 /* packet function */ -#define SLEEP_TOKEN 0x01 -#define WAKE_TOKEN_MAX_SIZE 8 - -/* Definitions of Data and Command sizes */ -#define WORD_ADDR_SIZE 1 -#define COUNT_SIZE 1 -#define CRC_SIZE 2 -#define CMD_OVERHEAD_SIZE (COUNT_SIZE + CRC_SIZE) - -/* size in bytes of the n prime */ -#define ATMEL_ECC_NIST_P256_N_SIZE 32 -#define ATMEL_ECC_PUBKEY_SIZE (2 * ATMEL_ECC_NIST_P256_N_SIZE) - -#define STATUS_RSP_SIZE 4 -#define ECDH_RSP_SIZE (32 + CMD_OVERHEAD_SIZE) -#define GENKEY_RSP_SIZE (ATMEL_ECC_PUBKEY_SIZE + \ - CMD_OVERHEAD_SIZE) -#define READ_RSP_SIZE (4 + CMD_OVERHEAD_SIZE) -#define MAX_RSP_SIZE GENKEY_RSP_SIZE - -/** - * atmel_ecc_cmd - structure used for communicating with the device. - * @word_addr: indicates the function of the packet sent to the device. This - * byte should have a value of COMMAND for normal operation. - * @count : number of bytes to be transferred to (or from) the device. - * @opcode : the command code. - * @param1 : the first parameter; always present. - * @param2 : the second parameter; always present. - * @data : optional remaining input data. Includes a 2-byte CRC. - * @rxsize : size of the data received from i2c client. - * @msecs : command execution time in milliseconds - */ -struct atmel_ecc_cmd { - u8 word_addr; - u8 count; - u8 opcode; - u8 param1; - u16 param2; - u8 data[MAX_RSP_SIZE]; - u8 msecs; - u16 rxsize; -} __packed; - -/* Status/Error codes */ -#define STATUS_SIZE 0x04 -#define STATUS_NOERR 0x00 -#define STATUS_WAKE_SUCCESSFUL 0x11 - -static const struct { - u8 value; - const char *error_text; -} error_list[] = { - { 0x01, "CheckMac or Verify miscompare" }, - { 0x03, "Parse Error" }, - { 0x05, "ECC Fault" }, - { 0x0F, "Execution Error" }, - { 0xEE, "Watchdog about to expire" }, - { 0xFF, "CRC or other communication error" }, -}; - -/* Definitions for eeprom organization */ -#define CONFIG_ZONE 0 - -/* Definitions for Indexes common to all commands */ -#define RSP_DATA_IDX 1 /* buffer index of data in response */ -#define DATA_SLOT_2 2 /* used for ECDH private key */ - -/* Definitions for the device lock state */ -#define DEVICE_LOCK_ADDR 0x15 -#define LOCK_VALUE_IDX (RSP_DATA_IDX + 2) -#define LOCK_CONFIG_IDX (RSP_DATA_IDX + 3) - -/* - * Wake High delay to data communication (microseconds). SDA should be stable - * high for this entire duration. - */ -#define TWHI_MIN 1500 -#define TWHI_MAX 1550 - -/* Wake Low duration */ -#define TWLO_USEC 60 - -/* Command execution time (milliseconds) */ -#define MAX_EXEC_TIME_ECDH 58 -#define MAX_EXEC_TIME_GENKEY 115 -#define MAX_EXEC_TIME_READ 1 - -/* Command opcode */ -#define OPCODE_ECDH 0x43 -#define OPCODE_GENKEY 0x40 -#define OPCODE_READ 0x02 - -/* Definitions for the READ Command */ -#define READ_COUNT 7 - -/* Definitions for the GenKey Command */ -#define GENKEY_COUNT 7 -#define GENKEY_MODE_PRIVATE 0x04 - -/* Definitions for the ECDH Command */ -#define ECDH_COUNT 71 -#define ECDH_PREFIX_MODE 0x00 - -#endif /* __ATMEL_ECC_H__ */ diff --git a/drivers/crypto/atmel-i2c.c b/drivers/crypto/atmel-i2c.c new file mode 100644 index 000000000000..dc876fab2882 --- /dev/null +++ b/drivers/crypto/atmel-i2c.c @@ -0,0 +1,364 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Microchip / Atmel ECC (I2C) driver. + * + * Copyright (c) 2017, Microchip Technology Inc. + * Author: Tudor Ambarus <tudor.ambarus@microchip.com> + */ + +#include <linux/bitrev.h> +#include <linux/crc16.h> +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/err.h> +#include <linux/errno.h> +#include <linux/i2c.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/scatterlist.h> +#include <linux/slab.h> +#include <linux/workqueue.h> +#include "atmel-i2c.h" + +/** + * atmel_i2c_checksum() - Generate 16-bit CRC as required by ATMEL ECC. + * CRC16 verification of the count, opcode, param1, param2 and data bytes. + * The checksum is saved in little-endian format in the least significant + * two bytes of the command. CRC polynomial is 0x8005 and the initial register + * value should be zero. + * + * @cmd : structure used for communicating with the device. + */ +static void atmel_i2c_checksum(struct atmel_i2c_cmd *cmd) +{ + u8 *data = &cmd->count; + size_t len = cmd->count - CRC_SIZE; + __le16 *__crc16 = (__le16 *)(data + len); + + *__crc16 = cpu_to_le16(bitrev16(crc16(0, data, len))); +} + +void atmel_i2c_init_read_cmd(struct atmel_i2c_cmd *cmd) +{ + cmd->word_addr = COMMAND; + cmd->opcode = OPCODE_READ; + /* + * Read the word from Configuration zone that contains the lock bytes + * (UserExtra, Selector, LockValue, LockConfig). + */ + cmd->param1 = CONFIG_ZONE; + cmd->param2 = cpu_to_le16(DEVICE_LOCK_ADDR); + cmd->count = READ_COUNT; + + atmel_i2c_checksum(cmd); + + cmd->msecs = MAX_EXEC_TIME_READ; + cmd->rxsize = READ_RSP_SIZE; +} +EXPORT_SYMBOL(atmel_i2c_init_read_cmd); + +void atmel_i2c_init_random_cmd(struct atmel_i2c_cmd *cmd) +{ + cmd->word_addr = COMMAND; + cmd->opcode = OPCODE_RANDOM; + cmd->param1 = 0; + cmd->param2 = 0; + cmd->count = RANDOM_COUNT; + + atmel_i2c_checksum(cmd); + + cmd->msecs = MAX_EXEC_TIME_RANDOM; + cmd->rxsize = RANDOM_RSP_SIZE; +} +EXPORT_SYMBOL(atmel_i2c_init_random_cmd); + +void atmel_i2c_init_genkey_cmd(struct atmel_i2c_cmd *cmd, u16 keyid) +{ + cmd->word_addr = COMMAND; + cmd->count = GENKEY_COUNT; + cmd->opcode = OPCODE_GENKEY; + cmd->param1 = GENKEY_MODE_PRIVATE; + /* a random private key will be generated and stored in slot keyID */ + cmd->param2 = cpu_to_le16(keyid); + + atmel_i2c_checksum(cmd); + + cmd->msecs = MAX_EXEC_TIME_GENKEY; + cmd->rxsize = GENKEY_RSP_SIZE; +} +EXPORT_SYMBOL(atmel_i2c_init_genkey_cmd); + +int atmel_i2c_init_ecdh_cmd(struct atmel_i2c_cmd *cmd, + struct scatterlist *pubkey) +{ + size_t copied; + + cmd->word_addr = COMMAND; + cmd->count = ECDH_COUNT; + cmd->opcode = OPCODE_ECDH; + cmd->param1 = ECDH_PREFIX_MODE; + /* private key slot */ + cmd->param2 = cpu_to_le16(DATA_SLOT_2); + + /* + * The device only supports NIST P256 ECC keys. The public key size will + * always be the same. Use a macro for the key size to avoid unnecessary + * computations. + */ + copied = sg_copy_to_buffer(pubkey, + sg_nents_for_len(pubkey, + ATMEL_ECC_PUBKEY_SIZE), + cmd->data, ATMEL_ECC_PUBKEY_SIZE); + if (copied != ATMEL_ECC_PUBKEY_SIZE) + return -EINVAL; + + atmel_i2c_checksum(cmd); + + cmd->msecs = MAX_EXEC_TIME_ECDH; + cmd->rxsize = ECDH_RSP_SIZE; + + return 0; +} +EXPORT_SYMBOL(atmel_i2c_init_ecdh_cmd); + +/* + * After wake and after execution of a command, there will be error, status, or + * result bytes in the device's output register that can be retrieved by the + * system. When the length of that group is four bytes, the codes returned are + * detailed in error_list. + */ +static int atmel_i2c_status(struct device *dev, u8 *status) +{ + size_t err_list_len = ARRAY_SIZE(error_list); + int i; + u8 err_id = status[1]; + + if (*status != STATUS_SIZE) + return 0; + + if (err_id == STATUS_WAKE_SUCCESSFUL || err_id == STATUS_NOERR) + return 0; + + for (i = 0; i < err_list_len; i++) + if (error_list[i].value == err_id) + break; + + /* if err_id is not in the error_list then ignore it */ + if (i != err_list_len) { + dev_err(dev, "%02x: %s:\n", err_id, error_list[i].error_text); + return err_id; + } + + return 0; +} + +static int atmel_i2c_wakeup(struct i2c_client *client) +{ + struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); + u8 status[STATUS_RSP_SIZE]; + int ret; + + /* + * The device ignores any levels or transitions on the SCL pin when the + * device is idle, asleep or during waking up. Don't check for error + * when waking up the device. + */ + i2c_master_send(client, i2c_priv->wake_token, i2c_priv->wake_token_sz); + + /* + * Wait to wake the device. Typical execution times for ecdh and genkey + * are around tens of milliseconds. Delta is chosen to 50 microseconds. + */ + usleep_range(TWHI_MIN, TWHI_MAX); + + ret = i2c_master_recv(client, status, STATUS_SIZE); + if (ret < 0) + return ret; + + return atmel_i2c_status(&client->dev, status); +} + +static int atmel_i2c_sleep(struct i2c_client *client) +{ + u8 sleep = SLEEP_TOKEN; + + return i2c_master_send(client, &sleep, 1); +} + +/* + * atmel_i2c_send_receive() - send a command to the device and receive its + * response. + * @client: i2c client device + * @cmd : structure used to communicate with the device + * + * After the device receives a Wake token, a watchdog counter starts within the + * device. After the watchdog timer expires, the device enters sleep mode + * regardless of whether some I/O transmission or command execution is in + * progress. If a command is attempted when insufficient time remains prior to + * watchdog timer execution, the device will return the watchdog timeout error + * code without attempting to execute the command. There is no way to reset the + * counter other than to put the device into sleep or idle mode and then + * wake it up again. + */ +int atmel_i2c_send_receive(struct i2c_client *client, struct atmel_i2c_cmd *cmd) +{ + struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); + int ret; + + mutex_lock(&i2c_priv->lock); + + ret = atmel_i2c_wakeup(client); + if (ret) + goto err; + + /* send the command */ + ret = i2c_master_send(client, (u8 *)cmd, cmd->count + WORD_ADDR_SIZE); + if (ret < 0) + goto err; + + /* delay the appropriate amount of time for command to execute */ + msleep(cmd->msecs); + + /* receive the response */ + ret = i2c_master_recv(client, cmd->data, cmd->rxsize); + if (ret < 0) + goto err; + + /* put the device into low-power mode */ + ret = atmel_i2c_sleep(client); + if (ret < 0) + goto err; + + mutex_unlock(&i2c_priv->lock); + return atmel_i2c_status(&client->dev, cmd->data); +err: + mutex_unlock(&i2c_priv->lock); + return ret; +} +EXPORT_SYMBOL(atmel_i2c_send_receive); + +static void atmel_i2c_work_handler(struct work_struct *work) +{ + struct atmel_i2c_work_data *work_data = + container_of(work, struct atmel_i2c_work_data, work); + struct atmel_i2c_cmd *cmd = &work_data->cmd; + struct i2c_client *client = work_data->client; + int status; + + status = atmel_i2c_send_receive(client, cmd); + work_data->cbk(work_data, work_data->areq, status); +} + +void atmel_i2c_enqueue(struct atmel_i2c_work_data *work_data, + void (*cbk)(struct atmel_i2c_work_data *work_data, + void *areq, int status), + void *areq) +{ + work_data->cbk = (void *)cbk; + work_data->areq = areq; + + INIT_WORK(&work_data->work, atmel_i2c_work_handler); + schedule_work(&work_data->work); +} +EXPORT_SYMBOL(atmel_i2c_enqueue); + +static inline size_t atmel_i2c_wake_token_sz(u32 bus_clk_rate) +{ + u32 no_of_bits = DIV_ROUND_UP(TWLO_USEC * bus_clk_rate, USEC_PER_SEC); + + /* return the size of the wake_token in bytes */ + return DIV_ROUND_UP(no_of_bits, 8); +} + +static int device_sanity_check(struct i2c_client *client) +{ + struct atmel_i2c_cmd *cmd; + int ret; + + cmd = kmalloc(sizeof(*cmd), GFP_KERNEL); + if (!cmd) + return -ENOMEM; + + atmel_i2c_init_read_cmd(cmd); + + ret = atmel_i2c_send_receive(client, cmd); + if (ret) + goto free_cmd; + + /* + * It is vital that the Configuration, Data and OTP zones be locked + * prior to release into the field of the system containing the device. + * Failure to lock these zones may permit modification of any secret + * keys and may lead to other security problems. + */ + if (cmd->data[LOCK_CONFIG_IDX] || cmd->data[LOCK_VALUE_IDX]) { + dev_err(&client->dev, "Configuration or Data and OTP zones are unlocked!\n"); + ret = -ENOTSUPP; + } + + /* fall through */ +free_cmd: + kfree(cmd); + return ret; +} + +int atmel_i2c_probe(struct i2c_client *client, const struct i2c_device_id *id) +{ + struct atmel_i2c_client_priv *i2c_priv; + struct device *dev = &client->dev; + int ret; + u32 bus_clk_rate; + + if (!i2c_check_functionality(client->adapter, I2C_FUNC_I2C)) { + dev_err(dev, "I2C_FUNC_I2C not supported\n"); + return -ENODEV; + } + + bus_clk_rate = i2c_acpi_find_bus_speed(&client->adapter->dev); + if (!bus_clk_rate) { + ret = device_property_read_u32(&client->adapter->dev, + "clock-frequency", &bus_clk_rate); + if (ret) { + dev_err(dev, "failed to read clock-frequency property\n"); + return ret; + } + } + + if (bus_clk_rate > 1000000L) { + dev_err(dev, "%d exceeds maximum supported clock frequency (1MHz)\n", + bus_clk_rate); + return -EINVAL; + } + + i2c_priv = devm_kmalloc(dev, sizeof(*i2c_priv), GFP_KERNEL); + if (!i2c_priv) + return -ENOMEM; + + i2c_priv->client = client; + mutex_init(&i2c_priv->lock); + + /* + * WAKE_TOKEN_MAX_SIZE was calculated for the maximum bus_clk_rate - + * 1MHz. The previous bus_clk_rate check ensures us that wake_token_sz + * will always be smaller than or equal to WAKE_TOKEN_MAX_SIZE. + */ + i2c_priv->wake_token_sz = atmel_i2c_wake_token_sz(bus_clk_rate); + + memset(i2c_priv->wake_token, 0, sizeof(i2c_priv->wake_token)); + + atomic_set(&i2c_priv->tfm_count, 0); + + i2c_set_clientdata(client, i2c_priv); + + ret = device_sanity_check(client); + if (ret) + return ret; + + return 0; +} +EXPORT_SYMBOL(atmel_i2c_probe); + +MODULE_AUTHOR("Tudor Ambarus <tudor.ambarus@microchip.com>"); +MODULE_DESCRIPTION("Microchip / Atmel ECC (I2C) driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/crypto/atmel-i2c.h b/drivers/crypto/atmel-i2c.h new file mode 100644 index 000000000000..21860b99c3e3 --- /dev/null +++ b/drivers/crypto/atmel-i2c.h @@ -0,0 +1,197 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2017, Microchip Technology Inc. + * Author: Tudor Ambarus <tudor.ambarus@microchip.com> + */ + +#ifndef __ATMEL_I2C_H__ +#define __ATMEL_I2C_H__ + +#include <linux/hw_random.h> +#include <linux/types.h> + +#define ATMEL_ECC_PRIORITY 300 + +#define COMMAND 0x03 /* packet function */ +#define SLEEP_TOKEN 0x01 +#define WAKE_TOKEN_MAX_SIZE 8 + +/* Definitions of Data and Command sizes */ +#define WORD_ADDR_SIZE 1 +#define COUNT_SIZE 1 +#define CRC_SIZE 2 +#define CMD_OVERHEAD_SIZE (COUNT_SIZE + CRC_SIZE) + +/* size in bytes of the n prime */ +#define ATMEL_ECC_NIST_P256_N_SIZE 32 +#define ATMEL_ECC_PUBKEY_SIZE (2 * ATMEL_ECC_NIST_P256_N_SIZE) + +#define STATUS_RSP_SIZE 4 +#define ECDH_RSP_SIZE (32 + CMD_OVERHEAD_SIZE) +#define GENKEY_RSP_SIZE (ATMEL_ECC_PUBKEY_SIZE + \ + CMD_OVERHEAD_SIZE) +#define READ_RSP_SIZE (4 + CMD_OVERHEAD_SIZE) +#define RANDOM_RSP_SIZE (32 + CMD_OVERHEAD_SIZE) +#define MAX_RSP_SIZE GENKEY_RSP_SIZE + +/** + * atmel_i2c_cmd - structure used for communicating with the device. + * @word_addr: indicates the function of the packet sent to the device. This + * byte should have a value of COMMAND for normal operation. + * @count : number of bytes to be transferred to (or from) the device. + * @opcode : the command code. + * @param1 : the first parameter; always present. + * @param2 : the second parameter; always present. + * @data : optional remaining input data. Includes a 2-byte CRC. + * @rxsize : size of the data received from i2c client. + * @msecs : command execution time in milliseconds + */ +struct atmel_i2c_cmd { + u8 word_addr; + u8 count; + u8 opcode; + u8 param1; + __le16 param2; + u8 data[MAX_RSP_SIZE]; + u8 msecs; + u16 rxsize; +} __packed; + +/* Status/Error codes */ +#define STATUS_SIZE 0x04 +#define STATUS_NOERR 0x00 +#define STATUS_WAKE_SUCCESSFUL 0x11 + +static const struct { + u8 value; + const char *error_text; +} error_list[] = { + { 0x01, "CheckMac or Verify miscompare" }, + { 0x03, "Parse Error" }, + { 0x05, "ECC Fault" }, + { 0x0F, "Execution Error" }, + { 0xEE, "Watchdog about to expire" }, + { 0xFF, "CRC or other communication error" }, +}; + +/* Definitions for eeprom organization */ +#define CONFIG_ZONE 0 + +/* Definitions for Indexes common to all commands */ +#define RSP_DATA_IDX 1 /* buffer index of data in response */ +#define DATA_SLOT_2 2 /* used for ECDH private key */ + +/* Definitions for the device lock state */ +#define DEVICE_LOCK_ADDR 0x15 +#define LOCK_VALUE_IDX (RSP_DATA_IDX + 2) +#define LOCK_CONFIG_IDX (RSP_DATA_IDX + 3) + +/* + * Wake High delay to data communication (microseconds). SDA should be stable + * high for this entire duration. + */ +#define TWHI_MIN 1500 +#define TWHI_MAX 1550 + +/* Wake Low duration */ +#define TWLO_USEC 60 + +/* Command execution time (milliseconds) */ +#define MAX_EXEC_TIME_ECDH 58 +#define MAX_EXEC_TIME_GENKEY 115 +#define MAX_EXEC_TIME_READ 1 +#define MAX_EXEC_TIME_RANDOM 50 + +/* Command opcode */ +#define OPCODE_ECDH 0x43 +#define OPCODE_GENKEY 0x40 +#define OPCODE_READ 0x02 +#define OPCODE_RANDOM 0x1b + +/* Definitions for the READ Command */ +#define READ_COUNT 7 + +/* Definitions for the RANDOM Command */ +#define RANDOM_COUNT 7 + +/* Definitions for the GenKey Command */ +#define GENKEY_COUNT 7 +#define GENKEY_MODE_PRIVATE 0x04 + +/* Definitions for the ECDH Command */ +#define ECDH_COUNT 71 +#define ECDH_PREFIX_MODE 0x00 + +/* Used for binding tfm objects to i2c clients. */ +struct atmel_ecc_driver_data { + struct list_head i2c_client_list; + spinlock_t i2c_list_lock; +} ____cacheline_aligned; + +/** + * atmel_i2c_client_priv - i2c_client private data + * @client : pointer to i2c client device + * @i2c_client_list_node: part of i2c_client_list + * @lock : lock for sending i2c commands + * @wake_token : wake token array of zeros + * @wake_token_sz : size in bytes of the wake_token + * @tfm_count : number of active crypto transformations on i2c client + * + * Reads and writes from/to the i2c client are sequential. The first byte + * transmitted to the device is treated as the byte size. Any attempt to send + * more than this number of bytes will cause the device to not ACK those bytes. + * After the host writes a single command byte to the input buffer, reads are + * prohibited until after the device completes command execution. Use a mutex + * when sending i2c commands. + */ +struct atmel_i2c_client_priv { + struct i2c_client *client; + struct list_head i2c_client_list_node; + struct mutex lock; + u8 wake_token[WAKE_TOKEN_MAX_SIZE]; + size_t wake_token_sz; + atomic_t tfm_count ____cacheline_aligned; + struct hwrng hwrng; +}; + +/** + * atmel_i2c_work_data - data structure representing the work + * @ctx : transformation context. + * @cbk : pointer to a callback function to be invoked upon completion of this + * request. This has the form: + * callback(struct atmel_i2c_work_data *work_data, void *areq, u8 status) + * where: + * @work_data: data structure representing the work + * @areq : optional pointer to an argument passed with the original + * request. + * @status : status returned from the i2c client device or i2c error. + * @areq: optional pointer to a user argument for use at callback time. + * @work: describes the task to be executed. + * @cmd : structure used for communicating with the device. + */ +struct atmel_i2c_work_data { + void *ctx; + struct i2c_client *client; + void (*cbk)(struct atmel_i2c_work_data *work_data, void *areq, + int status); + void *areq; + struct work_struct work; + struct atmel_i2c_cmd cmd; +}; + +int atmel_i2c_probe(struct i2c_client *client, const struct i2c_device_id *id); + +void atmel_i2c_enqueue(struct atmel_i2c_work_data *work_data, + void (*cbk)(struct atmel_i2c_work_data *work_data, + void *areq, int status), + void *areq); + +int atmel_i2c_send_receive(struct i2c_client *client, struct atmel_i2c_cmd *cmd); + +void atmel_i2c_init_read_cmd(struct atmel_i2c_cmd *cmd); +void atmel_i2c_init_random_cmd(struct atmel_i2c_cmd *cmd); +void atmel_i2c_init_genkey_cmd(struct atmel_i2c_cmd *cmd, u16 keyid); +int atmel_i2c_init_ecdh_cmd(struct atmel_i2c_cmd *cmd, + struct scatterlist *pubkey); + +#endif /* __ATMEL_I2C_H__ */ diff --git a/drivers/crypto/atmel-sha204a.c b/drivers/crypto/atmel-sha204a.c new file mode 100644 index 000000000000..ea0d2068ea4f --- /dev/null +++ b/drivers/crypto/atmel-sha204a.c @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Microchip / Atmel SHA204A (I2C) driver. + * + * Copyright (c) 2019 Linaro, Ltd. <ard.biesheuvel@linaro.org> + */ + +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/err.h> +#include <linux/errno.h> +#include <linux/i2c.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/scatterlist.h> +#include <linux/slab.h> +#include <linux/workqueue.h> +#include "atmel-i2c.h" + +static void atmel_sha204a_rng_done(struct atmel_i2c_work_data *work_data, + void *areq, int status) +{ + struct atmel_i2c_client_priv *i2c_priv = work_data->ctx; + struct hwrng *rng = areq; + + if (status) + dev_warn_ratelimited(&i2c_priv->client->dev, + "i2c transaction failed (%d)\n", + status); + + rng->priv = (unsigned long)work_data; + atomic_dec(&i2c_priv->tfm_count); +} + +static int atmel_sha204a_rng_read_nonblocking(struct hwrng *rng, void *data, + size_t max) +{ + struct atmel_i2c_client_priv *i2c_priv; + struct atmel_i2c_work_data *work_data; + + i2c_priv = container_of(rng, struct atmel_i2c_client_priv, hwrng); + + /* keep maximum 1 asynchronous read in flight at any time */ + if (!atomic_add_unless(&i2c_priv->tfm_count, 1, 1)) + return 0; + + if (rng->priv) { + work_data = (struct atmel_i2c_work_data *)rng->priv; + max = min(sizeof(work_data->cmd.data), max); + memcpy(data, &work_data->cmd.data, max); + rng->priv = 0; + } else { + work_data = kmalloc(sizeof(*work_data), GFP_ATOMIC); + if (!work_data) + return -ENOMEM; + + work_data->ctx = i2c_priv; + work_data->client = i2c_priv->client; + + max = 0; + } + + atmel_i2c_init_random_cmd(&work_data->cmd); + atmel_i2c_enqueue(work_data, atmel_sha204a_rng_done, rng); + + return max; +} + +static int atmel_sha204a_rng_read(struct hwrng *rng, void *data, size_t max, + bool wait) +{ + struct atmel_i2c_client_priv *i2c_priv; + struct atmel_i2c_cmd cmd; + int ret; + + if (!wait) + return atmel_sha204a_rng_read_nonblocking(rng, data, max); + + i2c_priv = container_of(rng, struct atmel_i2c_client_priv, hwrng); + + atmel_i2c_init_random_cmd(&cmd); + + ret = atmel_i2c_send_receive(i2c_priv->client, &cmd); + if (ret) + return ret; + + max = min(sizeof(cmd.data), max); + memcpy(data, cmd.data, max); + + return max; +} + +static int atmel_sha204a_probe(struct i2c_client *client, + const struct i2c_device_id *id) +{ + struct atmel_i2c_client_priv *i2c_priv; + int ret; + + ret = atmel_i2c_probe(client, id); + if (ret) + return ret; + + i2c_priv = i2c_get_clientdata(client); + + memset(&i2c_priv->hwrng, 0, sizeof(i2c_priv->hwrng)); + + i2c_priv->hwrng.name = dev_name(&client->dev); + i2c_priv->hwrng.read = atmel_sha204a_rng_read; + i2c_priv->hwrng.quality = 1024; + + ret = hwrng_register(&i2c_priv->hwrng); + if (ret) + dev_warn(&client->dev, "failed to register RNG (%d)\n", ret); + + return ret; +} + +static int atmel_sha204a_remove(struct i2c_client *client) +{ + struct atmel_i2c_client_priv *i2c_priv = i2c_get_clientdata(client); + + if (atomic_read(&i2c_priv->tfm_count)) { + dev_err(&client->dev, "Device is busy\n"); + return -EBUSY; + } + + if (i2c_priv->hwrng.priv) + kfree((void *)i2c_priv->hwrng.priv); + hwrng_unregister(&i2c_priv->hwrng); + + return 0; +} + +static const struct of_device_id atmel_sha204a_dt_ids[] = { + { .compatible = "atmel,atsha204a", }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, atmel_sha204a_dt_ids); + +static const struct i2c_device_id atmel_sha204a_id[] = { + { "atsha204a", 0 }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(i2c, atmel_sha204a_id); + +static struct i2c_driver atmel_sha204a_driver = { + .probe = atmel_sha204a_probe, + .remove = atmel_sha204a_remove, + .id_table = atmel_sha204a_id, + + .driver.name = "atmel-sha204a", + .driver.of_match_table = of_match_ptr(atmel_sha204a_dt_ids), +}; + +static int __init atmel_sha204a_init(void) +{ + return i2c_add_driver(&atmel_sha204a_driver); +} + +static void __exit atmel_sha204a_exit(void) +{ + flush_scheduled_work(); + i2c_del_driver(&atmel_sha204a_driver); +} + +module_init(atmel_sha204a_init); +module_exit(atmel_sha204a_exit); + +MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 18410c9e7b29..869602fcfd96 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -85,7 +85,7 @@ MODULE_PARM_DESC(aead_pri, "Priority for AEAD algos"); * 0x70 - ring 2 * 0x78 - ring 3 */ -char BCMHEADER[] = { 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28 }; +static char BCMHEADER[] = { 0x60, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x28 }; /* * Some SPU hw does not use BCM header on SPU messages. So BCM_HDR_LEN * is set dynamically after reading SPU type from device tree. @@ -2083,7 +2083,7 @@ static int __ahash_init(struct ahash_request *req) * Return: true if incremental hashing is not supported * false otherwise */ -bool spu_no_incr_hash(struct iproc_ctx_s *ctx) +static bool spu_no_incr_hash(struct iproc_ctx_s *ctx) { struct spu_hw *spu = &iproc_priv.spu; @@ -4809,7 +4809,7 @@ static int spu_dt_read(struct platform_device *pdev) return 0; } -int bcm_spu_probe(struct platform_device *pdev) +static int bcm_spu_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct spu_hw *spu = &iproc_priv.spu; @@ -4853,7 +4853,7 @@ failure: return err; } -int bcm_spu_remove(struct platform_device *pdev) +static int bcm_spu_remove(struct platform_device *pdev) { int i; struct device *dev = &pdev->dev; diff --git a/drivers/crypto/bcm/spu2.c b/drivers/crypto/bcm/spu2.c index cb477259a2e2..2add51024575 100644 --- a/drivers/crypto/bcm/spu2.c +++ b/drivers/crypto/bcm/spu2.c @@ -38,21 +38,21 @@ enum spu2_proto_sel { SPU2_DTLS_AEAD = 10 }; -char *spu2_cipher_type_names[] = { "None", "AES128", "AES192", "AES256", +static char *spu2_cipher_type_names[] = { "None", "AES128", "AES192", "AES256", "DES", "3DES" }; -char *spu2_cipher_mode_names[] = { "ECB", "CBC", "CTR", "CFB", "OFB", "XTS", - "CCM", "GCM" +static char *spu2_cipher_mode_names[] = { "ECB", "CBC", "CTR", "CFB", "OFB", + "XTS", "CCM", "GCM" }; -char *spu2_hash_type_names[] = { "None", "AES128", "AES192", "AES256", +static char *spu2_hash_type_names[] = { "None", "AES128", "AES192", "AES256", "Reserved", "Reserved", "MD5", "SHA1", "SHA224", "SHA256", "SHA384", "SHA512", "SHA512/224", "SHA512/256", "SHA3-224", "SHA3-256", "SHA3-384", "SHA3-512" }; -char *spu2_hash_mode_names[] = { "CMAC", "CBC-MAC", "XCBC-MAC", "HMAC", +static char *spu2_hash_mode_names[] = { "CMAC", "CBC-MAC", "XCBC-MAC", "HMAC", "Rabin", "CCM", "GCM", "Reserved" }; diff --git a/drivers/crypto/caam/Kconfig b/drivers/crypto/caam/Kconfig index 577c9844b322..3720ddabb507 100644 --- a/drivers/crypto/caam/Kconfig +++ b/drivers/crypto/caam/Kconfig @@ -2,6 +2,12 @@ config CRYPTO_DEV_FSL_CAAM_COMMON tristate +config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC + tristate + +config CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC + tristate + config CRYPTO_DEV_FSL_CAAM tristate "Freescale CAAM-Multicore platform driver backend" depends on FSL_SOC || ARCH_MXC || ARCH_LAYERSCAPE @@ -25,7 +31,7 @@ config CRYPTO_DEV_FSL_CAAM_DEBUG Selecting this will enable printing of various debug information in the CAAM driver. -config CRYPTO_DEV_FSL_CAAM_JR +menuconfig CRYPTO_DEV_FSL_CAAM_JR tristate "Freescale CAAM Job Ring driver backend" default y help @@ -86,8 +92,9 @@ config CRYPTO_DEV_FSL_CAAM_INTC_TIME_THLD threshold. Range is 1-65535. config CRYPTO_DEV_FSL_CAAM_CRYPTO_API - tristate "Register algorithm implementations with the Crypto API" + bool "Register algorithm implementations with the Crypto API" default y + select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC select CRYPTO_AEAD select CRYPTO_AUTHENC select CRYPTO_BLKCIPHER @@ -97,13 +104,11 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API scatterlist crypto API (such as the linux native IPSec stack) to the SEC4 via job ring. - To compile this as a module, choose M here: the module - will be called caamalg. - config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI - tristate "Queue Interface as Crypto API backend" + bool "Queue Interface as Crypto API backend" depends on FSL_DPAA && NET default y + select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC select CRYPTO_AUTHENC select CRYPTO_BLKCIPHER help @@ -114,33 +119,26 @@ config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI assigned to the kernel should also be more than the number of job rings. - To compile this as a module, choose M here: the module - will be called caamalg_qi. - config CRYPTO_DEV_FSL_CAAM_AHASH_API - tristate "Register hash algorithm implementations with Crypto API" + bool "Register hash algorithm implementations with Crypto API" default y + select CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC select CRYPTO_HASH help Selecting this will offload ahash for users of the scatterlist crypto API to the SEC4 via job ring. - To compile this as a module, choose M here: the module - will be called caamhash. - config CRYPTO_DEV_FSL_CAAM_PKC_API - tristate "Register public key cryptography implementations with Crypto API" + bool "Register public key cryptography implementations with Crypto API" default y select CRYPTO_RSA help Selecting this will allow SEC Public key support for RSA. Supported cryptographic primitives: encryption, decryption, signature and verification. - To compile this as a module, choose M here: the module - will be called caam_pkc. config CRYPTO_DEV_FSL_CAAM_RNG_API - tristate "Register caam device for hwrng API" + bool "Register caam device for hwrng API" default y select CRYPTO_RNG select HW_RANDOM @@ -148,9 +146,6 @@ config CRYPTO_DEV_FSL_CAAM_RNG_API Selecting this will register the SEC4 hardware rng to the hw_random API for suppying the kernel entropy pool. - To compile this as a module, choose M here: the module - will be called caamrng. - endif # CRYPTO_DEV_FSL_CAAM_JR endif # CRYPTO_DEV_FSL_CAAM @@ -160,6 +155,8 @@ config CRYPTO_DEV_FSL_DPAA2_CAAM depends on FSL_MC_DPIO depends on NETDEVICES select CRYPTO_DEV_FSL_CAAM_COMMON + select CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC + select CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC select CRYPTO_BLKCIPHER select CRYPTO_AUTHENC select CRYPTO_AEAD @@ -171,12 +168,3 @@ config CRYPTO_DEV_FSL_DPAA2_CAAM To compile this as a module, choose M here: the module will be called dpaa2_caam. - -config CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC - def_tristate (CRYPTO_DEV_FSL_CAAM_CRYPTO_API || \ - CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI || \ - CRYPTO_DEV_FSL_DPAA2_CAAM) - -config CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC - def_tristate (CRYPTO_DEV_FSL_CAAM_AHASH_API || \ - CRYPTO_DEV_FSL_DPAA2_CAAM) diff --git a/drivers/crypto/caam/Makefile b/drivers/crypto/caam/Makefile index 7bbfd06a11ff..9ab4e81ea21e 100644 --- a/drivers/crypto/caam/Makefile +++ b/drivers/crypto/caam/Makefile @@ -11,20 +11,20 @@ ccflags-y += -DVERSION=\"\" obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_COMMON) += error.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM) += caam.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_JR) += caam_jr.o -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += caamalg_qi.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_DESC) += caamalg_desc.o -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API_DESC) += caamhash_desc.o -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o -obj-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caam_pkc.o -caam-objs := ctrl.o -caam_jr-objs := jr.o key_gen.o -caam_pkc-y := caampkc.o pkc_desc.o +caam-y := ctrl.o +caam_jr-y := jr.o key_gen.o +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API) += caamalg.o +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += caamalg_qi.o +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API) += caamhash.o +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API) += caamrng.o +caam_jr-$(CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API) += caampkc.o pkc_desc.o + +caam-$(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI) += qi.o ifneq ($(CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API_QI),) ccflags-y += -DCONFIG_CAAM_QI - caam-objs += qi.o endif obj-$(CONFIG_CRYPTO_DEV_FSL_DPAA2_CAAM) += dpaa2_caam.o diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index c0ece44f303b..43f18253e5b6 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -77,13 +77,6 @@ #define DESC_MAX_USED_BYTES (CAAM_DESC_BYTES_MAX - DESC_JOB_IO_LEN) #define DESC_MAX_USED_LEN (DESC_MAX_USED_BYTES / CAAM_CMD_SZ) -#ifdef DEBUG -/* for print_hex_dumps with line references */ -#define debug(format, arg...) printk(format, arg) -#else -#define debug(format, arg...) -#endif - struct caam_alg_entry { int class1_alg_type; int class2_alg_type; @@ -583,13 +576,11 @@ static int aead_setkey(struct crypto_aead *aead, if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) goto badkey; -#ifdef DEBUG - printk(KERN_ERR "keylen %d enckeylen %d authkeylen %d\n", + dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n", keys.authkeylen + keys.enckeylen, keys.enckeylen, keys.authkeylen); - print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); /* * If DKP is supported, use it in the shared descriptor to generate @@ -623,11 +614,10 @@ static int aead_setkey(struct crypto_aead *aead, memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + keys.enckeylen, ctx->dir); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, - ctx->adata.keylen_pad + keys.enckeylen, 1); -#endif + + print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, + ctx->adata.keylen_pad + keys.enckeylen, 1); skip_split_key: ctx->cdata.keylen = keys.enckeylen; @@ -678,10 +668,8 @@ static int gcm_setkey(struct crypto_aead *aead, struct caam_ctx *ctx = crypto_aead_ctx(aead); struct device *jrdev = ctx->jrdev; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir); @@ -699,10 +687,8 @@ static int rfc4106_setkey(struct crypto_aead *aead, if (keylen < 4) return -EINVAL; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -725,10 +711,8 @@ static int rfc4543_setkey(struct crypto_aead *aead, if (keylen < 4) return -EINVAL; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); @@ -757,10 +741,8 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, OP_ALG_AAI_CTR_MOD128); const bool is_rfc3686 = alg->caam.rfc3686; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); /* * AES-CTR needs to load IV in CONTEXT1 reg * at an offset of 128bits (16bytes) @@ -916,7 +898,7 @@ static void caam_unmap(struct device *dev, struct scatterlist *src, } if (iv_dma) - dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); + dma_unmap_single(dev, iv_dma, ivsize, DMA_BIDIRECTIONAL); if (sec4_sg_bytes) dma_unmap_single(dev, sec4_sg_dma, sec4_sg_bytes, DMA_TO_DEVICE); @@ -949,9 +931,7 @@ static void aead_encrypt_done(struct device *jrdev, u32 *desc, u32 err, struct aead_request *req = context; struct aead_edesc *edesc; -#ifdef DEBUG - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct aead_edesc, hw_desc[0]); @@ -971,9 +951,7 @@ static void aead_decrypt_done(struct device *jrdev, u32 *desc, u32 err, struct aead_request *req = context; struct aead_edesc *edesc; -#ifdef DEBUG - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct aead_edesc, hw_desc[0]); @@ -1001,33 +979,32 @@ static void skcipher_encrypt_done(struct device *jrdev, u32 *desc, u32 err, struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); int ivsize = crypto_skcipher_ivsize(skcipher); -#ifdef DEBUG - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct skcipher_edesc, hw_desc[0]); if (err) caam_jr_strstatus(jrdev, err); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, - edesc->src_nents > 1 ? 100 : ivsize, 1); -#endif - caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->dst, - edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); - skcipher_unmap(jrdev, edesc, req); /* * The crypto API expects us to set the IV (req->iv) to the last - * ciphertext block. This is used e.g. by the CTS mode. + * ciphertext block (CBC mode) or last counter (CTR mode). + * This is used e.g. by the CTS mode. */ - if (ivsize) - scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - - ivsize, ivsize, 0); + if (ivsize) { + memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes, + ivsize); + + print_hex_dump_debug("dstiv @"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, + edesc->src_nents > 1 ? 100 : ivsize, 1); + } + + caam_dump_sg("dst @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->dst, + edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); kfree(edesc); @@ -1039,26 +1016,35 @@ static void skcipher_decrypt_done(struct device *jrdev, u32 *desc, u32 err, { struct skcipher_request *req = context; struct skcipher_edesc *edesc; -#ifdef DEBUG struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); int ivsize = crypto_skcipher_ivsize(skcipher); - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct skcipher_edesc, hw_desc[0]); if (err) caam_jr_strstatus(jrdev, err); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "dstiv @"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); -#endif - caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", + skcipher_unmap(jrdev, edesc, req); + + /* + * The crypto API expects us to set the IV (req->iv) to the last + * ciphertext block (CBC mode) or last counter (CTR mode). + * This is used e.g. by the CTS mode. + */ + if (ivsize) { + memcpy(req->iv, (u8 *)edesc->sec4_sg + edesc->sec4_sg_bytes, + ivsize); + + print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, + ivsize, 1); + } + + caam_dump_sg("dst @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->dst, edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); - skcipher_unmap(jrdev, edesc, req); kfree(edesc); skcipher_request_complete(req, err); @@ -1106,6 +1092,7 @@ static void init_aead_job(struct aead_request *req, if (unlikely(req->src != req->dst)) { if (!edesc->mapped_dst_nents) { dst_dma = 0; + out_options = 0; } else if (edesc->mapped_dst_nents == 1) { dst_dma = sg_dma_address(req->dst); out_options = 0; @@ -1249,6 +1236,7 @@ static void init_skcipher_job(struct skcipher_request *req, { struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); + struct device *jrdev = ctx->jrdev; int ivsize = crypto_skcipher_ivsize(skcipher); u32 *desc = edesc->hw_desc; u32 *sh_desc; @@ -1256,13 +1244,12 @@ static void init_skcipher_job(struct skcipher_request *req, dma_addr_t src_dma, dst_dma, ptr; int len, sec4_sg_index = 0; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "presciv@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); - pr_err("asked=%d, cryptlen%d\n", + print_hex_dump_debug("presciv@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, ivsize, 1); + dev_dbg(jrdev, "asked=%d, cryptlen%d\n", (int)edesc->src_nents > 1 ? 100 : req->cryptlen, req->cryptlen); -#endif - caam_dump_sg(KERN_ERR, "src @" __stringify(__LINE__)": ", + + caam_dump_sg("src @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->src, edesc->src_nents > 1 ? 100 : req->cryptlen, 1); @@ -1285,7 +1272,7 @@ static void init_skcipher_job(struct skcipher_request *req, if (likely(req->src == req->dst)) { dst_dma = src_dma + !!ivsize * sizeof(struct sec4_sg_entry); out_options = in_options; - } else if (edesc->mapped_dst_nents == 1) { + } else if (!ivsize && edesc->mapped_dst_nents == 1) { dst_dma = sg_dma_address(req->dst); } else { dst_dma = edesc->sec4_sg_dma + sec4_sg_index * @@ -1293,7 +1280,7 @@ static void init_skcipher_job(struct skcipher_request *req, out_options = LDST_SGF; } - append_seq_out_ptr(desc, dst_dma, req->cryptlen, out_options); + append_seq_out_ptr(desc, dst_dma, req->cryptlen + ivsize, out_options); } /* @@ -1309,37 +1296,36 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + int src_len, dst_len = 0; struct aead_edesc *edesc; int sec4_sg_index, sec4_sg_len, sec4_sg_bytes; unsigned int authsize = ctx->authsize; if (unlikely(req->dst != req->src)) { - src_nents = sg_nents_for_len(req->src, req->assoclen + - req->cryptlen); + src_len = req->assoclen + req->cryptlen; + dst_len = src_len + (encrypt ? authsize : (-authsize)); + + src_nents = sg_nents_for_len(req->src, src_len); if (unlikely(src_nents < 0)) { dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", - req->assoclen + req->cryptlen); + src_len); return ERR_PTR(src_nents); } - dst_nents = sg_nents_for_len(req->dst, req->assoclen + - req->cryptlen + - (encrypt ? authsize : - (-authsize))); + dst_nents = sg_nents_for_len(req->dst, dst_len); if (unlikely(dst_nents < 0)) { dev_err(jrdev, "Insufficient bytes (%d) in dst S/G\n", - req->assoclen + req->cryptlen + - (encrypt ? authsize : (-authsize))); + dst_len); return ERR_PTR(dst_nents); } } else { - src_nents = sg_nents_for_len(req->src, req->assoclen + - req->cryptlen + - (encrypt ? authsize : 0)); + src_len = req->assoclen + req->cryptlen + + (encrypt ? authsize : 0); + + src_nents = sg_nents_for_len(req->src, src_len); if (unlikely(src_nents < 0)) { dev_err(jrdev, "Insufficient bytes (%d) in src S/G\n", - req->assoclen + req->cryptlen + - (encrypt ? authsize : 0)); + src_len); return ERR_PTR(src_nents); } } @@ -1380,8 +1366,16 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, } } + /* + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. + */ sec4_sg_len = mapped_src_nents > 1 ? mapped_src_nents : 0; - sec4_sg_len += mapped_dst_nents > 1 ? mapped_dst_nents : 0; + if (mapped_dst_nents > 1) + sec4_sg_len += pad_sg_nents(mapped_dst_nents); + else + sec4_sg_len = pad_sg_nents(sec4_sg_len); + sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); /* allocate space for base edesc and hw desc commands, link tables */ @@ -1403,12 +1397,12 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, sec4_sg_index = 0; if (mapped_src_nents > 1) { - sg_to_sec4_sg_last(req->src, mapped_src_nents, + sg_to_sec4_sg_last(req->src, src_len, edesc->sec4_sg + sec4_sg_index, 0); sec4_sg_index += mapped_src_nents; } if (mapped_dst_nents > 1) { - sg_to_sec4_sg_last(req->dst, mapped_dst_nents, + sg_to_sec4_sg_last(req->dst, dst_len, edesc->sec4_sg + sec4_sg_index, 0); } @@ -1446,11 +1440,10 @@ static int gcm_encrypt(struct aead_request *req) /* Create and submit job descriptor */ init_gcm_job(req, edesc, all_contig, true); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); -#endif + + print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); desc = edesc->hw_desc; ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); @@ -1556,11 +1549,10 @@ static int aead_encrypt(struct aead_request *req) /* Create and submit job descriptor */ init_authenc_job(req, edesc, all_contig, true); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); -#endif + + print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); desc = edesc->hw_desc; ret = caam_jr_enqueue(jrdev, desc, aead_encrypt_done, req); @@ -1591,11 +1583,10 @@ static int gcm_decrypt(struct aead_request *req) /* Create and submit job descriptor*/ init_gcm_job(req, edesc, all_contig, false); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); -#endif + + print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); desc = edesc->hw_desc; ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); @@ -1627,7 +1618,7 @@ static int aead_decrypt(struct aead_request *req) u32 *desc; int ret = 0; - caam_dump_sg(KERN_ERR, "dec src@" __stringify(__LINE__)": ", + caam_dump_sg("dec src@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->src, req->assoclen + req->cryptlen, 1); @@ -1639,11 +1630,10 @@ static int aead_decrypt(struct aead_request *req) /* Create and submit job descriptor*/ init_authenc_job(req, edesc, all_contig, false); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "aead jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); -#endif + + print_hex_dump_debug("aead jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); desc = edesc->hw_desc; ret = caam_jr_enqueue(jrdev, desc, aead_decrypt_done, req); @@ -1719,7 +1709,29 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, else sec4_sg_ents = mapped_src_nents + !!ivsize; dst_sg_idx = sec4_sg_ents; - sec4_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; + + /* + * Input, output HW S/G tables: [IV, src][dst, IV] + * IV entries point to the same buffer + * If src == dst, S/G entries are reused (S/G tables overlap) + * + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. Logic: + * if (output S/G) + * pad output S/G, if needed + * else if (input S/G) ... + * pad input S/G, if needed + */ + if (ivsize || mapped_dst_nents > 1) { + if (req->src == req->dst) + sec4_sg_ents = !!ivsize + pad_sg_nents(sec4_sg_ents); + else + sec4_sg_ents += pad_sg_nents(mapped_dst_nents + + !!ivsize); + } else { + sec4_sg_ents = pad_sg_nents(sec4_sg_ents); + } + sec4_sg_bytes = sec4_sg_ents * sizeof(struct sec4_sg_entry); /* @@ -1744,10 +1756,10 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, /* Make sure IV is located in a DMAable area */ if (ivsize) { - iv = (u8 *)edesc->hw_desc + desc_bytes + sec4_sg_bytes; + iv = (u8 *)edesc->sec4_sg + sec4_sg_bytes; memcpy(iv, req->iv, ivsize); - iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_TO_DEVICE); + iv_dma = dma_map_single(jrdev, iv, ivsize, DMA_BIDIRECTIONAL); if (dma_mapping_error(jrdev, iv_dma)) { dev_err(jrdev, "unable to map IV\n"); caam_unmap(jrdev, req->src, req->dst, src_nents, @@ -1759,13 +1771,20 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, dma_to_sec4_sg_one(edesc->sec4_sg, iv_dma, ivsize, 0); } if (dst_sg_idx) - sg_to_sec4_sg_last(req->src, mapped_src_nents, edesc->sec4_sg + - !!ivsize, 0); + sg_to_sec4_sg(req->src, req->cryptlen, edesc->sec4_sg + + !!ivsize, 0); - if (mapped_dst_nents > 1) { - sg_to_sec4_sg_last(req->dst, mapped_dst_nents, - edesc->sec4_sg + dst_sg_idx, 0); - } + if (req->src != req->dst && (ivsize || mapped_dst_nents > 1)) + sg_to_sec4_sg(req->dst, req->cryptlen, edesc->sec4_sg + + dst_sg_idx, 0); + + if (ivsize) + dma_to_sec4_sg_one(edesc->sec4_sg + dst_sg_idx + + mapped_dst_nents, iv_dma, ivsize, 0); + + if (ivsize || mapped_dst_nents > 1) + sg_to_sec4_set_last(edesc->sec4_sg + dst_sg_idx + + mapped_dst_nents); if (sec4_sg_bytes) { edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, @@ -1782,11 +1801,9 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, edesc->iv_dma = iv_dma; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "skcipher sec4_sg@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, - sec4_sg_bytes, 1); -#endif + print_hex_dump_debug("skcipher sec4_sg@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, + sec4_sg_bytes, 1); return edesc; } @@ -1807,11 +1824,11 @@ static int skcipher_encrypt(struct skcipher_request *req) /* Create and submit job descriptor*/ init_skcipher_job(req, edesc, true); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "skcipher jobdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); -#endif + + print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); + desc = edesc->hw_desc; ret = caam_jr_enqueue(jrdev, desc, skcipher_encrypt_done, req); @@ -1830,7 +1847,6 @@ static int skcipher_decrypt(struct skcipher_request *req) struct skcipher_edesc *edesc; struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); - int ivsize = crypto_skcipher_ivsize(skcipher); struct device *jrdev = ctx->jrdev; u32 *desc; int ret = 0; @@ -1840,22 +1856,13 @@ static int skcipher_decrypt(struct skcipher_request *req) if (IS_ERR(edesc)) return PTR_ERR(edesc); - /* - * The crypto API expects us to set the IV (req->iv) to the last - * ciphertext block. - */ - if (ivsize) - scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen - - ivsize, ivsize, 0); - /* Create and submit job descriptor*/ init_skcipher_job(req, edesc, false); desc = edesc->hw_desc; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "skcipher jobdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, - desc_bytes(edesc->hw_desc), 1); -#endif + + print_hex_dump_debug("skcipher jobdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->hw_desc, + desc_bytes(edesc->hw_desc), 1); ret = caam_jr_enqueue(jrdev, desc, skcipher_decrypt_done, req); if (!ret) { @@ -3444,7 +3451,7 @@ static void caam_aead_exit(struct crypto_aead *tfm) caam_exit_common(crypto_aead_ctx(tfm)); } -static void __exit caam_algapi_exit(void) +void caam_algapi_exit(void) { int i; @@ -3489,43 +3496,15 @@ static void caam_aead_alg_init(struct caam_aead_alg *t_alg) alg->exit = caam_aead_exit; } -static int __init caam_algapi_init(void) +int caam_algapi_init(struct device *ctrldev) { - struct device_node *dev_node; - struct platform_device *pdev; - struct caam_drv_private *priv; + struct caam_drv_private *priv = dev_get_drvdata(ctrldev); int i = 0, err = 0; u32 aes_vid, aes_inst, des_inst, md_vid, md_inst, ccha_inst, ptha_inst; u32 arc4_inst; unsigned int md_limit = SHA512_DIGEST_SIZE; bool registered = false, gcm_support; - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); - if (!dev_node) - return -ENODEV; - } - - pdev = of_find_device_by_node(dev_node); - if (!pdev) { - of_node_put(dev_node); - return -ENODEV; - } - - priv = dev_get_drvdata(&pdev->dev); - of_node_put(dev_node); - - /* - * If priv is NULL, it's probably because the caam driver wasn't - * properly initialized (e.g. RNG4 init failed). Thus, bail out here. - */ - if (!priv) { - err = -ENODEV; - goto out_put_dev; - } - - /* * Register crypto algorithms the device supports. * First, detect presence and attributes of DES, AES, and MD blocks. @@ -3668,14 +3647,5 @@ static int __init caam_algapi_init(void) if (registered) pr_info("caam algorithms registered in /proc/crypto\n"); -out_put_dev: - put_device(&pdev->dev); return err; } - -module_init(caam_algapi_init); -module_exit(caam_algapi_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("FSL CAAM support for crypto API"); -MODULE_AUTHOR("Freescale Semiconductor - NMG/STC"); diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c index 1e1a376edc2f..72531837571e 100644 --- a/drivers/crypto/caam/caamalg_desc.c +++ b/drivers/crypto/caam/caamalg_desc.c @@ -33,12 +33,11 @@ static inline void append_dec_op1(u32 *desc, u32 type) } jump_cmd = append_jump(desc, JUMP_TEST_ALL | JUMP_COND_SHRD); - append_operation(desc, type | OP_ALG_AS_INITFINAL | - OP_ALG_DECRYPT); + append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT); uncond_jump_cmd = append_jump(desc, JUMP_TEST_ALL); set_jump_tgt_here(desc, jump_cmd); - append_operation(desc, type | OP_ALG_AS_INITFINAL | - OP_ALG_DECRYPT | OP_ALG_AAI_DK); + append_operation(desc, type | OP_ALG_AS_INIT | OP_ALG_DECRYPT | + OP_ALG_AAI_DK); set_jump_tgt_here(desc, uncond_jump_cmd); } @@ -115,11 +114,9 @@ void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_CONTEXT); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "aead null enc shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("aead null enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_aead_null_encap); @@ -204,11 +201,9 @@ void cnstr_shdsc_aead_null_decap(u32 * const desc, struct alginfo *adata, append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "aead null dec shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("aead null dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_aead_null_decap); @@ -358,10 +353,9 @@ void cnstr_shdsc_aead_encap(u32 * const desc, struct alginfo *cdata, append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_CONTEXT); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "aead enc shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("aead enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_aead_encap); @@ -475,10 +469,9 @@ void cnstr_shdsc_aead_decap(u32 * const desc, struct alginfo *cdata, append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS2 | FIFOLD_TYPE_LAST2 | FIFOLD_TYPE_ICV); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "aead dec shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("aead dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_aead_decap); @@ -613,11 +606,9 @@ copy_iv: append_seq_store(desc, icvsize, LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_CONTEXT); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "aead givenc shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("aead givenc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_aead_givencap); @@ -742,10 +733,9 @@ void cnstr_shdsc_gcm_encap(u32 * const desc, struct alginfo *cdata, append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "gcm enc shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("gcm enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_gcm_encap); @@ -838,10 +828,9 @@ void cnstr_shdsc_gcm_decap(u32 * const desc, struct alginfo *cdata, append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "gcm dec shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("gcm dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_gcm_decap); @@ -933,11 +922,9 @@ void cnstr_shdsc_rfc4106_encap(u32 * const desc, struct alginfo *cdata, append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "rfc4106 enc shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("rfc4106 enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_rfc4106_encap); @@ -1030,11 +1017,9 @@ void cnstr_shdsc_rfc4106_decap(u32 * const desc, struct alginfo *cdata, append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "rfc4106 dec shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("rfc4106 dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_rfc4106_decap); @@ -1115,11 +1100,9 @@ void cnstr_shdsc_rfc4543_encap(u32 * const desc, struct alginfo *cdata, append_seq_store(desc, icvsize, LDST_CLASS_1_CCB | LDST_SRCDST_BYTE_CONTEXT); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "rfc4543 enc shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("rfc4543 enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_rfc4543_encap); @@ -1205,11 +1188,9 @@ void cnstr_shdsc_rfc4543_decap(u32 * const desc, struct alginfo *cdata, append_seq_fifo_load(desc, icvsize, FIFOLD_CLASS_CLASS1 | FIFOLD_TYPE_ICV | FIFOLD_TYPE_LAST1); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "rfc4543 dec shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("rfc4543 dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_rfc4543_decap); @@ -1410,17 +1391,21 @@ void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata, LDST_OFFSET_SHIFT)); /* Load operation */ - append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + append_operation(desc, cdata->algtype | OP_ALG_AS_INIT | OP_ALG_ENCRYPT); /* Perform operation */ skcipher_append_src_dst(desc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "skcipher enc shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + /* Store IV */ + if (ivsize) + append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB | (ctx1_iv_off << + LDST_OFFSET_SHIFT)); + + print_hex_dump_debug("skcipher enc shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_skcipher_encap); @@ -1479,7 +1464,7 @@ void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata, /* Choose operation */ if (ctx1_iv_off) - append_operation(desc, cdata->algtype | OP_ALG_AS_INITFINAL | + append_operation(desc, cdata->algtype | OP_ALG_AS_INIT | OP_ALG_DECRYPT); else append_dec_op1(desc, cdata->algtype); @@ -1487,11 +1472,15 @@ void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata, /* Perform operation */ skcipher_append_src_dst(desc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "skcipher dec shdesc@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + /* Store IV */ + if (ivsize) + append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB | (ctx1_iv_off << + LDST_OFFSET_SHIFT)); + + print_hex_dump_debug("skcipher dec shdesc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); } EXPORT_SYMBOL(cnstr_shdsc_skcipher_decap); @@ -1538,11 +1527,13 @@ void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata) /* Perform operation */ skcipher_append_src_dst(desc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "xts skcipher enc shdesc@" __stringify(__LINE__) ": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + /* Store upper 8B of IV */ + append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x20 << LDST_OFFSET_SHIFT)); + + print_hex_dump_debug("xts skcipher enc shdesc@" __stringify(__LINE__) + ": ", DUMP_PREFIX_ADDRESS, 16, 4, + desc, desc_bytes(desc), 1); } EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_encap); @@ -1588,11 +1579,13 @@ void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata) /* Perform operation */ skcipher_append_src_dst(desc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "xts skcipher dec shdesc@" __stringify(__LINE__) ": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + /* Store upper 8B of IV */ + append_seq_store(desc, 8, LDST_SRCDST_BYTE_CONTEXT | LDST_CLASS_1_CCB | + (0x20 << LDST_OFFSET_SHIFT)); + + print_hex_dump_debug("xts skcipher dec shdesc@" __stringify(__LINE__) + ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); } EXPORT_SYMBOL(cnstr_shdsc_xts_skcipher_decap); diff --git a/drivers/crypto/caam/caamalg_desc.h b/drivers/crypto/caam/caamalg_desc.h index d5ca42ff961a..da4a4ee60c80 100644 --- a/drivers/crypto/caam/caamalg_desc.h +++ b/drivers/crypto/caam/caamalg_desc.h @@ -44,9 +44,9 @@ #define DESC_SKCIPHER_BASE (3 * CAAM_CMD_SZ) #define DESC_SKCIPHER_ENC_LEN (DESC_SKCIPHER_BASE + \ - 20 * CAAM_CMD_SZ) + 21 * CAAM_CMD_SZ) #define DESC_SKCIPHER_DEC_LEN (DESC_SKCIPHER_BASE + \ - 15 * CAAM_CMD_SZ) + 16 * CAAM_CMD_SZ) void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, unsigned int icvsize, int era); diff --git a/drivers/crypto/caam/caamalg_qi.c b/drivers/crypto/caam/caamalg_qi.c index d290d6b41825..32f0f8a72067 100644 --- a/drivers/crypto/caam/caamalg_qi.c +++ b/drivers/crypto/caam/caamalg_qi.c @@ -4,7 +4,7 @@ * Based on caamalg.c * * Copyright 2013-2016 Freescale Semiconductor, Inc. - * Copyright 2016-2018 NXP + * Copyright 2016-2019 NXP */ #include "compat.h" @@ -214,13 +214,11 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key, if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) goto badkey; -#ifdef DEBUG - dev_err(jrdev, "keylen %d enckeylen %d authkeylen %d\n", + dev_dbg(jrdev, "keylen %d enckeylen %d authkeylen %d\n", keys.authkeylen + keys.enckeylen, keys.enckeylen, keys.authkeylen); - print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); /* * If DKP is supported, use it in the shared descriptor to generate @@ -237,7 +235,7 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key, memcpy(ctx->key, keys.authkey, keys.authkeylen); memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); - dma_sync_single_for_device(jrdev, ctx->key_dma, + dma_sync_single_for_device(jrdev->parent, ctx->key_dma, ctx->adata.keylen_pad + keys.enckeylen, ctx->dir); goto skip_split_key; @@ -251,8 +249,9 @@ static int aead_setkey(struct crypto_aead *aead, const u8 *key, /* postpend encryption key to auth split key */ memcpy(ctx->key + ctx->adata.keylen_pad, keys.enckey, keys.enckeylen); - dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->adata.keylen_pad + - keys.enckeylen, ctx->dir); + dma_sync_single_for_device(jrdev->parent, ctx->key_dma, + ctx->adata.keylen_pad + keys.enckeylen, + ctx->dir); #ifdef DEBUG print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, ctx->key, @@ -386,13 +385,12 @@ static int gcm_setkey(struct crypto_aead *aead, struct device *jrdev = ctx->jrdev; int ret; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); - dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, ctx->dir); + dma_sync_single_for_device(jrdev->parent, ctx->key_dma, keylen, + ctx->dir); ctx->cdata.keylen = keylen; ret = gcm_set_sh_desc(aead); @@ -485,10 +483,8 @@ static int rfc4106_setkey(struct crypto_aead *aead, if (keylen < 4) return -EINVAL; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); /* @@ -496,8 +492,8 @@ static int rfc4106_setkey(struct crypto_aead *aead, * in the nonce. Update the AES key length. */ ctx->cdata.keylen = keylen - 4; - dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, - ctx->dir); + dma_sync_single_for_device(jrdev->parent, ctx->key_dma, + ctx->cdata.keylen, ctx->dir); ret = rfc4106_set_sh_desc(aead); if (ret) @@ -589,10 +585,8 @@ static int rfc4543_setkey(struct crypto_aead *aead, if (keylen < 4) return -EINVAL; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); memcpy(ctx->key, key, keylen); /* @@ -600,8 +594,8 @@ static int rfc4543_setkey(struct crypto_aead *aead, * in the nonce. Update the AES key length. */ ctx->cdata.keylen = keylen - 4; - dma_sync_single_for_device(jrdev, ctx->key_dma, ctx->cdata.keylen, - ctx->dir); + dma_sync_single_for_device(jrdev->parent, ctx->key_dma, + ctx->cdata.keylen, ctx->dir); ret = rfc4543_set_sh_desc(aead); if (ret) @@ -644,10 +638,9 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, const bool is_rfc3686 = alg->caam.rfc3686; int ret = 0; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key in @" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); -#endif + print_hex_dump_debug("key in @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); + /* * AES-CTR needs to load IV in CONTEXT1 reg * at an offset of 128bits (16bytes) @@ -838,7 +831,8 @@ static struct caam_drv_ctx *get_drv_ctx(struct caam_ctx *ctx, static void caam_unmap(struct device *dev, struct scatterlist *src, struct scatterlist *dst, int src_nents, int dst_nents, dma_addr_t iv_dma, int ivsize, - dma_addr_t qm_sg_dma, int qm_sg_bytes) + enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma, + int qm_sg_bytes) { if (dst != src) { if (src_nents) @@ -850,7 +844,7 @@ static void caam_unmap(struct device *dev, struct scatterlist *src, } if (iv_dma) - dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); + dma_unmap_single(dev, iv_dma, ivsize, iv_dir); if (qm_sg_bytes) dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); } @@ -863,7 +857,8 @@ static void aead_unmap(struct device *dev, int ivsize = crypto_aead_ivsize(aead); caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, - edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes); + edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma, + edesc->qm_sg_bytes); dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); } @@ -874,7 +869,8 @@ static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, int ivsize = crypto_skcipher_ivsize(skcipher); caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, - edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes); + edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma, + edesc->qm_sg_bytes); } static void aead_done(struct caam_drv_req *drv_req, u32 status) @@ -924,6 +920,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + int src_len, dst_len = 0; struct aead_edesc *edesc; dma_addr_t qm_sg_dma, iv_dma = 0; int ivsize = 0; @@ -945,13 +942,13 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, } if (likely(req->src == req->dst)) { - src_nents = sg_nents_for_len(req->src, req->assoclen + - req->cryptlen + - (encrypt ? authsize : 0)); + src_len = req->assoclen + req->cryptlen + + (encrypt ? authsize : 0); + + src_nents = sg_nents_for_len(req->src, src_len); if (unlikely(src_nents < 0)) { dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", - req->assoclen + req->cryptlen + - (encrypt ? authsize : 0)); + src_len); qi_cache_free(edesc); return ERR_PTR(src_nents); } @@ -964,23 +961,21 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, return ERR_PTR(-ENOMEM); } } else { - src_nents = sg_nents_for_len(req->src, req->assoclen + - req->cryptlen); + src_len = req->assoclen + req->cryptlen; + dst_len = src_len + (encrypt ? authsize : (-authsize)); + + src_nents = sg_nents_for_len(req->src, src_len); if (unlikely(src_nents < 0)) { dev_err(qidev, "Insufficient bytes (%d) in src S/G\n", - req->assoclen + req->cryptlen); + src_len); qi_cache_free(edesc); return ERR_PTR(src_nents); } - dst_nents = sg_nents_for_len(req->dst, req->assoclen + - req->cryptlen + - (encrypt ? authsize : - (-authsize))); + dst_nents = sg_nents_for_len(req->dst, dst_len); if (unlikely(dst_nents < 0)) { dev_err(qidev, "Insufficient bytes (%d) in dst S/G\n", - req->assoclen + req->cryptlen + - (encrypt ? authsize : (-authsize))); + dst_len); qi_cache_free(edesc); return ERR_PTR(dst_nents); } @@ -1019,9 +1014,24 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, /* * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. * Input is not contiguous. + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. Logic: + * if (src != dst && output S/G) + * pad output S/G, if needed + * else if (src == dst && S/G) + * overlapping S/Gs; pad one of them + * else if (input S/G) ... + * pad input S/G, if needed */ - qm_sg_ents = 1 + !!ivsize + mapped_src_nents + - (mapped_dst_nents > 1 ? mapped_dst_nents : 0); + qm_sg_ents = 1 + !!ivsize + mapped_src_nents; + if (mapped_dst_nents > 1) + qm_sg_ents += pad_sg_nents(mapped_dst_nents); + else if ((req->src == req->dst) && (mapped_src_nents > 1)) + qm_sg_ents = max(pad_sg_nents(qm_sg_ents), + 1 + !!ivsize + pad_sg_nents(mapped_src_nents)); + else + qm_sg_ents = pad_sg_nents(qm_sg_ents); + sg_table = &edesc->sgt[0]; qm_sg_bytes = qm_sg_ents * sizeof(*sg_table); if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > @@ -1029,7 +1039,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n", qm_sg_ents, ivsize); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1044,7 +1054,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, if (dma_mapping_error(qidev, iv_dma)) { dev_err(qidev, "unable to map IV\n"); caam_unmap(qidev, req->src, req->dst, src_nents, - dst_nents, 0, 0, 0, 0); + dst_nents, 0, 0, DMA_NONE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1063,7 +1073,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, if (dma_mapping_error(qidev, edesc->assoclen_dma)) { dev_err(qidev, "unable to map assoclen\n"); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, - iv_dma, ivsize, 0, 0); + iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1074,19 +1084,18 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); qm_sg_index++; } - sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0); + sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0); qm_sg_index += mapped_src_nents; if (mapped_dst_nents > 1) - sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + - qm_sg_index, 0); + sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0); qm_sg_dma = dma_map_single(qidev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); if (dma_mapping_error(qidev, qm_sg_dma)) { dev_err(qidev, "unable to map S/G table\n"); dma_unmap_single(qidev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, - iv_dma, ivsize, 0, 0); + iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1109,7 +1118,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dma_to_qm_sg_one_ext(&fd_sgt[0], qm_sg_dma + (1 + !!ivsize) * sizeof(*sg_table), out_len, 0); - } else if (mapped_dst_nents == 1) { + } else if (mapped_dst_nents <= 1) { dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), out_len, 0); } else { @@ -1182,33 +1191,28 @@ static void skcipher_done(struct caam_drv_req *drv_req, u32 status) struct device *qidev = caam_ctx->qidev; int ivsize = crypto_skcipher_ivsize(skcipher); -#ifdef DEBUG - dev_err(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status); -#endif + dev_dbg(qidev, "%s %d: status 0x%x\n", __func__, __LINE__, status); edesc = container_of(drv_req, typeof(*edesc), drv_req); if (status) caam_jr_strstatus(qidev, status); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "dstiv @" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->iv, - edesc->src_nents > 1 ? 100 : ivsize, 1); - caam_dump_sg(KERN_ERR, "dst @" __stringify(__LINE__)": ", + print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->iv, + edesc->src_nents > 1 ? 100 : ivsize, 1); + caam_dump_sg("dst @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->dst, edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); -#endif skcipher_unmap(qidev, edesc, req); /* * The crypto API expects us to set the IV (req->iv) to the last - * ciphertext block. This is used e.g. by the CTS mode. + * ciphertext block (CBC mode) or last counter (CTR mode). + * This is used e.g. by the CTS mode. */ - if (edesc->drv_req.drv_ctx->op_type == ENCRYPT) - scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - - ivsize, ivsize, 0); + memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, ivsize); qi_cache_free(edesc); skcipher_request_complete(req, status); @@ -1276,14 +1280,26 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, qm_sg_ents = 1 + mapped_src_nents; dst_sg_idx = qm_sg_ents; - qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; + /* + * Input, output HW S/G tables: [IV, src][dst, IV] + * IV entries point to the same buffer + * If src == dst, S/G entries are reused (S/G tables overlap) + * + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. + */ + if (req->src != req->dst) + qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1); + else + qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents); + qm_sg_bytes = qm_sg_ents * sizeof(struct qm_sg_entry); if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes + ivsize > CAAM_QI_MEMCACHE_SIZE)) { dev_err(qidev, "No space for %d S/G entries and/or %dB IV\n", qm_sg_ents, ivsize); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); return ERR_PTR(-ENOMEM); } @@ -1292,7 +1308,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, if (unlikely(!edesc)) { dev_err(qidev, "could not allocate extended descriptor\n"); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); return ERR_PTR(-ENOMEM); } @@ -1301,11 +1317,11 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, iv = (u8 *)(sg_table + qm_sg_ents); memcpy(iv, req->iv, ivsize); - iv_dma = dma_map_single(qidev, iv, ivsize, DMA_TO_DEVICE); + iv_dma = dma_map_single(qidev, iv, ivsize, DMA_BIDIRECTIONAL); if (dma_mapping_error(qidev, iv_dma)) { dev_err(qidev, "unable to map IV\n"); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1319,18 +1335,20 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, edesc->drv_req.drv_ctx = drv_ctx; dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); - sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0); + sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0); - if (mapped_dst_nents > 1) - sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + - dst_sg_idx, 0); + if (req->src != req->dst) + sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0); + + dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma, + ivsize, 0); edesc->qm_sg_dma = dma_map_single(qidev, sg_table, edesc->qm_sg_bytes, DMA_TO_DEVICE); if (dma_mapping_error(qidev, edesc->qm_sg_dma)) { dev_err(qidev, "unable to map S/G table\n"); caam_unmap(qidev, req->src, req->dst, src_nents, dst_nents, - iv_dma, ivsize, 0, 0); + iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1340,16 +1358,14 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req, dma_to_qm_sg_one_last_ext(&fd_sgt[1], edesc->qm_sg_dma, ivsize + req->cryptlen, 0); - if (req->src == req->dst) { + if (req->src == req->dst) dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + - sizeof(*sg_table), req->cryptlen, 0); - } else if (mapped_dst_nents > 1) { + sizeof(*sg_table), req->cryptlen + ivsize, + 0); + else dma_to_qm_sg_one_ext(&fd_sgt[0], edesc->qm_sg_dma + dst_sg_idx * - sizeof(*sg_table), req->cryptlen, 0); - } else { - dma_to_qm_sg_one(&fd_sgt[0], sg_dma_address(req->dst), - req->cryptlen, 0); - } + sizeof(*sg_table), req->cryptlen + ivsize, + 0); return edesc; } @@ -1359,7 +1375,6 @@ static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) struct skcipher_edesc *edesc; struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); - int ivsize = crypto_skcipher_ivsize(skcipher); int ret; if (unlikely(caam_congested)) @@ -1370,14 +1385,6 @@ static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) if (IS_ERR(edesc)) return PTR_ERR(edesc); - /* - * The crypto API expects us to set the IV (req->iv) to the last - * ciphertext block. - */ - if (!encrypt) - scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen - - ivsize, ivsize, 0); - ret = caam_qi_enqueue(ctx->qidev, &edesc->drv_req); if (!ret) { ret = -EINPROGRESS; @@ -2382,6 +2389,7 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, bool uses_dkp) { struct caam_drv_private *priv; + struct device *dev; /* * distribute tfms across job rings to ensure in-order @@ -2393,16 +2401,17 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, return PTR_ERR(ctx->jrdev); } - priv = dev_get_drvdata(ctx->jrdev->parent); + dev = ctx->jrdev->parent; + priv = dev_get_drvdata(dev); if (priv->era >= 6 && uses_dkp) ctx->dir = DMA_BIDIRECTIONAL; else ctx->dir = DMA_TO_DEVICE; - ctx->key_dma = dma_map_single(ctx->jrdev, ctx->key, sizeof(ctx->key), + ctx->key_dma = dma_map_single(dev, ctx->key, sizeof(ctx->key), ctx->dir); - if (dma_mapping_error(ctx->jrdev, ctx->key_dma)) { - dev_err(ctx->jrdev, "unable to map key\n"); + if (dma_mapping_error(dev, ctx->key_dma)) { + dev_err(dev, "unable to map key\n"); caam_jr_free(ctx->jrdev); return -ENOMEM; } @@ -2411,7 +2420,7 @@ static int caam_init_common(struct caam_ctx *ctx, struct caam_alg_entry *caam, ctx->cdata.algtype = OP_TYPE_CLASS1_ALG | caam->class1_alg_type; ctx->adata.algtype = OP_TYPE_CLASS2_ALG | caam->class2_alg_type; - ctx->qidev = priv->qidev; + ctx->qidev = dev; spin_lock_init(&ctx->lock); ctx->drv_ctx[ENCRYPT] = NULL; @@ -2445,7 +2454,8 @@ static void caam_exit_common(struct caam_ctx *ctx) caam_drv_ctx_rel(ctx->drv_ctx[ENCRYPT]); caam_drv_ctx_rel(ctx->drv_ctx[DECRYPT]); - dma_unmap_single(ctx->jrdev, ctx->key_dma, sizeof(ctx->key), ctx->dir); + dma_unmap_single(ctx->jrdev->parent, ctx->key_dma, sizeof(ctx->key), + ctx->dir); caam_jr_free(ctx->jrdev); } @@ -2460,7 +2470,7 @@ static void caam_aead_exit(struct crypto_aead *tfm) caam_exit_common(crypto_aead_ctx(tfm)); } -static void __exit caam_qi_algapi_exit(void) +void caam_qi_algapi_exit(void) { int i; @@ -2505,45 +2515,17 @@ static void caam_aead_alg_init(struct caam_aead_alg *t_alg) alg->exit = caam_aead_exit; } -static int __init caam_qi_algapi_init(void) +int caam_qi_algapi_init(struct device *ctrldev) { - struct device_node *dev_node; - struct platform_device *pdev; - struct device *ctrldev; - struct caam_drv_private *priv; + struct caam_drv_private *priv = dev_get_drvdata(ctrldev); int i = 0, err = 0; u32 aes_vid, aes_inst, des_inst, md_vid, md_inst; unsigned int md_limit = SHA512_DIGEST_SIZE; bool registered = false; - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); - if (!dev_node) - return -ENODEV; - } - - pdev = of_find_device_by_node(dev_node); - of_node_put(dev_node); - if (!pdev) - return -ENODEV; - - ctrldev = &pdev->dev; - priv = dev_get_drvdata(ctrldev); - - /* - * If priv is NULL, it's probably because the caam driver wasn't - * properly initialized (e.g. RNG4 init failed). Thus, bail out here. - */ - if (!priv || !priv->qi_present) { - err = -ENODEV; - goto out_put_dev; - } - if (caam_dpaa2) { dev_info(ctrldev, "caam/qi frontend driver not suitable for DPAA 2.x, aborting...\n"); - err = -ENODEV; - goto out_put_dev; + return -ENODEV; } /* @@ -2598,7 +2580,7 @@ static int __init caam_qi_algapi_init(void) err = crypto_register_skcipher(&t_alg->skcipher); if (err) { - dev_warn(priv->qidev, "%s alg registration failed\n", + dev_warn(ctrldev, "%s alg registration failed\n", t_alg->skcipher.base.cra_driver_name); continue; } @@ -2654,16 +2636,7 @@ static int __init caam_qi_algapi_init(void) } if (registered) - dev_info(priv->qidev, "algorithms registered in /proc/crypto\n"); + dev_info(ctrldev, "algorithms registered in /proc/crypto\n"); -out_put_dev: - put_device(ctrldev); return err; } - -module_init(caam_qi_algapi_init); -module_exit(caam_qi_algapi_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Support for crypto API using CAAM-QI backend"); -MODULE_AUTHOR("Freescale Semiconductor"); diff --git a/drivers/crypto/caam/caamalg_qi2.c b/drivers/crypto/caam/caamalg_qi2.c index 2b2980a8a9b9..06bf32c32cbd 100644 --- a/drivers/crypto/caam/caamalg_qi2.c +++ b/drivers/crypto/caam/caamalg_qi2.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause) /* * Copyright 2015-2016 Freescale Semiconductor Inc. - * Copyright 2017-2018 NXP + * Copyright 2017-2019 NXP */ #include "compat.h" @@ -140,7 +140,8 @@ static struct caam_request *to_caam_req(struct crypto_async_request *areq) static void caam_unmap(struct device *dev, struct scatterlist *src, struct scatterlist *dst, int src_nents, int dst_nents, dma_addr_t iv_dma, int ivsize, - dma_addr_t qm_sg_dma, int qm_sg_bytes) + enum dma_data_direction iv_dir, dma_addr_t qm_sg_dma, + int qm_sg_bytes) { if (dst != src) { if (src_nents) @@ -152,7 +153,7 @@ static void caam_unmap(struct device *dev, struct scatterlist *src, } if (iv_dma) - dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); + dma_unmap_single(dev, iv_dma, ivsize, iv_dir); if (qm_sg_bytes) dma_unmap_single(dev, qm_sg_dma, qm_sg_bytes, DMA_TO_DEVICE); @@ -371,6 +372,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; int src_nents, mapped_src_nents, dst_nents = 0, mapped_dst_nents = 0; + int src_len, dst_len = 0; struct aead_edesc *edesc; dma_addr_t qm_sg_dma, iv_dma = 0; int ivsize = 0; @@ -387,23 +389,21 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, } if (unlikely(req->dst != req->src)) { - src_nents = sg_nents_for_len(req->src, req->assoclen + - req->cryptlen); + src_len = req->assoclen + req->cryptlen; + dst_len = src_len + (encrypt ? authsize : (-authsize)); + + src_nents = sg_nents_for_len(req->src, src_len); if (unlikely(src_nents < 0)) { dev_err(dev, "Insufficient bytes (%d) in src S/G\n", - req->assoclen + req->cryptlen); + src_len); qi_cache_free(edesc); return ERR_PTR(src_nents); } - dst_nents = sg_nents_for_len(req->dst, req->assoclen + - req->cryptlen + - (encrypt ? authsize : - (-authsize))); + dst_nents = sg_nents_for_len(req->dst, dst_len); if (unlikely(dst_nents < 0)) { dev_err(dev, "Insufficient bytes (%d) in dst S/G\n", - req->assoclen + req->cryptlen + - (encrypt ? authsize : (-authsize))); + dst_len); qi_cache_free(edesc); return ERR_PTR(dst_nents); } @@ -434,13 +434,13 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, mapped_dst_nents = 0; } } else { - src_nents = sg_nents_for_len(req->src, req->assoclen + - req->cryptlen + - (encrypt ? authsize : 0)); + src_len = req->assoclen + req->cryptlen + + (encrypt ? authsize : 0); + + src_nents = sg_nents_for_len(req->src, src_len); if (unlikely(src_nents < 0)) { dev_err(dev, "Insufficient bytes (%d) in src S/G\n", - req->assoclen + req->cryptlen + - (encrypt ? authsize : 0)); + src_len); qi_cache_free(edesc); return ERR_PTR(src_nents); } @@ -460,9 +460,25 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, /* * Create S/G table: req->assoclen, [IV,] req->src [, req->dst]. * Input is not contiguous. + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. Logic: + * if (src != dst && output S/G) + * pad output S/G, if needed + * else if (src == dst && S/G) + * overlapping S/Gs; pad one of them + * else if (input S/G) ... + * pad input S/G, if needed */ - qm_sg_nents = 1 + !!ivsize + mapped_src_nents + - (mapped_dst_nents > 1 ? mapped_dst_nents : 0); + qm_sg_nents = 1 + !!ivsize + mapped_src_nents; + if (mapped_dst_nents > 1) + qm_sg_nents += pad_sg_nents(mapped_dst_nents); + else if ((req->src == req->dst) && (mapped_src_nents > 1)) + qm_sg_nents = max(pad_sg_nents(qm_sg_nents), + 1 + !!ivsize + + pad_sg_nents(mapped_src_nents)); + else + qm_sg_nents = pad_sg_nents(qm_sg_nents); + sg_table = &edesc->sgt[0]; qm_sg_bytes = qm_sg_nents * sizeof(*sg_table); if (unlikely(offsetof(struct aead_edesc, sgt) + qm_sg_bytes + ivsize > @@ -470,7 +486,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", qm_sg_nents, ivsize); caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -485,7 +501,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, if (dma_mapping_error(dev, iv_dma)) { dev_err(dev, "unable to map IV\n"); caam_unmap(dev, req->src, req->dst, src_nents, - dst_nents, 0, 0, 0, 0); + dst_nents, 0, 0, DMA_NONE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -509,7 +525,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, if (dma_mapping_error(dev, edesc->assoclen_dma)) { dev_err(dev, "unable to map assoclen\n"); caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, - iv_dma, ivsize, 0, 0); + iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -520,19 +536,18 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dma_to_qm_sg_one(sg_table + qm_sg_index, iv_dma, ivsize, 0); qm_sg_index++; } - sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + qm_sg_index, 0); + sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_index, 0); qm_sg_index += mapped_src_nents; if (mapped_dst_nents > 1) - sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + - qm_sg_index, 0); + sg_to_qm_sg_last(req->dst, dst_len, sg_table + qm_sg_index, 0); qm_sg_dma = dma_map_single(dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); if (dma_mapping_error(dev, qm_sg_dma)) { dev_err(dev, "unable to map S/G table\n"); dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, - iv_dma, ivsize, 0, 0); + iv_dma, ivsize, DMA_TO_DEVICE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -559,6 +574,14 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dpaa2_fl_set_addr(out_fle, qm_sg_dma + (1 + !!ivsize) * sizeof(*sg_table)); } + } else if (!mapped_dst_nents) { + /* + * crypto engine requires the output entry to be present when + * "frame list" FD is used. + * Since engine does not support FMT=2'b11 (unused entry type), + * leaving out_fle zeroized is the best option. + */ + goto skip_out_fle; } else if (mapped_dst_nents == 1) { dpaa2_fl_set_format(out_fle, dpaa2_fl_single); dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); @@ -570,6 +593,7 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, dpaa2_fl_set_len(out_fle, out_len); +skip_out_fle: return edesc; } @@ -1077,14 +1101,26 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) qm_sg_ents = 1 + mapped_src_nents; dst_sg_idx = qm_sg_ents; - qm_sg_ents += mapped_dst_nents > 1 ? mapped_dst_nents : 0; + /* + * Input, output HW S/G tables: [IV, src][dst, IV] + * IV entries point to the same buffer + * If src == dst, S/G entries are reused (S/G tables overlap) + * + * HW reads 4 S/G entries at a time; make sure the reads don't go beyond + * the end of the table by allocating more S/G entries. + */ + if (req->src != req->dst) + qm_sg_ents += pad_sg_nents(mapped_dst_nents + 1); + else + qm_sg_ents = 1 + pad_sg_nents(qm_sg_ents); + qm_sg_bytes = qm_sg_ents * sizeof(struct dpaa2_sg_entry); if (unlikely(offsetof(struct skcipher_edesc, sgt) + qm_sg_bytes + ivsize > CAAM_QI_MEMCACHE_SIZE)) { dev_err(dev, "No space for %d S/G entries and/or %dB IV\n", qm_sg_ents, ivsize); caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); return ERR_PTR(-ENOMEM); } @@ -1093,7 +1129,7 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) if (unlikely(!edesc)) { dev_err(dev, "could not allocate extended descriptor\n"); caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); return ERR_PTR(-ENOMEM); } @@ -1102,11 +1138,11 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) iv = (u8 *)(sg_table + qm_sg_ents); memcpy(iv, req->iv, ivsize); - iv_dma = dma_map_single(dev, iv, ivsize, DMA_TO_DEVICE); + iv_dma = dma_map_single(dev, iv, ivsize, DMA_BIDIRECTIONAL); if (dma_mapping_error(dev, iv_dma)) { dev_err(dev, "unable to map IV\n"); caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, 0, - 0, 0, 0); + 0, DMA_NONE, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1117,18 +1153,20 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) edesc->qm_sg_bytes = qm_sg_bytes; dma_to_qm_sg_one(sg_table, iv_dma, ivsize, 0); - sg_to_qm_sg_last(req->src, mapped_src_nents, sg_table + 1, 0); + sg_to_qm_sg(req->src, req->cryptlen, sg_table + 1, 0); - if (mapped_dst_nents > 1) - sg_to_qm_sg_last(req->dst, mapped_dst_nents, sg_table + - dst_sg_idx, 0); + if (req->src != req->dst) + sg_to_qm_sg(req->dst, req->cryptlen, sg_table + dst_sg_idx, 0); + + dma_to_qm_sg_one(sg_table + dst_sg_idx + mapped_dst_nents, iv_dma, + ivsize, 0); edesc->qm_sg_dma = dma_map_single(dev, sg_table, edesc->qm_sg_bytes, DMA_TO_DEVICE); if (dma_mapping_error(dev, edesc->qm_sg_dma)) { dev_err(dev, "unable to map S/G table\n"); caam_unmap(dev, req->src, req->dst, src_nents, dst_nents, - iv_dma, ivsize, 0, 0); + iv_dma, ivsize, DMA_BIDIRECTIONAL, 0, 0); qi_cache_free(edesc); return ERR_PTR(-ENOMEM); } @@ -1136,23 +1174,19 @@ static struct skcipher_edesc *skcipher_edesc_alloc(struct skcipher_request *req) memset(&req_ctx->fd_flt, 0, sizeof(req_ctx->fd_flt)); dpaa2_fl_set_final(in_fle, true); dpaa2_fl_set_len(in_fle, req->cryptlen + ivsize); - dpaa2_fl_set_len(out_fle, req->cryptlen); + dpaa2_fl_set_len(out_fle, req->cryptlen + ivsize); dpaa2_fl_set_format(in_fle, dpaa2_fl_sg); dpaa2_fl_set_addr(in_fle, edesc->qm_sg_dma); - if (req->src == req->dst) { - dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); + dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); + + if (req->src == req->dst) dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + sizeof(*sg_table)); - } else if (mapped_dst_nents > 1) { - dpaa2_fl_set_format(out_fle, dpaa2_fl_sg); + else dpaa2_fl_set_addr(out_fle, edesc->qm_sg_dma + dst_sg_idx * sizeof(*sg_table)); - } else { - dpaa2_fl_set_format(out_fle, dpaa2_fl_single); - dpaa2_fl_set_addr(out_fle, sg_dma_address(req->dst)); - } return edesc; } @@ -1164,7 +1198,8 @@ static void aead_unmap(struct device *dev, struct aead_edesc *edesc, int ivsize = crypto_aead_ivsize(aead); caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, - edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes); + edesc->iv_dma, ivsize, DMA_TO_DEVICE, edesc->qm_sg_dma, + edesc->qm_sg_bytes); dma_unmap_single(dev, edesc->assoclen_dma, 4, DMA_TO_DEVICE); } @@ -1175,7 +1210,8 @@ static void skcipher_unmap(struct device *dev, struct skcipher_edesc *edesc, int ivsize = crypto_skcipher_ivsize(skcipher); caam_unmap(dev, req->src, req->dst, edesc->src_nents, edesc->dst_nents, - edesc->iv_dma, ivsize, edesc->qm_sg_dma, edesc->qm_sg_bytes); + edesc->iv_dma, ivsize, DMA_BIDIRECTIONAL, edesc->qm_sg_dma, + edesc->qm_sg_bytes); } static void aead_encrypt_done(void *cbk_ctx, u32 status) @@ -1324,7 +1360,7 @@ static void skcipher_encrypt_done(void *cbk_ctx, u32 status) print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->iv, edesc->src_nents > 1 ? 100 : ivsize, 1); - caam_dump_sg(KERN_DEBUG, "dst @" __stringify(__LINE__)": ", + caam_dump_sg("dst @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->dst, edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); @@ -1332,10 +1368,10 @@ static void skcipher_encrypt_done(void *cbk_ctx, u32 status) /* * The crypto API expects us to set the IV (req->iv) to the last - * ciphertext block. This is used e.g. by the CTS mode. + * ciphertext block (CBC mode) or last counter (CTR mode). + * This is used e.g. by the CTS mode. */ - scatterwalk_map_and_copy(req->iv, req->dst, req->cryptlen - ivsize, - ivsize, 0); + memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, ivsize); qi_cache_free(edesc); skcipher_request_complete(req, ecode); @@ -1362,11 +1398,19 @@ static void skcipher_decrypt_done(void *cbk_ctx, u32 status) print_hex_dump_debug("dstiv @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->iv, edesc->src_nents > 1 ? 100 : ivsize, 1); - caam_dump_sg(KERN_DEBUG, "dst @" __stringify(__LINE__)": ", + caam_dump_sg("dst @" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, req->dst, edesc->dst_nents > 1 ? 100 : req->cryptlen, 1); skcipher_unmap(ctx->dev, edesc, req); + + /* + * The crypto API expects us to set the IV (req->iv) to the last + * ciphertext block (CBC mode) or last counter (CTR mode). + * This is used e.g. by the CTS mode. + */ + memcpy(req->iv, (u8 *)&edesc->sgt[0] + edesc->qm_sg_bytes, ivsize); + qi_cache_free(edesc); skcipher_request_complete(req, ecode); } @@ -1405,7 +1449,6 @@ static int skcipher_decrypt(struct skcipher_request *req) struct crypto_skcipher *skcipher = crypto_skcipher_reqtfm(req); struct caam_ctx *ctx = crypto_skcipher_ctx(skcipher); struct caam_request *caam_req = skcipher_request_ctx(req); - int ivsize = crypto_skcipher_ivsize(skcipher); int ret; /* allocate extended descriptor */ @@ -1413,13 +1456,6 @@ static int skcipher_decrypt(struct skcipher_request *req) if (IS_ERR(edesc)) return PTR_ERR(edesc); - /* - * The crypto API expects us to set the IV (req->iv) to the last - * ciphertext block. - */ - scatterwalk_map_and_copy(req->iv, req->src, req->cryptlen - ivsize, - ivsize, 0); - caam_req->flc = &ctx->flc[DECRYPT]; caam_req->flc_dma = ctx->flc_dma[DECRYPT]; caam_req->cbk = skcipher_decrypt_done; @@ -3380,9 +3416,9 @@ static int ahash_update_ctx(struct ahash_request *req) if (to_hash) { struct dpaa2_sg_entry *sg_table; + int src_len = req->nbytes - *next_buflen; - src_nents = sg_nents_for_len(req->src, - req->nbytes - (*next_buflen)); + src_nents = sg_nents_for_len(req->src, src_len); if (src_nents < 0) { dev_err(ctx->dev, "Invalid number of src SG.\n"); return src_nents; @@ -3409,7 +3445,7 @@ static int ahash_update_ctx(struct ahash_request *req) edesc->src_nents = src_nents; qm_sg_src_index = 1 + (*buflen ? 1 : 0); - qm_sg_bytes = (qm_sg_src_index + mapped_nents) * + qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * sizeof(*sg_table); sg_table = &edesc->sgt[0]; @@ -3423,7 +3459,7 @@ static int ahash_update_ctx(struct ahash_request *req) goto unmap_ctx; if (mapped_nents) { - sg_to_qm_sg_last(req->src, mapped_nents, + sg_to_qm_sg_last(req->src, src_len, sg_table + qm_sg_src_index, 0); if (*next_buflen) scatterwalk_map_and_copy(next_buf, req->src, @@ -3494,7 +3530,7 @@ static int ahash_final_ctx(struct ahash_request *req) gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; int buflen = *current_buflen(state); - int qm_sg_bytes, qm_sg_src_index; + int qm_sg_bytes; int digestsize = crypto_ahash_digestsize(ahash); struct ahash_edesc *edesc; struct dpaa2_sg_entry *sg_table; @@ -3505,8 +3541,7 @@ static int ahash_final_ctx(struct ahash_request *req) if (!edesc) return -ENOMEM; - qm_sg_src_index = 1 + (buflen ? 1 : 0); - qm_sg_bytes = qm_sg_src_index * sizeof(*sg_table); + qm_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * sizeof(*sg_table); sg_table = &edesc->sgt[0]; ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, @@ -3518,7 +3553,7 @@ static int ahash_final_ctx(struct ahash_request *req) if (ret) goto unmap_ctx; - dpaa2_sg_set_final(sg_table + qm_sg_src_index - 1, true); + dpaa2_sg_set_final(sg_table + (buflen ? 1 : 0), true); edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); @@ -3599,7 +3634,8 @@ static int ahash_finup_ctx(struct ahash_request *req) edesc->src_nents = src_nents; qm_sg_src_index = 1 + (buflen ? 1 : 0); - qm_sg_bytes = (qm_sg_src_index + mapped_nents) * sizeof(*sg_table); + qm_sg_bytes = pad_sg_nents(qm_sg_src_index + mapped_nents) * + sizeof(*sg_table); sg_table = &edesc->sgt[0]; ret = ctx_map_to_qm_sg(ctx->dev, state, ctx->ctx_len, sg_table, @@ -3611,7 +3647,7 @@ static int ahash_finup_ctx(struct ahash_request *req) if (ret) goto unmap_ctx; - sg_to_qm_sg_last(req->src, mapped_nents, sg_table + qm_sg_src_index, 0); + sg_to_qm_sg_last(req->src, req->nbytes, sg_table + qm_sg_src_index, 0); edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); @@ -3696,8 +3732,8 @@ static int ahash_digest(struct ahash_request *req) int qm_sg_bytes; struct dpaa2_sg_entry *sg_table = &edesc->sgt[0]; - qm_sg_bytes = mapped_nents * sizeof(*sg_table); - sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0); + qm_sg_bytes = pad_sg_nents(mapped_nents) * sizeof(*sg_table); + sg_to_qm_sg_last(req->src, req->nbytes, sg_table, 0); edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); if (dma_mapping_error(ctx->dev, edesc->qm_sg_dma)) { @@ -3840,9 +3876,9 @@ static int ahash_update_no_ctx(struct ahash_request *req) if (to_hash) { struct dpaa2_sg_entry *sg_table; + int src_len = req->nbytes - *next_buflen; - src_nents = sg_nents_for_len(req->src, - req->nbytes - *next_buflen); + src_nents = sg_nents_for_len(req->src, src_len); if (src_nents < 0) { dev_err(ctx->dev, "Invalid number of src SG.\n"); return src_nents; @@ -3868,14 +3904,15 @@ static int ahash_update_no_ctx(struct ahash_request *req) } edesc->src_nents = src_nents; - qm_sg_bytes = (1 + mapped_nents) * sizeof(*sg_table); + qm_sg_bytes = pad_sg_nents(1 + mapped_nents) * + sizeof(*sg_table); sg_table = &edesc->sgt[0]; ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); if (ret) goto unmap_ctx; - sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0); + sg_to_qm_sg_last(req->src, src_len, sg_table + 1, 0); if (*next_buflen) scatterwalk_map_and_copy(next_buf, req->src, @@ -3987,14 +4024,14 @@ static int ahash_finup_no_ctx(struct ahash_request *req) } edesc->src_nents = src_nents; - qm_sg_bytes = (2 + mapped_nents) * sizeof(*sg_table); + qm_sg_bytes = pad_sg_nents(2 + mapped_nents) * sizeof(*sg_table); sg_table = &edesc->sgt[0]; ret = buf_map_to_qm_sg(ctx->dev, sg_table, state); if (ret) goto unmap; - sg_to_qm_sg_last(req->src, mapped_nents, sg_table + 1, 0); + sg_to_qm_sg_last(req->src, req->nbytes, sg_table + 1, 0); edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); @@ -4064,9 +4101,9 @@ static int ahash_update_first(struct ahash_request *req) if (to_hash) { struct dpaa2_sg_entry *sg_table; + int src_len = req->nbytes - *next_buflen; - src_nents = sg_nents_for_len(req->src, - req->nbytes - (*next_buflen)); + src_nents = sg_nents_for_len(req->src, src_len); if (src_nents < 0) { dev_err(ctx->dev, "Invalid number of src SG.\n"); return src_nents; @@ -4101,8 +4138,9 @@ static int ahash_update_first(struct ahash_request *req) if (mapped_nents > 1) { int qm_sg_bytes; - sg_to_qm_sg_last(req->src, mapped_nents, sg_table, 0); - qm_sg_bytes = mapped_nents * sizeof(*sg_table); + sg_to_qm_sg_last(req->src, src_len, sg_table, 0); + qm_sg_bytes = pad_sg_nents(mapped_nents) * + sizeof(*sg_table); edesc->qm_sg_dma = dma_map_single(ctx->dev, sg_table, qm_sg_bytes, DMA_TO_DEVICE); diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index 7205d9f4029e..e4ac5d591ad6 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -82,14 +82,6 @@ #define HASH_MSG_LEN 8 #define MAX_CTX_LEN (HASH_MSG_LEN + SHA512_DIGEST_SIZE) -#ifdef DEBUG -/* for print_hex_dumps with line references */ -#define debug(format, arg...) printk(format, arg) -#else -#define debug(format, arg...) -#endif - - static struct list_head hash_list; /* ahash per-session context */ @@ -243,11 +235,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash) ctx->ctx_len, true, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_update_dma, desc_bytes(desc), ctx->dir); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "ahash update shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + + print_hex_dump_debug("ahash update shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); /* ahash_update_first shared descriptor */ desc = ctx->sh_desc_update_first; @@ -255,11 +246,9 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash) ctx->ctx_len, false, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma, desc_bytes(desc), ctx->dir); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "ahash update first shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("ahash update first shdesc@"__stringify(__LINE__) + ": ", DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); /* ahash_final shared descriptor */ desc = ctx->sh_desc_fin; @@ -267,11 +256,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash) ctx->ctx_len, true, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_fin_dma, desc_bytes(desc), ctx->dir); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ahash final shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + + print_hex_dump_debug("ahash final shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); /* ahash_digest shared descriptor */ desc = ctx->sh_desc_digest; @@ -279,12 +267,10 @@ static int ahash_set_sh_desc(struct crypto_ahash *ahash) ctx->ctx_len, false, ctrlpriv->era); dma_sync_single_for_device(jrdev, ctx->sh_desc_digest_dma, desc_bytes(desc), ctx->dir); -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "ahash digest shdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + + print_hex_dump_debug("ahash digest shdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); return 0; } @@ -328,9 +314,9 @@ static int axcbc_set_sh_desc(struct crypto_ahash *ahash) ctx->ctx_len, ctx->key_dma); dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma, desc_bytes(desc), ctx->dir); - print_hex_dump_debug("axcbc update first shdesc@" __stringify(__LINE__)" : ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), - 1); + print_hex_dump_debug("axcbc update first shdesc@" __stringify(__LINE__) + " : ", DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); /* shared descriptor for ahash_digest */ desc = ctx->sh_desc_digest; @@ -377,8 +363,8 @@ static int acmac_set_sh_desc(struct crypto_ahash *ahash) ctx->ctx_len, 0); dma_sync_single_for_device(jrdev, ctx->sh_desc_update_first_dma, desc_bytes(desc), ctx->dir); - print_hex_dump_debug("acmac update first shdesc@" __stringify(__LINE__)" : ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, + print_hex_dump_debug("acmac update first shdesc@" __stringify(__LINE__) + " : ", DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); /* shared descriptor for ahash_digest */ @@ -429,12 +415,11 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, append_seq_store(desc, digestsize, LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_CONTEXT); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "key_in@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1); - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("key_in@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, *keylen, 1); + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); result.err = 0; init_completion(&result.completion); @@ -444,11 +429,10 @@ static int hash_digest_key(struct caam_hash_ctx *ctx, u32 *keylen, u8 *key, /* in progress */ wait_for_completion(&result.completion); ret = result.err; -#ifdef DEBUG - print_hex_dump(KERN_ERR, - "digested key@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key, digestsize, 1); -#endif + + print_hex_dump_debug("digested key@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key, + digestsize, 1); } dma_unmap_single(jrdev, key_dma, *keylen, DMA_BIDIRECTIONAL); @@ -463,15 +447,14 @@ static int ahash_setkey(struct crypto_ahash *ahash, const u8 *key, unsigned int keylen) { struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); + struct device *jrdev = ctx->jrdev; int blocksize = crypto_tfm_alg_blocksize(&ahash->base); int digestsize = crypto_ahash_digestsize(ahash); struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctx->jrdev->parent); int ret; u8 *hashed_key = NULL; -#ifdef DEBUG - printk(KERN_ERR "keylen %d\n", keylen); -#endif + dev_dbg(jrdev, "keylen %d\n", keylen); if (keylen > blocksize) { hashed_key = kmemdup(key, keylen, GFP_KERNEL | GFP_DMA); @@ -600,11 +583,9 @@ static void ahash_done(struct device *jrdev, u32 *desc, u32 err, struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); int digestsize = crypto_ahash_digestsize(ahash); struct caam_hash_state *state = ahash_request_ctx(req); -#ifdef DEBUG struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); if (err) @@ -614,11 +595,9 @@ static void ahash_done(struct device *jrdev, u32 *desc, u32 err, memcpy(req->result, state->caam_ctx, digestsize); kfree(edesc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, - ctx->ctx_len, 1); -#endif + print_hex_dump_debug("ctx@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, + ctx->ctx_len, 1); req->base.complete(&req->base, err); } @@ -631,11 +610,9 @@ static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err, struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); -#ifdef DEBUG int digestsize = crypto_ahash_digestsize(ahash); - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); if (err) @@ -645,15 +622,13 @@ static void ahash_done_bi(struct device *jrdev, u32 *desc, u32 err, switch_buf(state); kfree(edesc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, - ctx->ctx_len, 1); + print_hex_dump_debug("ctx@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, + ctx->ctx_len, 1); if (req->result) - print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->result, - digestsize, 1); -#endif + print_hex_dump_debug("result@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->result, + digestsize, 1); req->base.complete(&req->base, err); } @@ -666,11 +641,9 @@ static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err, struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); int digestsize = crypto_ahash_digestsize(ahash); struct caam_hash_state *state = ahash_request_ctx(req); -#ifdef DEBUG struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); if (err) @@ -680,11 +653,9 @@ static void ahash_done_ctx_src(struct device *jrdev, u32 *desc, u32 err, memcpy(req->result, state->caam_ctx, digestsize); kfree(edesc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, - ctx->ctx_len, 1); -#endif + print_hex_dump_debug("ctx@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, + ctx->ctx_len, 1); req->base.complete(&req->base, err); } @@ -697,11 +668,9 @@ static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err, struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); struct caam_hash_ctx *ctx = crypto_ahash_ctx(ahash); struct caam_hash_state *state = ahash_request_ctx(req); -#ifdef DEBUG int digestsize = crypto_ahash_digestsize(ahash); - dev_err(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(jrdev, "%s %d: err 0x%x\n", __func__, __LINE__, err); edesc = container_of(desc, struct ahash_edesc, hw_desc[0]); if (err) @@ -711,15 +680,13 @@ static void ahash_done_ctx_dst(struct device *jrdev, u32 *desc, u32 err, switch_buf(state); kfree(edesc); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ctx@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, - ctx->ctx_len, 1); + print_hex_dump_debug("ctx@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, state->caam_ctx, + ctx->ctx_len, 1); if (req->result) - print_hex_dump(KERN_ERR, "result@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, req->result, - digestsize, 1); -#endif + print_hex_dump_debug("result@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, req->result, + digestsize, 1); req->base.complete(&req->base, err); } @@ -759,9 +726,10 @@ static int ahash_edesc_add_src(struct caam_hash_ctx *ctx, if (nents > 1 || first_sg) { struct sec4_sg_entry *sg = edesc->sec4_sg; - unsigned int sgsize = sizeof(*sg) * (first_sg + nents); + unsigned int sgsize = sizeof(*sg) * + pad_sg_nents(first_sg + nents); - sg_to_sec4_sg_last(req->src, nents, sg + first_sg, 0); + sg_to_sec4_sg_last(req->src, to_hash, sg + first_sg, 0); src_dma = dma_map_single(ctx->jrdev, sg, sgsize, DMA_TO_DEVICE); if (dma_mapping_error(ctx->jrdev, src_dma)) { @@ -819,8 +787,10 @@ static int ahash_update_ctx(struct ahash_request *req) } if (to_hash) { - src_nents = sg_nents_for_len(req->src, - req->nbytes - (*next_buflen)); + int pad_nents; + int src_len = req->nbytes - *next_buflen; + + src_nents = sg_nents_for_len(req->src, src_len); if (src_nents < 0) { dev_err(jrdev, "Invalid number of src SG.\n"); return src_nents; @@ -838,15 +808,14 @@ static int ahash_update_ctx(struct ahash_request *req) } sec4_sg_src_index = 1 + (*buflen ? 1 : 0); - sec4_sg_bytes = (sec4_sg_src_index + mapped_nents) * - sizeof(struct sec4_sg_entry); + pad_nents = pad_sg_nents(sec4_sg_src_index + mapped_nents); + sec4_sg_bytes = pad_nents * sizeof(struct sec4_sg_entry); /* * allocate space for base edesc and hw desc commands, * link tables */ - edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index + mapped_nents, - ctx->sh_desc_update, + edesc = ahash_edesc_alloc(ctx, pad_nents, ctx->sh_desc_update, ctx->sh_desc_update_dma, flags); if (!edesc) { dma_unmap_sg(jrdev, req->src, src_nents, DMA_TO_DEVICE); @@ -866,7 +835,7 @@ static int ahash_update_ctx(struct ahash_request *req) goto unmap_ctx; if (mapped_nents) - sg_to_sec4_sg_last(req->src, mapped_nents, + sg_to_sec4_sg_last(req->src, src_len, edesc->sec4_sg + sec4_sg_src_index, 0); else @@ -893,11 +862,9 @@ static int ahash_update_ctx(struct ahash_request *req) append_seq_out_ptr(desc, state->ctx_dma, ctx->ctx_len, 0); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done_bi, req); if (ret) @@ -910,13 +877,12 @@ static int ahash_update_ctx(struct ahash_request *req) *buflen = *next_buflen; *next_buflen = last_buflen; } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); - print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, next_buf, - *next_buflen, 1); -#endif + + print_hex_dump_debug("buf@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); + print_hex_dump_debug("next buf@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, next_buf, + *next_buflen, 1); return ret; unmap_ctx: @@ -935,18 +901,17 @@ static int ahash_final_ctx(struct ahash_request *req) GFP_KERNEL : GFP_ATOMIC; int buflen = *current_buflen(state); u32 *desc; - int sec4_sg_bytes, sec4_sg_src_index; + int sec4_sg_bytes; int digestsize = crypto_ahash_digestsize(ahash); struct ahash_edesc *edesc; int ret; - sec4_sg_src_index = 1 + (buflen ? 1 : 0); - sec4_sg_bytes = sec4_sg_src_index * sizeof(struct sec4_sg_entry); + sec4_sg_bytes = pad_sg_nents(1 + (buflen ? 1 : 0)) * + sizeof(struct sec4_sg_entry); /* allocate space for base edesc and hw desc commands, link tables */ - edesc = ahash_edesc_alloc(ctx, sec4_sg_src_index, - ctx->sh_desc_fin, ctx->sh_desc_fin_dma, - flags); + edesc = ahash_edesc_alloc(ctx, 4, ctx->sh_desc_fin, + ctx->sh_desc_fin_dma, flags); if (!edesc) return -ENOMEM; @@ -963,7 +928,7 @@ static int ahash_final_ctx(struct ahash_request *req) if (ret) goto unmap_ctx; - sg_to_sec4_set_last(edesc->sec4_sg + sec4_sg_src_index - 1); + sg_to_sec4_set_last(edesc->sec4_sg + (buflen ? 1 : 0)); edesc->sec4_sg_dma = dma_map_single(jrdev, edesc->sec4_sg, sec4_sg_bytes, DMA_TO_DEVICE); @@ -977,10 +942,9 @@ static int ahash_final_ctx(struct ahash_request *req) LDST_SGF); append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req); if (ret) @@ -1058,10 +1022,9 @@ static int ahash_finup_ctx(struct ahash_request *req) append_seq_out_ptr(desc, state->ctx_dma, digestsize, 0); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_src, req); if (ret) @@ -1135,10 +1098,9 @@ static int ahash_digest(struct ahash_request *req) return -ENOMEM; } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); if (!ret) { @@ -1190,10 +1152,9 @@ static int ahash_final_no_ctx(struct ahash_request *req) if (ret) goto unmap; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); if (!ret) { @@ -1246,8 +1207,10 @@ static int ahash_update_no_ctx(struct ahash_request *req) } if (to_hash) { - src_nents = sg_nents_for_len(req->src, - req->nbytes - *next_buflen); + int pad_nents; + int src_len = req->nbytes - *next_buflen; + + src_nents = sg_nents_for_len(req->src, src_len); if (src_nents < 0) { dev_err(jrdev, "Invalid number of src SG.\n"); return src_nents; @@ -1264,14 +1227,14 @@ static int ahash_update_no_ctx(struct ahash_request *req) mapped_nents = 0; } - sec4_sg_bytes = (1 + mapped_nents) * - sizeof(struct sec4_sg_entry); + pad_nents = pad_sg_nents(1 + mapped_nents); + sec4_sg_bytes = pad_nents * sizeof(struct sec4_sg_entry); /* * allocate space for base edesc and hw desc commands, * link tables */ - edesc = ahash_edesc_alloc(ctx, 1 + mapped_nents, + edesc = ahash_edesc_alloc(ctx, pad_nents, ctx->sh_desc_update_first, ctx->sh_desc_update_first_dma, flags); @@ -1287,8 +1250,7 @@ static int ahash_update_no_ctx(struct ahash_request *req) if (ret) goto unmap_ctx; - sg_to_sec4_sg_last(req->src, mapped_nents, - edesc->sec4_sg + 1, 0); + sg_to_sec4_sg_last(req->src, src_len, edesc->sec4_sg + 1, 0); if (*next_buflen) { scatterwalk_map_and_copy(next_buf, req->src, @@ -1313,11 +1275,9 @@ static int ahash_update_no_ctx(struct ahash_request *req) if (ret) goto unmap_ctx; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req); if (ret) @@ -1333,13 +1293,12 @@ static int ahash_update_no_ctx(struct ahash_request *req) *buflen = *next_buflen; *next_buflen = 0; } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "buf@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); - print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, next_buf, - *next_buflen, 1); -#endif + + print_hex_dump_debug("buf@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, buf, *buflen, 1); + print_hex_dump_debug("next buf@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen, + 1); return ret; unmap_ctx: @@ -1414,10 +1373,9 @@ static int ahash_finup_no_ctx(struct ahash_request *req) if (ret) goto unmap; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done, req); if (!ret) { @@ -1517,11 +1475,9 @@ static int ahash_update_first(struct ahash_request *req) if (ret) goto unmap_ctx; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, - desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); ret = caam_jr_enqueue(jrdev, desc, ahash_done_ctx_dst, req); if (ret) @@ -1539,11 +1495,10 @@ static int ahash_update_first(struct ahash_request *req) req->nbytes, 0); switch_buf(state); } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "next buf@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, next_buf, - *next_buflen, 1); -#endif + + print_hex_dump_debug("next buf@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, next_buf, *next_buflen, + 1); return ret; unmap_ctx: @@ -1930,7 +1885,7 @@ static void caam_hash_cra_exit(struct crypto_tfm *tfm) caam_jr_free(ctx->jrdev); } -static void __exit caam_algapi_hash_exit(void) +void caam_algapi_hash_exit(void) { struct caam_hash_alg *t_alg, *n; @@ -1988,40 +1943,13 @@ caam_hash_alloc(struct caam_hash_template *template, return t_alg; } -static int __init caam_algapi_hash_init(void) +int caam_algapi_hash_init(struct device *ctrldev) { - struct device_node *dev_node; - struct platform_device *pdev; int i = 0, err = 0; - struct caam_drv_private *priv; + struct caam_drv_private *priv = dev_get_drvdata(ctrldev); unsigned int md_limit = SHA512_DIGEST_SIZE; u32 md_inst, md_vid; - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); - if (!dev_node) - return -ENODEV; - } - - pdev = of_find_device_by_node(dev_node); - if (!pdev) { - of_node_put(dev_node); - return -ENODEV; - } - - priv = dev_get_drvdata(&pdev->dev); - of_node_put(dev_node); - - /* - * If priv is NULL, it's probably because the caam driver wasn't - * properly initialized (e.g. RNG4 init failed). Thus, bail out here. - */ - if (!priv) { - err = -ENODEV; - goto out_put_dev; - } - /* * Register crypto algorithms the device supports. First, identify * presence and attributes of MD block. @@ -2042,10 +1970,8 @@ static int __init caam_algapi_hash_init(void) * Skip registration of any hashing algorithms if MD block * is not present. */ - if (!md_inst) { - err = -ENODEV; - goto out_put_dev; - } + if (!md_inst) + return -ENODEV; /* Limit digest size based on LP256 */ if (md_vid == CHA_VER_VID_MD_LP256) @@ -2102,14 +2028,5 @@ static int __init caam_algapi_hash_init(void) list_add_tail(&t_alg->entry, &hash_list); } -out_put_dev: - put_device(&pdev->dev); return err; } - -module_init(caam_algapi_hash_init); -module_exit(caam_algapi_hash_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("FSL CAAM support for ahash functions of crypto API"); -MODULE_AUTHOR("Freescale Semiconductor - NMG"); diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c index fe24485274e1..80574106af29 100644 --- a/drivers/crypto/caam/caampkc.c +++ b/drivers/crypto/caam/caampkc.c @@ -3,7 +3,7 @@ * caam - Freescale FSL CAAM support for Public Key Cryptography * * Copyright 2016 Freescale Semiconductor, Inc. - * Copyright 2018 NXP + * Copyright 2018-2019 NXP * * There is no Shared Descriptor for PKC so that the Job Descriptor must carry * all the desired key parameters, input and output pointers. @@ -24,12 +24,18 @@ sizeof(struct rsa_priv_f2_pdb)) #define DESC_RSA_PRIV_F3_LEN (2 * CAAM_CMD_SZ + \ sizeof(struct rsa_priv_f3_pdb)) +#define CAAM_RSA_MAX_INPUT_SIZE 512 /* for a 4096-bit modulus */ + +/* buffer filled with zeros, used for padding */ +static u8 *zero_buffer; static void rsa_io_unmap(struct device *dev, struct rsa_edesc *edesc, struct akcipher_request *req) { + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + dma_unmap_sg(dev, req->dst, edesc->dst_nents, DMA_FROM_DEVICE); - dma_unmap_sg(dev, req->src, edesc->src_nents, DMA_TO_DEVICE); + dma_unmap_sg(dev, req_ctx->fixup_src, edesc->src_nents, DMA_TO_DEVICE); if (edesc->sec4_sg_bytes) dma_unmap_single(dev, edesc->sec4_sg_dma, edesc->sec4_sg_bytes, @@ -168,6 +174,13 @@ static void rsa_priv_f3_done(struct device *dev, u32 *desc, u32 err, akcipher_request_complete(req, err); } +/** + * Count leading zeros, need it to strip, from a given scatterlist + * + * @sgl : scatterlist to count zeros from + * @nbytes: number of zeros, in bytes, to strip + * @flags : operation flags + */ static int caam_rsa_count_leading_zeros(struct scatterlist *sgl, unsigned int nbytes, unsigned int flags) @@ -187,7 +200,8 @@ static int caam_rsa_count_leading_zeros(struct scatterlist *sgl, lzeros = 0; len = 0; while (nbytes > 0) { - while (len && !*buff) { + /* do not strip more than given bytes */ + while (len && !*buff && lzeros < nbytes) { lzeros++; len--; buff++; @@ -218,6 +232,7 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); struct device *dev = ctx->dev; struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + struct caam_rsa_key *key = &ctx->key; struct rsa_edesc *edesc; gfp_t flags = (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL : GFP_ATOMIC; @@ -225,22 +240,45 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, int sgc; int sec4_sg_index, sec4_sg_len = 0, sec4_sg_bytes; int src_nents, dst_nents; + unsigned int diff_size = 0; int lzeros; - lzeros = caam_rsa_count_leading_zeros(req->src, req->src_len, sg_flags); - if (lzeros < 0) - return ERR_PTR(lzeros); - - req->src_len -= lzeros; - req->src = scatterwalk_ffwd(req_ctx->src, req->src, lzeros); + if (req->src_len > key->n_sz) { + /* + * strip leading zeros and + * return the number of zeros to skip + */ + lzeros = caam_rsa_count_leading_zeros(req->src, req->src_len - + key->n_sz, sg_flags); + if (lzeros < 0) + return ERR_PTR(lzeros); + + req_ctx->fixup_src = scatterwalk_ffwd(req_ctx->src, req->src, + lzeros); + req_ctx->fixup_src_len = req->src_len - lzeros; + } else { + /* + * input src is less then n key modulus, + * so there will be zero padding + */ + diff_size = key->n_sz - req->src_len; + req_ctx->fixup_src = req->src; + req_ctx->fixup_src_len = req->src_len; + } - src_nents = sg_nents_for_len(req->src, req->src_len); + src_nents = sg_nents_for_len(req_ctx->fixup_src, + req_ctx->fixup_src_len); dst_nents = sg_nents_for_len(req->dst, req->dst_len); - if (src_nents > 1) - sec4_sg_len = src_nents; + if (!diff_size && src_nents == 1) + sec4_sg_len = 0; /* no need for an input hw s/g table */ + else + sec4_sg_len = src_nents + !!diff_size; + sec4_sg_index = sec4_sg_len; if (dst_nents > 1) - sec4_sg_len += dst_nents; + sec4_sg_len += pad_sg_nents(dst_nents); + else + sec4_sg_len = pad_sg_nents(sec4_sg_len); sec4_sg_bytes = sec4_sg_len * sizeof(struct sec4_sg_entry); @@ -250,7 +288,7 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, if (!edesc) return ERR_PTR(-ENOMEM); - sgc = dma_map_sg(dev, req->src, src_nents, DMA_TO_DEVICE); + sgc = dma_map_sg(dev, req_ctx->fixup_src, src_nents, DMA_TO_DEVICE); if (unlikely(!sgc)) { dev_err(dev, "unable to map source\n"); goto src_fail; @@ -263,14 +301,16 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, } edesc->sec4_sg = (void *)edesc + sizeof(*edesc) + desclen; + if (diff_size) + dma_to_sec4_sg_one(edesc->sec4_sg, ctx->padding_dma, diff_size, + 0); + + if (sec4_sg_index) + sg_to_sec4_sg_last(req_ctx->fixup_src, req_ctx->fixup_src_len, + edesc->sec4_sg + !!diff_size, 0); - sec4_sg_index = 0; - if (src_nents > 1) { - sg_to_sec4_sg_last(req->src, src_nents, edesc->sec4_sg, 0); - sec4_sg_index += src_nents; - } if (dst_nents > 1) - sg_to_sec4_sg_last(req->dst, dst_nents, + sg_to_sec4_sg_last(req->dst, req->dst_len, edesc->sec4_sg + sec4_sg_index, 0); /* Save nents for later use in Job Descriptor */ @@ -289,12 +329,16 @@ static struct rsa_edesc *rsa_edesc_alloc(struct akcipher_request *req, edesc->sec4_sg_bytes = sec4_sg_bytes; + print_hex_dump_debug("caampkc sec4_sg@" __stringify(__LINE__) ": ", + DUMP_PREFIX_ADDRESS, 16, 4, edesc->sec4_sg, + edesc->sec4_sg_bytes, 1); + return edesc; sec4_sg_fail: dma_unmap_sg(dev, req->dst, dst_nents, DMA_FROM_DEVICE); dst_fail: - dma_unmap_sg(dev, req->src, src_nents, DMA_TO_DEVICE); + dma_unmap_sg(dev, req_ctx->fixup_src, src_nents, DMA_TO_DEVICE); src_fail: kfree(edesc); return ERR_PTR(-ENOMEM); @@ -304,6 +348,7 @@ static int set_rsa_pub_pdb(struct akcipher_request *req, struct rsa_edesc *edesc) { struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); struct caam_rsa_key *key = &ctx->key; struct device *dev = ctx->dev; @@ -328,7 +373,7 @@ static int set_rsa_pub_pdb(struct akcipher_request *req, pdb->f_dma = edesc->sec4_sg_dma; sec4_sg_index += edesc->src_nents; } else { - pdb->f_dma = sg_dma_address(req->src); + pdb->f_dma = sg_dma_address(req_ctx->fixup_src); } if (edesc->dst_nents > 1) { @@ -340,7 +385,7 @@ static int set_rsa_pub_pdb(struct akcipher_request *req, } pdb->sgf |= (key->e_sz << RSA_PDB_E_SHIFT) | key->n_sz; - pdb->f_len = req->src_len; + pdb->f_len = req_ctx->fixup_src_len; return 0; } @@ -373,7 +418,9 @@ static int set_rsa_priv_f1_pdb(struct akcipher_request *req, pdb->g_dma = edesc->sec4_sg_dma; sec4_sg_index += edesc->src_nents; } else { - pdb->g_dma = sg_dma_address(req->src); + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + + pdb->g_dma = sg_dma_address(req_ctx->fixup_src); } if (edesc->dst_nents > 1) { @@ -436,7 +483,9 @@ static int set_rsa_priv_f2_pdb(struct akcipher_request *req, pdb->g_dma = edesc->sec4_sg_dma; sec4_sg_index += edesc->src_nents; } else { - pdb->g_dma = sg_dma_address(req->src); + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + + pdb->g_dma = sg_dma_address(req_ctx->fixup_src); } if (edesc->dst_nents > 1) { @@ -523,7 +572,9 @@ static int set_rsa_priv_f3_pdb(struct akcipher_request *req, pdb->g_dma = edesc->sec4_sg_dma; sec4_sg_index += edesc->src_nents; } else { - pdb->g_dma = sg_dma_address(req->src); + struct caam_rsa_req_ctx *req_ctx = akcipher_request_ctx(req); + + pdb->g_dma = sg_dma_address(req_ctx->fixup_src); } if (edesc->dst_nents > 1) { @@ -978,6 +1029,15 @@ static int caam_rsa_init_tfm(struct crypto_akcipher *tfm) return PTR_ERR(ctx->dev); } + ctx->padding_dma = dma_map_single(ctx->dev, zero_buffer, + CAAM_RSA_MAX_INPUT_SIZE - 1, + DMA_TO_DEVICE); + if (dma_mapping_error(ctx->dev, ctx->padding_dma)) { + dev_err(ctx->dev, "unable to map padding\n"); + caam_jr_free(ctx->dev); + return -ENOMEM; + } + return 0; } @@ -987,6 +1047,8 @@ static void caam_rsa_exit_tfm(struct crypto_akcipher *tfm) struct caam_rsa_ctx *ctx = akcipher_tfm_ctx(tfm); struct caam_rsa_key *key = &ctx->key; + dma_unmap_single(ctx->dev, ctx->padding_dma, CAAM_RSA_MAX_INPUT_SIZE - + 1, DMA_TO_DEVICE); caam_rsa_free_key(key); caam_jr_free(ctx->dev); } @@ -1010,41 +1072,12 @@ static struct akcipher_alg caam_rsa = { }; /* Public Key Cryptography module initialization handler */ -static int __init caam_pkc_init(void) +int caam_pkc_init(struct device *ctrldev) { - struct device_node *dev_node; - struct platform_device *pdev; - struct device *ctrldev; - struct caam_drv_private *priv; + struct caam_drv_private *priv = dev_get_drvdata(ctrldev); u32 pk_inst; int err; - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); - if (!dev_node) - return -ENODEV; - } - - pdev = of_find_device_by_node(dev_node); - if (!pdev) { - of_node_put(dev_node); - return -ENODEV; - } - - ctrldev = &pdev->dev; - priv = dev_get_drvdata(ctrldev); - of_node_put(dev_node); - - /* - * If priv is NULL, it's probably because the caam driver wasn't - * properly initialized (e.g. RNG4 init failed). Thus, bail out here. - */ - if (!priv) { - err = -ENODEV; - goto out_put_dev; - } - /* Determine public key hardware accelerator presence. */ if (priv->era < 10) pk_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) & @@ -1053,31 +1086,29 @@ static int __init caam_pkc_init(void) pk_inst = rd_reg32(&priv->ctrl->vreg.pkha) & CHA_VER_NUM_MASK; /* Do not register algorithms if PKHA is not present. */ - if (!pk_inst) { - err = -ENODEV; - goto out_put_dev; - } + if (!pk_inst) + return 0; + + /* allocate zero buffer, used for padding input */ + zero_buffer = kzalloc(CAAM_RSA_MAX_INPUT_SIZE - 1, GFP_DMA | + GFP_KERNEL); + if (!zero_buffer) + return -ENOMEM; err = crypto_register_akcipher(&caam_rsa); - if (err) + if (err) { + kfree(zero_buffer); dev_warn(ctrldev, "%s alg registration failed\n", caam_rsa.base.cra_driver_name); - else + } else { dev_info(ctrldev, "caam pkc algorithms registered in /proc/crypto\n"); + } -out_put_dev: - put_device(ctrldev); return err; } -static void __exit caam_pkc_exit(void) +void caam_pkc_exit(void) { + kfree(zero_buffer); crypto_unregister_akcipher(&caam_rsa); } - -module_init(caam_pkc_init); -module_exit(caam_pkc_exit); - -MODULE_LICENSE("Dual BSD/GPL"); -MODULE_DESCRIPTION("FSL CAAM support for PKC functions of crypto API"); -MODULE_AUTHOR("Freescale Semiconductor"); diff --git a/drivers/crypto/caam/caampkc.h b/drivers/crypto/caam/caampkc.h index 82645bcf8b27..2c488c9a3812 100644 --- a/drivers/crypto/caam/caampkc.h +++ b/drivers/crypto/caam/caampkc.h @@ -89,18 +89,25 @@ struct caam_rsa_key { * caam_rsa_ctx - per session context. * @key : RSA key in DMA zone * @dev : device structure + * @padding_dma : dma address of padding, for adding it to the input */ struct caam_rsa_ctx { struct caam_rsa_key key; struct device *dev; + dma_addr_t padding_dma; + }; /** * caam_rsa_req_ctx - per request context. - * @src: input scatterlist (stripped of leading zeros) + * @src : input scatterlist (stripped of leading zeros) + * @fixup_src : input scatterlist (that might be stripped of leading zeros) + * @fixup_src_len : length of the fixup_src input scatterlist */ struct caam_rsa_req_ctx { struct scatterlist src[2]; + struct scatterlist *fixup_src; + unsigned int fixup_src_len; }; /** diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c index 95eb5402c59f..561bcb535184 100644 --- a/drivers/crypto/caam/caamrng.c +++ b/drivers/crypto/caam/caamrng.c @@ -3,7 +3,7 @@ * caam - Freescale FSL CAAM support for hw_random * * Copyright 2011 Freescale Semiconductor, Inc. - * Copyright 2018 NXP + * Copyright 2018-2019 NXP * * Based on caamalg.c crypto API driver. * @@ -113,10 +113,8 @@ static void rng_done(struct device *jrdev, u32 *desc, u32 err, void *context) /* Buffer refilled, invalidate cache */ dma_sync_single_for_cpu(jrdev, bd->addr, RN_BUF_SIZE, DMA_FROM_DEVICE); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "rng refreshed buf@: ", - DUMP_PREFIX_ADDRESS, 16, 4, bd->buf, RN_BUF_SIZE, 1); -#endif + print_hex_dump_debug("rng refreshed buf@: ", DUMP_PREFIX_ADDRESS, 16, 4, + bd->buf, RN_BUF_SIZE, 1); } static inline int submit_job(struct caam_rng_ctx *ctx, int to_current) @@ -209,10 +207,10 @@ static inline int rng_create_sh_desc(struct caam_rng_ctx *ctx) dev_err(jrdev, "unable to map shared descriptor\n"); return -ENOMEM; } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "rng shdesc@: ", DUMP_PREFIX_ADDRESS, 16, 4, - desc, desc_bytes(desc), 1); -#endif + + print_hex_dump_debug("rng shdesc@: ", DUMP_PREFIX_ADDRESS, 16, 4, + desc, desc_bytes(desc), 1); + return 0; } @@ -233,10 +231,10 @@ static inline int rng_create_job_desc(struct caam_rng_ctx *ctx, int buf_id) } append_seq_out_ptr_intlen(desc, bd->addr, RN_BUF_SIZE, 0); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "rng job desc@: ", DUMP_PREFIX_ADDRESS, 16, 4, - desc, desc_bytes(desc), 1); -#endif + + print_hex_dump_debug("rng job desc@: ", DUMP_PREFIX_ADDRESS, 16, 4, + desc, desc_bytes(desc), 1); + return 0; } @@ -296,47 +294,20 @@ static struct hwrng caam_rng = { .read = caam_read, }; -static void __exit caam_rng_exit(void) +void caam_rng_exit(void) { caam_jr_free(rng_ctx->jrdev); hwrng_unregister(&caam_rng); kfree(rng_ctx); } -static int __init caam_rng_init(void) +int caam_rng_init(struct device *ctrldev) { struct device *dev; - struct device_node *dev_node; - struct platform_device *pdev; - struct caam_drv_private *priv; u32 rng_inst; + struct caam_drv_private *priv = dev_get_drvdata(ctrldev); int err; - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec-v4.0"); - if (!dev_node) { - dev_node = of_find_compatible_node(NULL, NULL, "fsl,sec4.0"); - if (!dev_node) - return -ENODEV; - } - - pdev = of_find_device_by_node(dev_node); - if (!pdev) { - of_node_put(dev_node); - return -ENODEV; - } - - priv = dev_get_drvdata(&pdev->dev); - of_node_put(dev_node); - - /* - * If priv is NULL, it's probably because the caam driver wasn't - * properly initialized (e.g. RNG4 init failed). Thus, bail out here. - */ - if (!priv) { - err = -ENODEV; - goto out_put_dev; - } - /* Check for an instantiated RNG before registration */ if (priv->era < 10) rng_inst = (rd_reg32(&priv->ctrl->perfmon.cha_num_ls) & @@ -344,16 +315,13 @@ static int __init caam_rng_init(void) else rng_inst = rd_reg32(&priv->ctrl->vreg.rng) & CHA_VER_NUM_MASK; - if (!rng_inst) { - err = -ENODEV; - goto out_put_dev; - } + if (!rng_inst) + return 0; dev = caam_jr_alloc(); if (IS_ERR(dev)) { pr_err("Job Ring Device allocation for transform failed\n"); - err = PTR_ERR(dev); - goto out_put_dev; + return PTR_ERR(dev); } rng_ctx = kmalloc(sizeof(*rng_ctx), GFP_DMA | GFP_KERNEL); if (!rng_ctx) { @@ -364,7 +332,6 @@ static int __init caam_rng_init(void) if (err) goto free_rng_ctx; - put_device(&pdev->dev); dev_info(dev, "registering rng-caam\n"); return hwrng_register(&caam_rng); @@ -372,14 +339,5 @@ free_rng_ctx: kfree(rng_ctx); free_caam_alloc: caam_jr_free(dev); -out_put_dev: - put_device(&pdev->dev); return err; } - -module_init(caam_rng_init); -module_exit(caam_rng_exit); - -MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("FSL CAAM support for hw_random API"); -MODULE_AUTHOR("Freescale Semiconductor - NMG"); diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index fec39c35c877..4e43ca4d3656 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -3,7 +3,7 @@ * Controller-level driver, kernel property detection, initialization * * Copyright 2008-2012 Freescale Semiconductor, Inc. - * Copyright 2018 NXP + * Copyright 2018-2019 NXP */ #include <linux/device.h> @@ -323,8 +323,8 @@ static int caam_remove(struct platform_device *pdev) of_platform_depopulate(ctrldev); #ifdef CONFIG_CAAM_QI - if (ctrlpriv->qidev) - caam_qi_shutdown(ctrlpriv->qidev); + if (ctrlpriv->qi_init) + caam_qi_shutdown(ctrldev); #endif /* @@ -540,7 +540,8 @@ static int caam_probe(struct platform_device *pdev) ctrlpriv->caam_ipg = clk; if (!of_machine_is_compatible("fsl,imx7d") && - !of_machine_is_compatible("fsl,imx7s")) { + !of_machine_is_compatible("fsl,imx7s") && + !of_machine_is_compatible("fsl,imx7ulp")) { clk = caam_drv_identify_clk(&pdev->dev, "mem"); if (IS_ERR(clk)) { ret = PTR_ERR(clk); @@ -562,7 +563,8 @@ static int caam_probe(struct platform_device *pdev) if (!of_machine_is_compatible("fsl,imx6ul") && !of_machine_is_compatible("fsl,imx7d") && - !of_machine_is_compatible("fsl,imx7s")) { + !of_machine_is_compatible("fsl,imx7s") && + !of_machine_is_compatible("fsl,imx7ulp")) { clk = caam_drv_identify_clk(&pdev->dev, "emi_slow"); if (IS_ERR(clk)) { ret = PTR_ERR(clk); @@ -702,12 +704,7 @@ static int caam_probe(struct platform_device *pdev) } ctrlpriv->era = caam_get_era(ctrl); - - ret = of_platform_populate(nprop, caam_match, NULL, dev); - if (ret) { - dev_err(dev, "JR platform devices creation error\n"); - goto iounmap_ctrl; - } + ctrlpriv->domain = iommu_get_domain_for_dev(dev); #ifdef CONFIG_DEBUG_FS /* @@ -721,19 +718,6 @@ static int caam_probe(struct platform_device *pdev) ctrlpriv->ctl = debugfs_create_dir("ctl", ctrlpriv->dfs_root); #endif - ring = 0; - for_each_available_child_of_node(nprop, np) - if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") || - of_device_is_compatible(np, "fsl,sec4.0-job-ring")) { - ctrlpriv->jr[ring] = (struct caam_job_ring __iomem __force *) - ((__force uint8_t *)ctrl + - (ring + JR_BLOCK_NUMBER) * - BLOCK_OFFSET - ); - ctrlpriv->total_jobrs++; - ring++; - } - /* Check to see if (DPAA 1.x) QI present. If so, enable */ ctrlpriv->qi_present = !!(comp_params & CTPR_MS_QI_MASK); if (ctrlpriv->qi_present && !caam_dpaa2) { @@ -752,6 +736,25 @@ static int caam_probe(struct platform_device *pdev) #endif } + ret = of_platform_populate(nprop, caam_match, NULL, dev); + if (ret) { + dev_err(dev, "JR platform devices creation error\n"); + goto shutdown_qi; + } + + ring = 0; + for_each_available_child_of_node(nprop, np) + if (of_device_is_compatible(np, "fsl,sec-v4.0-job-ring") || + of_device_is_compatible(np, "fsl,sec4.0-job-ring")) { + ctrlpriv->jr[ring] = (struct caam_job_ring __iomem __force *) + ((__force uint8_t *)ctrl + + (ring + JR_BLOCK_NUMBER) * + BLOCK_OFFSET + ); + ctrlpriv->total_jobrs++; + ring++; + } + /* If no QI and no rings specified, quit and go home */ if ((!ctrlpriv->qi_present) && (!ctrlpriv->total_jobrs)) { dev_err(dev, "no queues configured, terminating\n"); @@ -898,6 +901,11 @@ caam_remove: caam_remove(pdev); return ret; +shutdown_qi: +#ifdef CONFIG_CAAM_QI + if (ctrlpriv->qi_init) + caam_qi_shutdown(dev); +#endif iounmap_ctrl: iounmap(ctrl); disable_caam_emi_slow: diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h index 2980b8ef1fb1..5988a26a2441 100644 --- a/drivers/crypto/caam/desc_constr.h +++ b/drivers/crypto/caam/desc_constr.h @@ -3,6 +3,7 @@ * caam descriptor construction helper functions * * Copyright 2008-2012 Freescale Semiconductor, Inc. + * Copyright 2019 NXP */ #ifndef DESC_CONSTR_H @@ -37,6 +38,16 @@ extern bool caam_little_end; +/* + * HW fetches 4 S/G table entries at a time, irrespective of how many entries + * are in the table. It's SW's responsibility to make sure these accesses + * do not have side effects. + */ +static inline int pad_sg_nents(int sg_nents) +{ + return ALIGN(sg_nents, 4); +} + static inline int desc_len(u32 * const desc) { return caam32_to_cpu(*desc) & HDR_DESCLEN_MASK; diff --git a/drivers/crypto/caam/error.c b/drivers/crypto/caam/error.c index 4da844e4b61d..4f0d45865aa2 100644 --- a/drivers/crypto/caam/error.c +++ b/drivers/crypto/caam/error.c @@ -13,7 +13,7 @@ #ifdef DEBUG #include <linux/highmem.h> -void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type, +void caam_dump_sg(const char *prefix_str, int prefix_type, int rowsize, int groupsize, struct scatterlist *sg, size_t tlen, bool ascii) { @@ -35,15 +35,15 @@ void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type, buf = it_page + it->offset; len = min_t(size_t, tlen, it->length); - print_hex_dump(level, prefix_str, prefix_type, rowsize, - groupsize, buf, len, ascii); + print_hex_dump_debug(prefix_str, prefix_type, rowsize, + groupsize, buf, len, ascii); tlen -= len; kunmap_atomic(it_page); } } #else -void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type, +void caam_dump_sg(const char *prefix_str, int prefix_type, int rowsize, int groupsize, struct scatterlist *sg, size_t tlen, bool ascii) {} diff --git a/drivers/crypto/caam/error.h b/drivers/crypto/caam/error.h index 8c6b83e02a70..d9726e66edbf 100644 --- a/drivers/crypto/caam/error.h +++ b/drivers/crypto/caam/error.h @@ -17,7 +17,7 @@ void caam_strstatus(struct device *dev, u32 status, bool qi_v2); #define caam_jr_strstatus(jrdev, status) caam_strstatus(jrdev, status, false) #define caam_qi2_strstatus(qidev, status) caam_strstatus(qidev, status, true) -void caam_dump_sg(const char *level, const char *prefix_str, int prefix_type, +void caam_dump_sg(const char *prefix_str, int prefix_type, int rowsize, int groupsize, struct scatterlist *sg, size_t tlen, bool ascii); diff --git a/drivers/crypto/caam/intern.h b/drivers/crypto/caam/intern.h index 3392615dc91b..6af84bbc612c 100644 --- a/drivers/crypto/caam/intern.h +++ b/drivers/crypto/caam/intern.h @@ -4,7 +4,7 @@ * Private/internal definitions between modules * * Copyright 2008-2011 Freescale Semiconductor, Inc. - * + * Copyright 2019 NXP */ #ifndef INTERN_H @@ -63,10 +63,6 @@ struct caam_drv_private_jr { * Driver-private storage for a single CAAM block instance */ struct caam_drv_private { -#ifdef CONFIG_CAAM_QI - struct device *qidev; -#endif - /* Physical-presence section */ struct caam_ctrl __iomem *ctrl; /* controller region */ struct caam_deco __iomem *deco; /* DECO/CCB views */ @@ -74,12 +70,17 @@ struct caam_drv_private { struct caam_queue_if __iomem *qi; /* QI control region */ struct caam_job_ring __iomem *jr[4]; /* JobR's register space */ + struct iommu_domain *domain; + /* * Detected geometry block. Filled in from device tree if powerpc, * or from register-based version detection code */ u8 total_jobrs; /* Total Job Rings in device */ u8 qi_present; /* Nonzero if QI present in device */ +#ifdef CONFIG_CAAM_QI + u8 qi_init; /* Nonzero if QI has been initialized */ +#endif u8 mc_en; /* Nonzero if MC f/w is active */ int secvio_irq; /* Security violation interrupt number */ int virt_en; /* Virtualization enabled in CAAM */ @@ -107,8 +108,95 @@ struct caam_drv_private { #endif }; -void caam_jr_algapi_init(struct device *dev); -void caam_jr_algapi_remove(struct device *dev); +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API + +int caam_algapi_init(struct device *dev); +void caam_algapi_exit(void); + +#else + +static inline int caam_algapi_init(struct device *dev) +{ + return 0; +} + +static inline void caam_algapi_exit(void) +{ +} + +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_CRYPTO_API */ + +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API + +int caam_algapi_hash_init(struct device *dev); +void caam_algapi_hash_exit(void); + +#else + +static inline int caam_algapi_hash_init(struct device *dev) +{ + return 0; +} + +static inline void caam_algapi_hash_exit(void) +{ +} + +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_AHASH_API */ + +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API + +int caam_pkc_init(struct device *dev); +void caam_pkc_exit(void); + +#else + +static inline int caam_pkc_init(struct device *dev) +{ + return 0; +} + +static inline void caam_pkc_exit(void) +{ +} + +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_PKC_API */ + +#ifdef CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API + +int caam_rng_init(struct device *dev); +void caam_rng_exit(void); + +#else + +static inline int caam_rng_init(struct device *dev) +{ + return 0; +} + +static inline void caam_rng_exit(void) +{ +} + +#endif /* CONFIG_CRYPTO_DEV_FSL_CAAM_RNG_API */ + +#ifdef CONFIG_CAAM_QI + +int caam_qi_algapi_init(struct device *dev); +void caam_qi_algapi_exit(void); + +#else + +static inline int caam_qi_algapi_init(struct device *dev) +{ + return 0; +} + +static inline void caam_qi_algapi_exit(void) +{ +} + +#endif /* CONFIG_CAAM_QI */ #ifdef CONFIG_DEBUG_FS static int caam_debugfs_u64_get(void *data, u64 *val) diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index 1de2562d0982..cea811fed320 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -4,6 +4,7 @@ * JobR backend functionality * * Copyright 2008-2012 Freescale Semiconductor, Inc. + * Copyright 2019 NXP */ #include <linux/of_irq.h> @@ -23,6 +24,43 @@ struct jr_driver_data { } ____cacheline_aligned; static struct jr_driver_data driver_data; +static DEFINE_MUTEX(algs_lock); +static unsigned int active_devs; + +static void register_algs(struct device *dev) +{ + mutex_lock(&algs_lock); + + if (++active_devs != 1) + goto algs_unlock; + + caam_algapi_init(dev); + caam_algapi_hash_init(dev); + caam_pkc_init(dev); + caam_rng_init(dev); + caam_qi_algapi_init(dev); + +algs_unlock: + mutex_unlock(&algs_lock); +} + +static void unregister_algs(void) +{ + mutex_lock(&algs_lock); + + if (--active_devs != 0) + goto algs_unlock; + + caam_qi_algapi_exit(); + + caam_rng_exit(); + caam_pkc_exit(); + caam_algapi_hash_exit(); + caam_algapi_exit(); + +algs_unlock: + mutex_unlock(&algs_lock); +} static int caam_reset_hw_jr(struct device *dev) { @@ -109,6 +147,9 @@ static int caam_jr_remove(struct platform_device *pdev) return -EBUSY; } + /* Unregister JR-based RNG & crypto algorithms */ + unregister_algs(); + /* Remove the node from Physical JobR list maintained by driver */ spin_lock(&driver_data.jr_alloc_lock); list_del(&jrpriv->list_node); @@ -541,6 +582,8 @@ static int caam_jr_probe(struct platform_device *pdev) atomic_set(&jrpriv->tfm_count, 0); + register_algs(jrdev->parent); + return 0; } diff --git a/drivers/crypto/caam/key_gen.c b/drivers/crypto/caam/key_gen.c index 8d0713fae6ac..48dd3536060d 100644 --- a/drivers/crypto/caam/key_gen.c +++ b/drivers/crypto/caam/key_gen.c @@ -16,9 +16,7 @@ void split_key_done(struct device *dev, u32 *desc, u32 err, { struct split_key_result *res = context; -#ifdef DEBUG - dev_err(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); -#endif + dev_dbg(dev, "%s %d: err 0x%x\n", __func__, __LINE__, err); if (err) caam_jr_strstatus(dev, err); @@ -55,12 +53,10 @@ int gen_split_key(struct device *jrdev, u8 *key_out, adata->keylen_pad = split_key_pad_len(adata->algtype & OP_ALG_ALGSEL_MASK); -#ifdef DEBUG - dev_err(jrdev, "split keylen %d split keylen padded %d\n", + dev_dbg(jrdev, "split keylen %d split keylen padded %d\n", adata->keylen, adata->keylen_pad); - print_hex_dump(KERN_ERR, "ctx.key@" __stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1); -#endif + print_hex_dump_debug("ctx.key@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key_in, keylen, 1); if (adata->keylen_pad > max_keylen) return -EINVAL; @@ -102,10 +98,9 @@ int gen_split_key(struct device *jrdev, u8 *key_out, append_fifo_store(desc, dma_addr, adata->keylen, LDST_CLASS_2_CCB | FIFOST_TYPE_SPLIT_KEK); -#ifdef DEBUG - print_hex_dump(KERN_ERR, "jobdesc@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), 1); -#endif + print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); result.err = 0; init_completion(&result.completion); @@ -115,11 +110,10 @@ int gen_split_key(struct device *jrdev, u8 *key_out, /* in progress */ wait_for_completion(&result.completion); ret = result.err; -#ifdef DEBUG - print_hex_dump(KERN_ERR, "ctx.key@"__stringify(__LINE__)": ", - DUMP_PREFIX_ADDRESS, 16, 4, key_out, - adata->keylen_pad, 1); -#endif + + print_hex_dump_debug("ctx.key@"__stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, key_out, + adata->keylen_pad, 1); } dma_unmap_single(jrdev, dma_addr, adata->keylen_pad, DMA_BIDIRECTIONAL); diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c index 9f08f84cca59..0fe618e3804a 100644 --- a/drivers/crypto/caam/qi.c +++ b/drivers/crypto/caam/qi.c @@ -4,7 +4,7 @@ * Queue Interface backend functionality * * Copyright 2013-2016 Freescale Semiconductor, Inc. - * Copyright 2016-2017 NXP + * Copyright 2016-2017, 2019 NXP */ #include <linux/cpumask.h> @@ -18,6 +18,7 @@ #include "desc_constr.h" #define PREHDR_RSLS_SHIFT 31 +#define PREHDR_ABS BIT(25) /* * Use a reasonable backlog of frames (per CPU) as congestion threshold, @@ -58,11 +59,9 @@ static DEFINE_PER_CPU(int, last_cpu); /* * caam_qi_priv - CAAM QI backend private params * @cgr: QMan congestion group - * @qi_pdev: platform device for QI backend */ struct caam_qi_priv { struct qman_cgr cgr; - struct platform_device *qi_pdev; }; static struct caam_qi_priv qipriv ____cacheline_aligned; @@ -95,6 +94,16 @@ static u64 times_congested; */ static struct kmem_cache *qi_cache; +static void *caam_iova_to_virt(struct iommu_domain *domain, + dma_addr_t iova_addr) +{ + phys_addr_t phys_addr; + + phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr; + + return phys_to_virt(phys_addr); +} + int caam_qi_enqueue(struct device *qidev, struct caam_drv_req *req) { struct qm_fd fd; @@ -135,6 +144,7 @@ static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq, const struct qm_fd *fd; struct caam_drv_req *drv_req; struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev); + struct caam_drv_private *priv = dev_get_drvdata(qidev); fd = &msg->ern.fd; @@ -143,7 +153,7 @@ static void caam_fq_ern_cb(struct qman_portal *qm, struct qman_fq *fq, return; } - drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd)); + drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd)); if (!drv_req) { dev_err(qidev, "Can't find original request for CAAM response\n"); @@ -346,6 +356,7 @@ int caam_drv_ctx_update(struct caam_drv_ctx *drv_ctx, u32 *sh_desc) */ drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) | num_words); + drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS); memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc)); dma_sync_single_for_device(qidev, drv_ctx->context_a, sizeof(drv_ctx->sh_desc) + @@ -401,6 +412,7 @@ struct caam_drv_ctx *caam_drv_ctx_init(struct device *qidev, */ drv_ctx->prehdr[0] = cpu_to_caam32((1 << PREHDR_RSLS_SHIFT) | num_words); + drv_ctx->prehdr[1] = cpu_to_caam32(PREHDR_ABS); memcpy(drv_ctx->sh_desc, sh_desc, desc_bytes(sh_desc)); size = sizeof(drv_ctx->prehdr) + sizeof(drv_ctx->sh_desc); hwdesc = dma_map_single(qidev, drv_ctx->prehdr, size, @@ -488,7 +500,7 @@ EXPORT_SYMBOL(caam_drv_ctx_rel); void caam_qi_shutdown(struct device *qidev) { int i; - struct caam_qi_priv *priv = dev_get_drvdata(qidev); + struct caam_qi_priv *priv = &qipriv; const cpumask_t *cpus = qman_affine_cpus(); for_each_cpu(i, cpus) { @@ -506,8 +518,6 @@ void caam_qi_shutdown(struct device *qidev) qman_release_cgrid(priv->cgr.cgrid); kmem_cache_destroy(qi_cache); - - platform_device_unregister(priv->qi_pdev); } static void cgr_cb(struct qman_portal *qm, struct qman_cgr *cgr, int congested) @@ -550,6 +560,7 @@ static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p, struct caam_drv_req *drv_req; const struct qm_fd *fd; struct device *qidev = &(raw_cpu_ptr(&pcpu_qipriv)->net_dev.dev); + struct caam_drv_private *priv = dev_get_drvdata(qidev); u32 status; if (caam_qi_napi_schedule(p, caam_napi)) @@ -572,7 +583,7 @@ static enum qman_cb_dqrr_result caam_rsp_fq_dqrr_cb(struct qman_portal *p, return qman_cb_dqrr_consume; } - drv_req = (struct caam_drv_req *)phys_to_virt(qm_fd_addr_get64(fd)); + drv_req = caam_iova_to_virt(priv->domain, qm_fd_addr_get64(fd)); if (unlikely(!drv_req)) { dev_err(qidev, "Can't find original request for caam response\n"); @@ -692,33 +703,17 @@ static void free_rsp_fqs(void) int caam_qi_init(struct platform_device *caam_pdev) { int err, i; - struct platform_device *qi_pdev; struct device *ctrldev = &caam_pdev->dev, *qidev; struct caam_drv_private *ctrlpriv; const cpumask_t *cpus = qman_affine_cpus(); - static struct platform_device_info qi_pdev_info = { - .name = "caam_qi", - .id = PLATFORM_DEVID_NONE - }; - - qi_pdev_info.parent = ctrldev; - qi_pdev_info.dma_mask = dma_get_mask(ctrldev); - qi_pdev = platform_device_register_full(&qi_pdev_info); - if (IS_ERR(qi_pdev)) - return PTR_ERR(qi_pdev); - set_dma_ops(&qi_pdev->dev, get_dma_ops(ctrldev)); ctrlpriv = dev_get_drvdata(ctrldev); - qidev = &qi_pdev->dev; - - qipriv.qi_pdev = qi_pdev; - dev_set_drvdata(qidev, &qipriv); + qidev = ctrldev; /* Initialize the congestion detection */ err = init_cgr(qidev); if (err) { dev_err(qidev, "CGR initialization failed: %d\n", err); - platform_device_unregister(qi_pdev); return err; } @@ -727,7 +722,6 @@ int caam_qi_init(struct platform_device *caam_pdev) if (err) { dev_err(qidev, "Can't allocate CAAM response FQs: %d\n", err); free_rsp_fqs(); - platform_device_unregister(qi_pdev); return err; } @@ -750,15 +744,11 @@ int caam_qi_init(struct platform_device *caam_pdev) napi_enable(irqtask); } - /* Hook up QI device to parent controlling caam device */ - ctrlpriv->qidev = qidev; - qi_cache = kmem_cache_create("caamqicache", CAAM_QI_MEMCACHE_SIZE, 0, SLAB_CACHE_DMA, NULL); if (!qi_cache) { dev_err(qidev, "Can't allocate CAAM cache\n"); free_rsp_fqs(); - platform_device_unregister(qi_pdev); return -ENOMEM; } @@ -766,6 +756,8 @@ int caam_qi_init(struct platform_device *caam_pdev) debugfs_create_file("qi_congested", 0444, ctrlpriv->ctl, ×_congested, &caam_fops_u64_ro); #endif + + ctrlpriv->qi_init = 1; dev_info(qidev, "Linux CAAM Queue I/F driver initialised\n"); return 0; } diff --git a/drivers/crypto/caam/sg_sw_qm.h b/drivers/crypto/caam/sg_sw_qm.h index b3e1aaaeffea..d56cc7efbc13 100644 --- a/drivers/crypto/caam/sg_sw_qm.h +++ b/drivers/crypto/caam/sg_sw_qm.h @@ -54,15 +54,19 @@ static inline void dma_to_qm_sg_one_last_ext(struct qm_sg_entry *qm_sg_ptr, * but does not have final bit; instead, returns last entry */ static inline struct qm_sg_entry * -sg_to_qm_sg(struct scatterlist *sg, int sg_count, +sg_to_qm_sg(struct scatterlist *sg, int len, struct qm_sg_entry *qm_sg_ptr, u16 offset) { - while (sg_count && sg) { - dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), - sg_dma_len(sg), offset); + int ent_len; + + while (len) { + ent_len = min_t(int, sg_dma_len(sg), len); + + dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), ent_len, + offset); qm_sg_ptr++; sg = sg_next(sg); - sg_count--; + len -= ent_len; } return qm_sg_ptr - 1; } @@ -71,10 +75,10 @@ sg_to_qm_sg(struct scatterlist *sg, int sg_count, * convert scatterlist to h/w link table format * scatterlist must have been previously dma mapped */ -static inline void sg_to_qm_sg_last(struct scatterlist *sg, int sg_count, +static inline void sg_to_qm_sg_last(struct scatterlist *sg, int len, struct qm_sg_entry *qm_sg_ptr, u16 offset) { - qm_sg_ptr = sg_to_qm_sg(sg, sg_count, qm_sg_ptr, offset); + qm_sg_ptr = sg_to_qm_sg(sg, len, qm_sg_ptr, offset); qm_sg_entry_set_f(qm_sg_ptr, qm_sg_entry_get_len(qm_sg_ptr)); } diff --git a/drivers/crypto/caam/sg_sw_qm2.h b/drivers/crypto/caam/sg_sw_qm2.h index c9378402a5f8..b8b737d2b0ea 100644 --- a/drivers/crypto/caam/sg_sw_qm2.h +++ b/drivers/crypto/caam/sg_sw_qm2.h @@ -25,15 +25,19 @@ static inline void dma_to_qm_sg_one(struct dpaa2_sg_entry *qm_sg_ptr, * but does not have final bit; instead, returns last entry */ static inline struct dpaa2_sg_entry * -sg_to_qm_sg(struct scatterlist *sg, int sg_count, +sg_to_qm_sg(struct scatterlist *sg, int len, struct dpaa2_sg_entry *qm_sg_ptr, u16 offset) { - while (sg_count && sg) { - dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), - sg_dma_len(sg), offset); + int ent_len; + + while (len) { + ent_len = min_t(int, sg_dma_len(sg), len); + + dma_to_qm_sg_one(qm_sg_ptr, sg_dma_address(sg), ent_len, + offset); qm_sg_ptr++; sg = sg_next(sg); - sg_count--; + len -= ent_len; } return qm_sg_ptr - 1; } @@ -42,11 +46,11 @@ sg_to_qm_sg(struct scatterlist *sg, int sg_count, * convert scatterlist to h/w link table format * scatterlist must have been previously dma mapped */ -static inline void sg_to_qm_sg_last(struct scatterlist *sg, int sg_count, +static inline void sg_to_qm_sg_last(struct scatterlist *sg, int len, struct dpaa2_sg_entry *qm_sg_ptr, u16 offset) { - qm_sg_ptr = sg_to_qm_sg(sg, sg_count, qm_sg_ptr, offset); + qm_sg_ptr = sg_to_qm_sg(sg, len, qm_sg_ptr, offset); dpaa2_sg_set_final(qm_sg_ptr, true); } diff --git a/drivers/crypto/caam/sg_sw_sec4.h b/drivers/crypto/caam/sg_sw_sec4.h index dbfa9fce33e0..07e1ee99273b 100644 --- a/drivers/crypto/caam/sg_sw_sec4.h +++ b/drivers/crypto/caam/sg_sw_sec4.h @@ -35,11 +35,9 @@ static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr, sec4_sg_ptr->bpid_offset = cpu_to_caam32(offset & SEC4_SG_OFFSET_MASK); } -#ifdef DEBUG - print_hex_dump(KERN_ERR, "sec4_sg_ptr@: ", - DUMP_PREFIX_ADDRESS, 16, 4, sec4_sg_ptr, - sizeof(struct sec4_sg_entry), 1); -#endif + + print_hex_dump_debug("sec4_sg_ptr@: ", DUMP_PREFIX_ADDRESS, 16, 4, + sec4_sg_ptr, sizeof(struct sec4_sg_entry), 1); } /* @@ -47,15 +45,19 @@ static inline void dma_to_sec4_sg_one(struct sec4_sg_entry *sec4_sg_ptr, * but does not have final bit; instead, returns last entry */ static inline struct sec4_sg_entry * -sg_to_sec4_sg(struct scatterlist *sg, int sg_count, +sg_to_sec4_sg(struct scatterlist *sg, int len, struct sec4_sg_entry *sec4_sg_ptr, u16 offset) { - while (sg_count) { - dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg), - sg_dma_len(sg), offset); + int ent_len; + + while (len) { + ent_len = min_t(int, sg_dma_len(sg), len); + + dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg), ent_len, + offset); sec4_sg_ptr++; sg = sg_next(sg); - sg_count--; + len -= ent_len; } return sec4_sg_ptr - 1; } @@ -72,11 +74,11 @@ static inline void sg_to_sec4_set_last(struct sec4_sg_entry *sec4_sg_ptr) * convert scatterlist to h/w link table format * scatterlist must have been previously dma mapped */ -static inline void sg_to_sec4_sg_last(struct scatterlist *sg, int sg_count, +static inline void sg_to_sec4_sg_last(struct scatterlist *sg, int len, struct sec4_sg_entry *sec4_sg_ptr, u16 offset) { - sec4_sg_ptr = sg_to_sec4_sg(sg, sg_count, sec4_sg_ptr, offset); + sec4_sg_ptr = sg_to_sec4_sg(sg, len, sec4_sg_ptr, offset); sg_to_sec4_set_last(sec4_sg_ptr); } diff --git a/drivers/crypto/cavium/cpt/cptvf_algs.c b/drivers/crypto/cavium/cpt/cptvf_algs.c index e9f4704494fb..ff3cb1f8f2b6 100644 --- a/drivers/crypto/cavium/cpt/cptvf_algs.c +++ b/drivers/crypto/cavium/cpt/cptvf_algs.c @@ -7,7 +7,6 @@ #include <crypto/aes.h> #include <crypto/algapi.h> #include <crypto/authenc.h> -#include <crypto/crypto_wq.h> #include <crypto/des.h> #include <crypto/xts.h> #include <linux/crypto.h> diff --git a/drivers/crypto/cavium/nitrox/nitrox_debugfs.h b/drivers/crypto/cavium/nitrox/nitrox_debugfs.h index f177b79bbab0..09c4cf2513fb 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_debugfs.h +++ b/drivers/crypto/cavium/nitrox/nitrox_debugfs.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __NITROX_DEBUGFS_H #define __NITROX_DEBUGFS_H diff --git a/drivers/crypto/cavium/nitrox/nitrox_mbx.h b/drivers/crypto/cavium/nitrox/nitrox_mbx.h index 5008399775a9..7c93d0282174 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_mbx.h +++ b/drivers/crypto/cavium/nitrox/nitrox_mbx.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef __NITROX_MBX_H #define __NITROX_MBX_H diff --git a/drivers/crypto/ccp/ccp-crypto-aes.c b/drivers/crypto/ccp/ccp-crypto-aes.c index ea3d6de55ff6..58c6dddfc5e1 100644 --- a/drivers/crypto/ccp/ccp-crypto-aes.c +++ b/drivers/crypto/ccp/ccp-crypto-aes.c @@ -2,7 +2,7 @@ /* * AMD Cryptographic Coprocessor (CCP) AES crypto API support * - * Copyright (C) 2013,2016 Advanced Micro Devices, Inc. + * Copyright (C) 2013-2019 Advanced Micro Devices, Inc. * * Author: Tom Lendacky <thomas.lendacky@amd.com> */ @@ -76,8 +76,7 @@ static int ccp_aes_crypt(struct ablkcipher_request *req, bool encrypt) return -EINVAL; if (((ctx->u.aes.mode == CCP_AES_MODE_ECB) || - (ctx->u.aes.mode == CCP_AES_MODE_CBC) || - (ctx->u.aes.mode == CCP_AES_MODE_CFB)) && + (ctx->u.aes.mode == CCP_AES_MODE_CBC)) && (req->nbytes & (AES_BLOCK_SIZE - 1))) return -EINVAL; @@ -288,7 +287,7 @@ static struct ccp_aes_def aes_algs[] = { .version = CCP_VERSION(3, 0), .name = "cfb(aes)", .driver_name = "cfb-aes-ccp", - .blocksize = AES_BLOCK_SIZE, + .blocksize = 1, .ivsize = AES_BLOCK_SIZE, .alg_defaults = &ccp_aes_defaults, }, diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c index cc3e96c4f5fb..f79eede71c62 100644 --- a/drivers/crypto/ccp/ccp-dev.c +++ b/drivers/crypto/ccp/ccp-dev.c @@ -32,56 +32,62 @@ struct ccp_tasklet_data { }; /* Human-readable error strings */ +#define CCP_MAX_ERROR_CODE 64 static char *ccp_error_codes[] = { "", - "ERR 01: ILLEGAL_ENGINE", - "ERR 02: ILLEGAL_KEY_ID", - "ERR 03: ILLEGAL_FUNCTION_TYPE", - "ERR 04: ILLEGAL_FUNCTION_MODE", - "ERR 05: ILLEGAL_FUNCTION_ENCRYPT", - "ERR 06: ILLEGAL_FUNCTION_SIZE", - "ERR 07: Zlib_MISSING_INIT_EOM", - "ERR 08: ILLEGAL_FUNCTION_RSVD", - "ERR 09: ILLEGAL_BUFFER_LENGTH", - "ERR 10: VLSB_FAULT", - "ERR 11: ILLEGAL_MEM_ADDR", - "ERR 12: ILLEGAL_MEM_SEL", - "ERR 13: ILLEGAL_CONTEXT_ID", - "ERR 14: ILLEGAL_KEY_ADDR", - "ERR 15: 0xF Reserved", - "ERR 16: Zlib_ILLEGAL_MULTI_QUEUE", - "ERR 17: Zlib_ILLEGAL_JOBID_CHANGE", - "ERR 18: CMD_TIMEOUT", - "ERR 19: IDMA0_AXI_SLVERR", - "ERR 20: IDMA0_AXI_DECERR", - "ERR 21: 0x15 Reserved", - "ERR 22: IDMA1_AXI_SLAVE_FAULT", - "ERR 23: IDMA1_AIXI_DECERR", - "ERR 24: 0x18 Reserved", - "ERR 25: ZLIBVHB_AXI_SLVERR", - "ERR 26: ZLIBVHB_AXI_DECERR", - "ERR 27: 0x1B Reserved", - "ERR 27: ZLIB_UNEXPECTED_EOM", - "ERR 27: ZLIB_EXTRA_DATA", - "ERR 30: ZLIB_BTYPE", - "ERR 31: ZLIB_UNDEFINED_SYMBOL", - "ERR 32: ZLIB_UNDEFINED_DISTANCE_S", - "ERR 33: ZLIB_CODE_LENGTH_SYMBOL", - "ERR 34: ZLIB _VHB_ILLEGAL_FETCH", - "ERR 35: ZLIB_UNCOMPRESSED_LEN", - "ERR 36: ZLIB_LIMIT_REACHED", - "ERR 37: ZLIB_CHECKSUM_MISMATCH0", - "ERR 38: ODMA0_AXI_SLVERR", - "ERR 39: ODMA0_AXI_DECERR", - "ERR 40: 0x28 Reserved", - "ERR 41: ODMA1_AXI_SLVERR", - "ERR 42: ODMA1_AXI_DECERR", - "ERR 43: LSB_PARITY_ERR", + "ILLEGAL_ENGINE", + "ILLEGAL_KEY_ID", + "ILLEGAL_FUNCTION_TYPE", + "ILLEGAL_FUNCTION_MODE", + "ILLEGAL_FUNCTION_ENCRYPT", + "ILLEGAL_FUNCTION_SIZE", + "Zlib_MISSING_INIT_EOM", + "ILLEGAL_FUNCTION_RSVD", + "ILLEGAL_BUFFER_LENGTH", + "VLSB_FAULT", + "ILLEGAL_MEM_ADDR", + "ILLEGAL_MEM_SEL", + "ILLEGAL_CONTEXT_ID", + "ILLEGAL_KEY_ADDR", + "0xF Reserved", + "Zlib_ILLEGAL_MULTI_QUEUE", + "Zlib_ILLEGAL_JOBID_CHANGE", + "CMD_TIMEOUT", + "IDMA0_AXI_SLVERR", + "IDMA0_AXI_DECERR", + "0x15 Reserved", + "IDMA1_AXI_SLAVE_FAULT", + "IDMA1_AIXI_DECERR", + "0x18 Reserved", + "ZLIBVHB_AXI_SLVERR", + "ZLIBVHB_AXI_DECERR", + "0x1B Reserved", + "ZLIB_UNEXPECTED_EOM", + "ZLIB_EXTRA_DATA", + "ZLIB_BTYPE", + "ZLIB_UNDEFINED_SYMBOL", + "ZLIB_UNDEFINED_DISTANCE_S", + "ZLIB_CODE_LENGTH_SYMBOL", + "ZLIB _VHB_ILLEGAL_FETCH", + "ZLIB_UNCOMPRESSED_LEN", + "ZLIB_LIMIT_REACHED", + "ZLIB_CHECKSUM_MISMATCH0", + "ODMA0_AXI_SLVERR", + "ODMA0_AXI_DECERR", + "0x28 Reserved", + "ODMA1_AXI_SLVERR", + "ODMA1_AXI_DECERR", }; -void ccp_log_error(struct ccp_device *d, int e) +void ccp_log_error(struct ccp_device *d, unsigned int e) { - dev_err(d->dev, "CCP error: %s (0x%x)\n", ccp_error_codes[e], e); + if (WARN_ON(e >= CCP_MAX_ERROR_CODE)) + return; + + if (e < ARRAY_SIZE(ccp_error_codes)) + dev_err(d->dev, "CCP error %d: %s\n", e, ccp_error_codes[e]); + else + dev_err(d->dev, "CCP error %d: Unknown Error\n", e); } /* List of CCPs, CCP count, read-write access lock, and access functions diff --git a/drivers/crypto/ccp/ccp-dev.h b/drivers/crypto/ccp/ccp-dev.h index 90523a069bff..5e624920fd99 100644 --- a/drivers/crypto/ccp/ccp-dev.h +++ b/drivers/crypto/ccp/ccp-dev.h @@ -629,7 +629,7 @@ struct ccp5_desc { void ccp_add_device(struct ccp_device *ccp); void ccp_del_device(struct ccp_device *ccp); -extern void ccp_log_error(struct ccp_device *, int); +extern void ccp_log_error(struct ccp_device *, unsigned int); struct ccp_device *ccp_alloc_struct(struct sp_device *sp); bool ccp_queues_suspended(struct ccp_device *ccp); diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c index db8de89d990f..866b2e05ca77 100644 --- a/drivers/crypto/ccp/ccp-ops.c +++ b/drivers/crypto/ccp/ccp-ops.c @@ -2,7 +2,7 @@ /* * AMD Cryptographic Coprocessor (CCP) driver * - * Copyright (C) 2013,2018 Advanced Micro Devices, Inc. + * Copyright (C) 2013-2019 Advanced Micro Devices, Inc. * * Author: Tom Lendacky <thomas.lendacky@amd.com> * Author: Gary R Hook <gary.hook@amd.com> @@ -890,8 +890,7 @@ static int ccp_run_aes_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) return -EINVAL; if (((aes->mode == CCP_AES_MODE_ECB) || - (aes->mode == CCP_AES_MODE_CBC) || - (aes->mode == CCP_AES_MODE_CFB)) && + (aes->mode == CCP_AES_MODE_CBC)) && (aes->src_len & (AES_BLOCK_SIZE - 1))) return -EINVAL; @@ -1264,6 +1263,9 @@ static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) int ret; /* Error checks */ + if (cmd_q->ccp->vdata->version < CCP_VERSION(5, 0)) + return -EINVAL; + if (!cmd_q->ccp->vdata->perform->des3) return -EINVAL; @@ -1346,8 +1348,6 @@ static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) * passthru option to convert from big endian to little endian. */ if (des3->mode != CCP_DES3_MODE_ECB) { - u32 load_mode; - op.sb_ctx = cmd_q->sb_ctx; ret = ccp_init_dm_workarea(&ctx, cmd_q, @@ -1363,12 +1363,8 @@ static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) if (ret) goto e_ctx; - if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) - load_mode = CCP_PASSTHRU_BYTESWAP_NOOP; - else - load_mode = CCP_PASSTHRU_BYTESWAP_256BIT; ret = ccp_copy_to_sb(cmd_q, &ctx, op.jobid, op.sb_ctx, - load_mode); + CCP_PASSTHRU_BYTESWAP_256BIT); if (ret) { cmd->engine_error = cmd_q->cmd_error; goto e_ctx; @@ -1430,10 +1426,6 @@ static int ccp_run_des3_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) } /* ...but we only need the last DES3_EDE_BLOCK_SIZE bytes */ - if (cmd_q->ccp->vdata->version == CCP_VERSION(3, 0)) - dm_offset = CCP_SB_BYTES - des3->iv_len; - else - dm_offset = 0; ccp_get_dm_area(&ctx, dm_offset, des3->iv, 0, DES3_EDE_BLOCK_SIZE); } diff --git a/drivers/crypto/ccree/cc_driver.c b/drivers/crypto/ccree/cc_driver.c index 86ac7b443355..980aa04b655b 100644 --- a/drivers/crypto/ccree/cc_driver.c +++ b/drivers/crypto/ccree/cc_driver.c @@ -48,6 +48,7 @@ struct cc_hw_data { }; #define CC_NUM_IDRS 4 +#define CC_HW_RESET_LOOP_COUNT 10 /* Note: PIDR3 holds CMOD/Rev so ignored for HW identification purposes */ static const u32 pidr_0124_offsets[CC_NUM_IDRS] = { @@ -133,6 +134,9 @@ static irqreturn_t cc_isr(int irq, void *dev_id) u32 imr; /* STAT_OP_TYPE_GENERIC STAT_PHASE_0: Interrupt */ + /* if driver suspended return, probebly shared interrupt */ + if (cc_pm_is_dev_suspended(dev)) + return IRQ_NONE; /* read the interrupt status */ irr = cc_ioread(drvdata, CC_REG(HOST_IRR)); @@ -188,6 +192,31 @@ static irqreturn_t cc_isr(int irq, void *dev_id) return IRQ_HANDLED; } +bool cc_wait_for_reset_completion(struct cc_drvdata *drvdata) +{ + unsigned int val; + unsigned int i; + + /* 712/710/63 has no reset completion indication, always return true */ + if (drvdata->hw_rev <= CC_HW_REV_712) + return true; + + for (i = 0; i < CC_HW_RESET_LOOP_COUNT; i++) { + /* in cc7x3 NVM_IS_IDLE indicates that CC reset is + * completed and device is fully functional + */ + val = cc_ioread(drvdata, CC_REG(NVM_IS_IDLE)); + if (val & CC_NVM_IS_IDLE_MASK) { + /* hw indicate reset completed */ + return true; + } + /* allow scheduling other process on the processor */ + schedule(); + } + /* reset not completed */ + return false; +} + int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe) { unsigned int val, cache_params; @@ -315,15 +344,6 @@ static int init_cc_resources(struct platform_device *plat_dev) return new_drvdata->irq; } - rc = devm_request_irq(dev, new_drvdata->irq, cc_isr, - IRQF_SHARED, "ccree", new_drvdata); - if (rc) { - dev_err(dev, "Could not register to interrupt %d\n", - new_drvdata->irq); - return rc; - } - dev_dbg(dev, "Registered to IRQ: %d\n", new_drvdata->irq); - init_completion(&new_drvdata->hw_queue_avail); if (!plat_dev->dev.dma_mask) @@ -352,6 +372,11 @@ static int init_cc_resources(struct platform_device *plat_dev) new_drvdata->sec_disabled = cc_sec_disable; + /* wait for Crytpcell reset completion */ + if (!cc_wait_for_reset_completion(new_drvdata)) { + dev_err(dev, "Cryptocell reset not completed"); + } + if (hw_rev->rev <= CC_HW_REV_712) { /* Verify correct mapping */ val = cc_ioread(new_drvdata, new_drvdata->sig_offset); @@ -383,6 +408,24 @@ static int init_cc_resources(struct platform_device *plat_dev) } sig_cidr = val; + /* Check HW engine configuration */ + val = cc_ioread(new_drvdata, CC_REG(HOST_REMOVE_INPUT_PINS)); + switch (val) { + case CC_PINS_FULL: + /* This is fine */ + break; + case CC_PINS_SLIM: + if (new_drvdata->std_bodies & CC_STD_NIST) { + dev_warn(dev, "703 mode forced due to HW configuration.\n"); + new_drvdata->std_bodies = CC_STD_OSCCA; + } + break; + default: + dev_err(dev, "Unsupported engines configration.\n"); + rc = -EINVAL; + goto post_clk_err; + } + /* Check security disable state */ val = cc_ioread(new_drvdata, CC_REG(SECURITY_DISABLED)); val &= CC_SECURITY_DISABLED_MASK; @@ -401,6 +444,15 @@ static int init_cc_resources(struct platform_device *plat_dev) /* Display HW versions */ dev_info(dev, "ARM CryptoCell %s Driver: HW version 0x%08X/0x%8X, Driver version %s\n", hw_rev->name, hw_rev_pidr, sig_cidr, DRV_MODULE_VERSION); + /* register the driver isr function */ + rc = devm_request_irq(dev, new_drvdata->irq, cc_isr, + IRQF_SHARED, "ccree", new_drvdata); + if (rc) { + dev_err(dev, "Could not register to interrupt %d\n", + new_drvdata->irq); + goto post_clk_err; + } + dev_dbg(dev, "Registered to IRQ: %d\n", new_drvdata->irq); rc = init_cc_regs(new_drvdata, true); if (rc) { diff --git a/drivers/crypto/ccree/cc_driver.h b/drivers/crypto/ccree/cc_driver.h index b76181335c08..7cd99380bf1f 100644 --- a/drivers/crypto/ccree/cc_driver.h +++ b/drivers/crypto/ccree/cc_driver.h @@ -53,6 +53,9 @@ enum cc_std_body { #define CC_COHERENT_CACHE_PARAMS 0xEEE +#define CC_PINS_FULL 0x0 +#define CC_PINS_SLIM 0x9F + /* Maximum DMA mask supported by IP */ #define DMA_BIT_MASK_LEN 48 @@ -67,6 +70,8 @@ enum cc_std_body { #define CC_SECURITY_DISABLED_MASK BIT(CC_SECURITY_DISABLED_VALUE_BIT_SHIFT) +#define CC_NVM_IS_IDLE_MASK BIT(CC_NVM_IS_IDLE_VALUE_BIT_SHIFT) + #define AXIM_MON_COMP_VALUE GENMASK(CC_AXIM_MON_COMP_VALUE_BIT_SIZE + \ CC_AXIM_MON_COMP_VALUE_BIT_SHIFT, \ CC_AXIM_MON_COMP_VALUE_BIT_SHIFT) @@ -216,6 +221,7 @@ static inline void dump_byte_array(const char *name, const u8 *the_array, __dump_byte_array(name, the_array, size); } +bool cc_wait_for_reset_completion(struct cc_drvdata *drvdata); int init_cc_regs(struct cc_drvdata *drvdata, bool is_probe); void fini_cc_regs(struct cc_drvdata *drvdata); int cc_clk_on(struct cc_drvdata *drvdata); diff --git a/drivers/crypto/ccree/cc_host_regs.h b/drivers/crypto/ccree/cc_host_regs.h index d0764147573f..efe3e1d8b87b 100644 --- a/drivers/crypto/ccree/cc_host_regs.h +++ b/drivers/crypto/ccree/cc_host_regs.h @@ -114,6 +114,9 @@ #define CC_HOST_ICR_DSCRPTR_WATERMARK_QUEUE0_CLEAR_BIT_SIZE 0x1UL #define CC_HOST_ICR_AXIM_COMP_INT_CLEAR_BIT_SHIFT 0x17UL #define CC_HOST_ICR_AXIM_COMP_INT_CLEAR_BIT_SIZE 0x1UL +#define CC_NVM_IS_IDLE_REG_OFFSET 0x0A10UL +#define CC_NVM_IS_IDLE_VALUE_BIT_SHIFT 0x0UL +#define CC_NVM_IS_IDLE_VALUE_BIT_SIZE 0x1UL #define CC_SECURITY_DISABLED_REG_OFFSET 0x0A1CUL #define CC_SECURITY_DISABLED_VALUE_BIT_SHIFT 0x0UL #define CC_SECURITY_DISABLED_VALUE_BIT_SIZE 0x1UL @@ -203,6 +206,23 @@ #define CC_HOST_POWER_DOWN_EN_REG_OFFSET 0xA78UL #define CC_HOST_POWER_DOWN_EN_VALUE_BIT_SHIFT 0x0UL #define CC_HOST_POWER_DOWN_EN_VALUE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REG_OFFSET 0x0A7CUL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_AES_ENGINE_BIT_SHIFT 0x0UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_AES_ENGINE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_AES_MAC_ENGINE_BIT_SHIFT 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_AES_MAC_ENGINE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_GHASH_ENGINE_BIT_SHIFT 0x2UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_GHASH_ENGINE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_DES_ENGINE_BIT_SHIFT 0x3UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_DES_ENGINE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_HASH_ENGINE_BIT_SHIFT 0x4UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_HASH_ENGINE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_SM3_ENGINE_BIT_SHIFT 0x5UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_SM3_ENGINE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_SM4_ENGINE_BIT_SHIFT 0x6UL +#define CC_HOST_REMOVE_INPUT_PINS_REMOVE_SM4_ENGINE_BIT_SIZE 0x1UL +#define CC_HOST_REMOVE_INPUT_PINS_OTP_DISCONNECTED_BIT_SHIFT 0x7UL +#define CC_HOST_REMOVE_INPUT_PINS_OTP_DISCONNECTED_BIT_SIZE 0x1UL // -------------------------------------- // BLOCK: ID_REGISTERS // -------------------------------------- diff --git a/drivers/crypto/ccree/cc_pm.c b/drivers/crypto/ccree/cc_pm.c index 2dad9c9543c6..899a52f05b7a 100644 --- a/drivers/crypto/ccree/cc_pm.c +++ b/drivers/crypto/ccree/cc_pm.c @@ -49,6 +49,11 @@ int cc_pm_resume(struct device *dev) dev_err(dev, "failed getting clock back on. We're toast.\n"); return rc; } + /* wait for Crytpcell reset completion */ + if (!cc_wait_for_reset_completion(drvdata)) { + dev_err(dev, "Cryptocell reset not completed"); + return -EBUSY; + } cc_iowrite(drvdata, CC_REG(HOST_POWER_DOWN_EN), POWER_DOWN_DISABLE); rc = init_cc_regs(drvdata, false); @@ -101,6 +106,12 @@ int cc_pm_put_suspend(struct device *dev) return rc; } +bool cc_pm_is_dev_suspended(struct device *dev) +{ + /* check device state using runtime api */ + return pm_runtime_suspended(dev); +} + int cc_pm_init(struct cc_drvdata *drvdata) { struct device *dev = drvdata_to_dev(drvdata); diff --git a/drivers/crypto/ccree/cc_pm.h b/drivers/crypto/ccree/cc_pm.h index 6190cdba5dad..a7d98a5da2e1 100644 --- a/drivers/crypto/ccree/cc_pm.h +++ b/drivers/crypto/ccree/cc_pm.h @@ -22,6 +22,7 @@ int cc_pm_suspend(struct device *dev); int cc_pm_resume(struct device *dev); int cc_pm_get(struct device *dev); int cc_pm_put_suspend(struct device *dev); +bool cc_pm_is_dev_suspended(struct device *dev); #else @@ -54,6 +55,12 @@ static inline int cc_pm_put_suspend(struct device *dev) return 0; } +static inline bool cc_pm_is_dev_suspended(struct device *dev) +{ + /* if PM not supported device is never suspend */ + return false; +} + #endif #endif /*__POWER_MGR_H__*/ diff --git a/drivers/crypto/hisilicon/sec/sec_drv.h b/drivers/crypto/hisilicon/sec/sec_drv.h index 2d2f186674ba..4d9063a8b10b 100644 --- a/drivers/crypto/hisilicon/sec/sec_drv.h +++ b/drivers/crypto/hisilicon/sec/sec_drv.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2016-2017 Hisilicon Limited. */ #ifndef _SEC_DRV_H_ diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index 86c699c14f84..df43a2c6933b 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -398,6 +398,12 @@ static int safexcel_hw_init(struct safexcel_crypto_priv *priv) /* Processing Engine configuration */ + /* Token & context configuration */ + val = EIP197_PE_EIP96_TOKEN_CTRL_CTX_UPDATES | + EIP197_PE_EIP96_TOKEN_CTRL_REUSE_CTX | + EIP197_PE_EIP96_TOKEN_CTRL_POST_REUSE_CTX; + writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL(pe)); + /* H/W capabilities selection */ val = EIP197_FUNCTION_RSVD; val |= EIP197_PROTOCOL_ENCRYPT_ONLY | EIP197_PROTOCOL_HASH_ONLY; @@ -589,9 +595,9 @@ inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv, if (rdesc->result_data.error_code & 0x407f) { /* Fatal error (bits 0-7, 14) */ dev_err(priv->dev, - "cipher: result: result descriptor error (%d)\n", + "cipher: result: result descriptor error (0x%x)\n", rdesc->result_data.error_code); - return -EIO; + return -EINVAL; } else if (rdesc->result_data.error_code == BIT(9)) { /* Authentication failed */ return -EBADMSG; @@ -720,11 +726,10 @@ handle_results: } acknowledge: - if (i) { + if (i) writel(EIP197_xDR_PROC_xD_PKT(i) | EIP197_xDR_PROC_xD_COUNT(tot_descs * priv->config.rd_offset), EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT); - } /* If the number of requests overflowed the counter, try to proceed more * requests. diff --git a/drivers/crypto/inside-secure/safexcel.h b/drivers/crypto/inside-secure/safexcel.h index 65624a81f0fd..e0c202f33674 100644 --- a/drivers/crypto/inside-secure/safexcel.h +++ b/drivers/crypto/inside-secure/safexcel.h @@ -118,6 +118,7 @@ #define EIP197_PE_ICE_SCRATCH_CTRL(n) (0x0d04 + (0x2000 * (n))) #define EIP197_PE_ICE_FPP_CTRL(n) (0x0d80 + (0x2000 * (n))) #define EIP197_PE_ICE_RAM_CTRL(n) (0x0ff0 + (0x2000 * (n))) +#define EIP197_PE_EIP96_TOKEN_CTRL(n) (0x1000 + (0x2000 * (n))) #define EIP197_PE_EIP96_FUNCTION_EN(n) (0x1004 + (0x2000 * (n))) #define EIP197_PE_EIP96_CONTEXT_CTRL(n) (0x1008 + (0x2000 * (n))) #define EIP197_PE_EIP96_CONTEXT_STAT(n) (0x100c + (0x2000 * (n))) @@ -249,6 +250,11 @@ #define EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN BIT(0) #define EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN BIT(1) +/* EIP197_PE_EIP96_TOKEN_CTRL */ +#define EIP197_PE_EIP96_TOKEN_CTRL_CTX_UPDATES BIT(16) +#define EIP197_PE_EIP96_TOKEN_CTRL_REUSE_CTX BIT(19) +#define EIP197_PE_EIP96_TOKEN_CTRL_POST_REUSE_CTX BIT(20) + /* EIP197_PE_EIP96_FUNCTION_EN */ #define EIP197_FUNCTION_RSVD (BIT(6) | BIT(15) | BIT(20) | BIT(23)) #define EIP197_PROTOCOL_HASH_ONLY BIT(0) @@ -333,6 +339,7 @@ struct safexcel_context_record { #define CONTEXT_CONTROL_IV3 BIT(8) #define CONTEXT_CONTROL_DIGEST_CNT BIT(9) #define CONTEXT_CONTROL_COUNTER_MODE BIT(10) +#define CONTEXT_CONTROL_CRYPTO_STORE BIT(12) #define CONTEXT_CONTROL_HASH_STORE BIT(19) /* The hash counter given to the engine in the context has a granularity of @@ -425,6 +432,10 @@ struct safexcel_token { #define EIP197_TOKEN_HASH_RESULT_VERIFY BIT(16) +#define EIP197_TOKEN_CTX_OFFSET(x) (x) +#define EIP197_TOKEN_DIRECTION_EXTERNAL BIT(11) +#define EIP197_TOKEN_EXEC_IF_SUCCESSFUL (0x1 << 12) + #define EIP197_TOKEN_STAT_LAST_HASH BIT(0) #define EIP197_TOKEN_STAT_LAST_PACKET BIT(1) #define EIP197_TOKEN_OPCODE_DIRECTION 0x0 @@ -432,6 +443,7 @@ struct safexcel_token { #define EIP197_TOKEN_OPCODE_NOOP EIP197_TOKEN_OPCODE_INSERT #define EIP197_TOKEN_OPCODE_RETRIEVE 0x4 #define EIP197_TOKEN_OPCODE_VERIFY 0xd +#define EIP197_TOKEN_OPCODE_CTX_ACCESS 0xe #define EIP197_TOKEN_OPCODE_BYPASS GENMASK(3, 0) static inline void eip197_noop_token(struct safexcel_token *token) @@ -442,6 +454,8 @@ static inline void eip197_noop_token(struct safexcel_token *token) /* Instructions */ #define EIP197_TOKEN_INS_INSERT_HASH_DIGEST 0x1c +#define EIP197_TOKEN_INS_ORIGIN_IV0 0x14 +#define EIP197_TOKEN_INS_ORIGIN_LEN(x) ((x) << 5) #define EIP197_TOKEN_INS_TYPE_OUTPUT BIT(5) #define EIP197_TOKEN_INS_TYPE_HASH BIT(6) #define EIP197_TOKEN_INS_TYPE_CRYTO BIT(7) @@ -468,6 +482,7 @@ struct safexcel_control_data_desc { #define EIP197_OPTION_MAGIC_VALUE BIT(0) #define EIP197_OPTION_64BIT_CTX BIT(1) +#define EIP197_OPTION_RC_AUTO (0x2 << 3) #define EIP197_OPTION_CTX_CTRL_IN_CMD BIT(8) #define EIP197_OPTION_2_TOKEN_IV_CMD GENMASK(11, 10) #define EIP197_OPTION_4_TOKEN_IV_CMD GENMASK(11, 9) @@ -629,7 +644,7 @@ struct safexcel_ahash_export_state { u32 digest; u32 state[SHA512_DIGEST_SIZE / sizeof(u32)]; - u8 cache[SHA512_BLOCK_SIZE]; + u8 cache[SHA512_BLOCK_SIZE << 1]; }; /* diff --git a/drivers/crypto/inside-secure/safexcel_cipher.c b/drivers/crypto/inside-secure/safexcel_cipher.c index de4be10b172f..8cdbdbe35681 100644 --- a/drivers/crypto/inside-secure/safexcel_cipher.c +++ b/drivers/crypto/inside-secure/safexcel_cipher.c @@ -51,6 +51,8 @@ struct safexcel_cipher_ctx { struct safexcel_cipher_req { enum safexcel_cipher_direction direction; + /* Number of result descriptors associated to the request */ + unsigned int rdescs; bool needs_inv; }; @@ -59,27 +61,26 @@ static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, u32 length) { struct safexcel_token *token; - unsigned offset = 0; + u32 offset = 0, block_sz = 0; if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) { switch (ctx->alg) { case SAFEXCEL_DES: - offset = DES_BLOCK_SIZE / sizeof(u32); - memcpy(cdesc->control_data.token, iv, DES_BLOCK_SIZE); + block_sz = DES_BLOCK_SIZE; cdesc->control_data.options |= EIP197_OPTION_2_TOKEN_IV_CMD; break; case SAFEXCEL_3DES: - offset = DES3_EDE_BLOCK_SIZE / sizeof(u32); - memcpy(cdesc->control_data.token, iv, DES3_EDE_BLOCK_SIZE); + block_sz = DES3_EDE_BLOCK_SIZE; cdesc->control_data.options |= EIP197_OPTION_2_TOKEN_IV_CMD; break; - case SAFEXCEL_AES: - offset = AES_BLOCK_SIZE / sizeof(u32); - memcpy(cdesc->control_data.token, iv, AES_BLOCK_SIZE); + block_sz = AES_BLOCK_SIZE; cdesc->control_data.options |= EIP197_OPTION_4_TOKEN_IV_CMD; break; } + + offset = block_sz / sizeof(u32); + memcpy(cdesc->control_data.token, iv, block_sz); } token = (struct safexcel_token *)(cdesc->control_data.token + offset); @@ -91,6 +92,25 @@ static void safexcel_skcipher_token(struct safexcel_cipher_ctx *ctx, u8 *iv, token[0].instructions = EIP197_TOKEN_INS_LAST | EIP197_TOKEN_INS_TYPE_CRYTO | EIP197_TOKEN_INS_TYPE_OUTPUT; + + if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) { + u32 last = (EIP197_MAX_TOKENS - 1) - offset; + + token[last].opcode = EIP197_TOKEN_OPCODE_CTX_ACCESS; + token[last].packet_length = EIP197_TOKEN_DIRECTION_EXTERNAL | + EIP197_TOKEN_EXEC_IF_SUCCESSFUL| + EIP197_TOKEN_CTX_OFFSET(0x2); + token[last].stat = EIP197_TOKEN_STAT_LAST_HASH | + EIP197_TOKEN_STAT_LAST_PACKET; + token[last].instructions = + EIP197_TOKEN_INS_ORIGIN_LEN(block_sz / sizeof(u32)) | + EIP197_TOKEN_INS_ORIGIN_IV0; + + /* Store the updated IV values back in the internal context + * registers. + */ + cdesc->control_data.control1 |= CONTEXT_CONTROL_CRYPTO_STORE; + } } static void safexcel_aead_token(struct safexcel_cipher_ctx *ctx, u8 *iv, @@ -333,7 +353,10 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin *ret = 0; - do { + if (unlikely(!sreq->rdescs)) + return 0; + + while (sreq->rdescs--) { rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); if (IS_ERR(rdesc)) { dev_err(priv->dev, @@ -346,21 +369,15 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin *ret = safexcel_rdesc_check_errors(priv, rdesc); ndesc++; - } while (!rdesc->last_seg); + } safexcel_complete(priv, ring); if (src == dst) { - dma_unmap_sg(priv->dev, src, - sg_nents_for_len(src, cryptlen), - DMA_BIDIRECTIONAL); + dma_unmap_sg(priv->dev, src, sg_nents(src), DMA_BIDIRECTIONAL); } else { - dma_unmap_sg(priv->dev, src, - sg_nents_for_len(src, cryptlen), - DMA_TO_DEVICE); - dma_unmap_sg(priv->dev, dst, - sg_nents_for_len(dst, cryptlen), - DMA_FROM_DEVICE); + dma_unmap_sg(priv->dev, src, sg_nents(src), DMA_TO_DEVICE); + dma_unmap_sg(priv->dev, dst, sg_nents(dst), DMA_FROM_DEVICE); } *should_complete = true; @@ -385,26 +402,21 @@ static int safexcel_send_req(struct crypto_async_request *base, int ring, int i, ret = 0; if (src == dst) { - nr_src = dma_map_sg(priv->dev, src, - sg_nents_for_len(src, totlen), + nr_src = dma_map_sg(priv->dev, src, sg_nents(src), DMA_BIDIRECTIONAL); nr_dst = nr_src; if (!nr_src) return -EINVAL; } else { - nr_src = dma_map_sg(priv->dev, src, - sg_nents_for_len(src, totlen), + nr_src = dma_map_sg(priv->dev, src, sg_nents(src), DMA_TO_DEVICE); if (!nr_src) return -EINVAL; - nr_dst = dma_map_sg(priv->dev, dst, - sg_nents_for_len(dst, totlen), + nr_dst = dma_map_sg(priv->dev, dst, sg_nents(dst), DMA_FROM_DEVICE); if (!nr_dst) { - dma_unmap_sg(priv->dev, src, - sg_nents_for_len(src, totlen), - DMA_TO_DEVICE); + dma_unmap_sg(priv->dev, src, nr_src, DMA_TO_DEVICE); return -EINVAL; } } @@ -454,7 +466,7 @@ static int safexcel_send_req(struct crypto_async_request *base, int ring, /* result descriptors */ for_each_sg(dst, sg, nr_dst, i) { - bool first = !i, last = (i == nr_dst - 1); + bool first = !i, last = sg_is_last(sg); u32 len = sg_dma_len(sg); rdesc = safexcel_add_rdesc(priv, ring, first, last, @@ -483,16 +495,10 @@ cdesc_rollback: safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr); if (src == dst) { - dma_unmap_sg(priv->dev, src, - sg_nents_for_len(src, totlen), - DMA_BIDIRECTIONAL); + dma_unmap_sg(priv->dev, src, nr_src, DMA_BIDIRECTIONAL); } else { - dma_unmap_sg(priv->dev, src, - sg_nents_for_len(src, totlen), - DMA_TO_DEVICE); - dma_unmap_sg(priv->dev, dst, - sg_nents_for_len(dst, totlen), - DMA_FROM_DEVICE); + dma_unmap_sg(priv->dev, src, nr_src, DMA_TO_DEVICE); + dma_unmap_sg(priv->dev, dst, nr_dst, DMA_FROM_DEVICE); } return ret; @@ -501,6 +507,7 @@ cdesc_rollback: static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, int ring, struct crypto_async_request *base, + struct safexcel_cipher_req *sreq, bool *should_complete, int *ret) { struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(base->tfm); @@ -509,7 +516,10 @@ static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, *ret = 0; - do { + if (unlikely(!sreq->rdescs)) + return 0; + + while (sreq->rdescs--) { rdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].rdr); if (IS_ERR(rdesc)) { dev_err(priv->dev, @@ -522,7 +532,7 @@ static int safexcel_handle_inv_result(struct safexcel_crypto_priv *priv, *ret = safexcel_rdesc_check_errors(priv, rdesc); ndesc++; - } while (!rdesc->last_seg); + } safexcel_complete(priv, ring); @@ -560,16 +570,35 @@ static int safexcel_skcipher_handle_result(struct safexcel_crypto_priv *priv, { struct skcipher_request *req = skcipher_request_cast(async); struct safexcel_cipher_req *sreq = skcipher_request_ctx(req); + struct safexcel_cipher_ctx *ctx = crypto_tfm_ctx(async->tfm); int err; if (sreq->needs_inv) { sreq->needs_inv = false; - err = safexcel_handle_inv_result(priv, ring, async, + err = safexcel_handle_inv_result(priv, ring, async, sreq, should_complete, ret); } else { err = safexcel_handle_req_result(priv, ring, async, req->src, req->dst, req->cryptlen, sreq, should_complete, ret); + + if (ctx->mode == CONTEXT_CONTROL_CRYPTO_MODE_CBC) { + u32 block_sz = 0; + + switch (ctx->alg) { + case SAFEXCEL_DES: + block_sz = DES_BLOCK_SIZE; + break; + case SAFEXCEL_3DES: + block_sz = DES3_EDE_BLOCK_SIZE; + break; + case SAFEXCEL_AES: + block_sz = AES_BLOCK_SIZE; + break; + } + + memcpy(req->iv, ctx->base.ctxr->data, block_sz); + } } return err; @@ -587,7 +616,7 @@ static int safexcel_aead_handle_result(struct safexcel_crypto_priv *priv, if (sreq->needs_inv) { sreq->needs_inv = false; - err = safexcel_handle_inv_result(priv, ring, async, + err = safexcel_handle_inv_result(priv, ring, async, sreq, should_complete, ret); } else { err = safexcel_handle_req_result(priv, ring, async, req->src, @@ -633,6 +662,8 @@ static int safexcel_skcipher_send(struct crypto_async_request *async, int ring, ret = safexcel_send_req(async, ring, sreq, req->src, req->dst, req->cryptlen, 0, 0, req->iv, commands, results); + + sreq->rdescs = *results; return ret; } @@ -655,6 +686,7 @@ static int safexcel_aead_send(struct crypto_async_request *async, int ring, req->cryptlen, req->assoclen, crypto_aead_authsize(tfm), req->iv, commands, results); + sreq->rdescs = *results; return ret; } diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index ac9282c1a5ec..a80a5e757b1f 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -41,19 +41,21 @@ struct safexcel_ahash_req { u64 len[2]; u64 processed[2]; - u8 cache[SHA512_BLOCK_SIZE] __aligned(sizeof(u32)); + u8 cache[SHA512_BLOCK_SIZE << 1] __aligned(sizeof(u32)); dma_addr_t cache_dma; unsigned int cache_sz; - u8 cache_next[SHA512_BLOCK_SIZE] __aligned(sizeof(u32)); + u8 cache_next[SHA512_BLOCK_SIZE << 1] __aligned(sizeof(u32)); }; static inline u64 safexcel_queued_len(struct safexcel_ahash_req *req) { - if (req->len[1] > req->processed[1]) - return 0xffffffff - (req->len[0] - req->processed[0]); + u64 len, processed; - return req->len[0] - req->processed[0]; + len = (0xffffffff * req->len[1]) + req->len[0]; + processed = (0xffffffff * req->processed[1]) + req->processed[0]; + + return len - processed; } static void safexcel_hash_token(struct safexcel_command_desc *cdesc, @@ -87,6 +89,9 @@ static void safexcel_context_control(struct safexcel_ahash_ctx *ctx, cdesc->control_data.control0 |= ctx->alg; cdesc->control_data.control0 |= req->digest; + if (!req->finish) + cdesc->control_data.control0 |= CONTEXT_CONTROL_NO_FINISH_HASH; + if (req->digest == CONTEXT_CONTROL_DIGEST_PRECOMPUTED) { if (req->processed[0] || req->processed[1]) { if (ctx->alg == CONTEXT_CONTROL_CRYPTO_ALG_MD5) @@ -105,9 +110,6 @@ static void safexcel_context_control(struct safexcel_ahash_ctx *ctx, cdesc->control_data.control0 |= CONTEXT_CONTROL_RESTART_HASH; } - if (!req->finish) - cdesc->control_data.control0 |= CONTEXT_CONTROL_NO_FINISH_HASH; - /* * Copy the input digest if needed, and setup the context * fields. Do this now as we need it to setup the first command @@ -183,6 +185,7 @@ static int safexcel_handle_req_result(struct safexcel_crypto_priv *priv, int rin dma_unmap_single(priv->dev, sreq->cache_dma, sreq->cache_sz, DMA_TO_DEVICE); sreq->cache_dma = 0; + sreq->cache_sz = 0; } if (sreq->finish) @@ -209,11 +212,15 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, struct safexcel_command_desc *cdesc, *first_cdesc = NULL; struct safexcel_result_desc *rdesc; struct scatterlist *sg; - int i, extra, n_cdesc = 0, ret = 0; - u64 queued, len, cache_len; + int i, extra = 0, n_cdesc = 0, ret = 0; + u64 queued, len, cache_len, cache_max; + + cache_max = crypto_ahash_blocksize(ahash); + if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC) + cache_max <<= 1; queued = len = safexcel_queued_len(req); - if (queued <= crypto_ahash_blocksize(ahash)) + if (queued <= cache_max) cache_len = queued; else cache_len = queued - areq->nbytes; @@ -223,26 +230,23 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, * fit into full blocks, cache it for the next send() call. */ extra = queued & (crypto_ahash_blocksize(ahash) - 1); + + if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC && + extra < crypto_ahash_blocksize(ahash)) + extra += crypto_ahash_blocksize(ahash); + + /* If this is not the last request and the queued data + * is a multiple of a block, cache the last one for now. + */ if (!extra) - /* If this is not the last request and the queued data - * is a multiple of a block, cache the last one for now. - */ extra = crypto_ahash_blocksize(ahash); - if (extra) { - sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), - req->cache_next, extra, - areq->nbytes - extra); - - queued -= extra; - len -= extra; + sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), + req->cache_next, extra, + areq->nbytes - extra); - if (!queued) { - *commands = 0; - *results = 0; - return 0; - } - } + queued -= extra; + len -= extra; } /* Add a command descriptor for the cached data, if any */ @@ -269,8 +273,7 @@ static int safexcel_ahash_send_req(struct crypto_async_request *async, int ring, } /* Now handle the current ahash request buffer(s) */ - req->nents = dma_map_sg(priv->dev, areq->src, - sg_nents_for_len(areq->src, areq->nbytes), + req->nents = dma_map_sg(priv->dev, areq->src, sg_nents(areq->src), DMA_TO_DEVICE); if (!req->nents) { ret = -ENOMEM; @@ -345,6 +348,7 @@ unmap_cache: if (req->cache_dma) { dma_unmap_single(priv->dev, req->cache_dma, req->cache_sz, DMA_TO_DEVICE); + req->cache_dma = 0; req->cache_sz = 0; } @@ -486,7 +490,7 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm) struct safexcel_inv_result result = {}; int ring = ctx->base.ring; - memset(req, 0, sizeof(struct ahash_request)); + memset(req, 0, EIP197_AHASH_REQ_SIZE); /* create invalidation request */ init_completion(&result.completion); @@ -519,10 +523,9 @@ static int safexcel_ahash_exit_inv(struct crypto_tfm *tfm) /* safexcel_ahash_cache: cache data until at least one request can be sent to * the engine, aka. when there is at least 1 block size in the pipe. */ -static int safexcel_ahash_cache(struct ahash_request *areq) +static int safexcel_ahash_cache(struct ahash_request *areq, u32 cache_max) { struct safexcel_ahash_req *req = ahash_request_ctx(areq); - struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); u64 queued, cache_len; /* queued: everything accepted by the driver which will be handled by @@ -539,7 +542,7 @@ static int safexcel_ahash_cache(struct ahash_request *areq) * In case there isn't enough bytes to proceed (less than a * block size), cache the data until we have enough. */ - if (cache_len + areq->nbytes <= crypto_ahash_blocksize(ahash)) { + if (cache_len + areq->nbytes <= cache_max) { sg_pcopy_to_buffer(areq->src, sg_nents(areq->src), req->cache + cache_len, areq->nbytes, 0); @@ -599,6 +602,7 @@ static int safexcel_ahash_update(struct ahash_request *areq) { struct safexcel_ahash_req *req = ahash_request_ctx(areq); struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); + u32 cache_max; /* If the request is 0 length, do nothing */ if (!areq->nbytes) @@ -608,7 +612,11 @@ static int safexcel_ahash_update(struct ahash_request *areq) if (req->len[0] < areq->nbytes) req->len[1]++; - safexcel_ahash_cache(areq); + cache_max = crypto_ahash_blocksize(ahash); + if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC) + cache_max <<= 1; + + safexcel_ahash_cache(areq, cache_max); /* * We're not doing partial updates when performing an hmac request. @@ -621,7 +629,7 @@ static int safexcel_ahash_update(struct ahash_request *areq) return safexcel_ahash_enqueue(areq); if (!req->last_req && - safexcel_queued_len(req) > crypto_ahash_blocksize(ahash)) + safexcel_queued_len(req) > cache_max) return safexcel_ahash_enqueue(areq); return 0; @@ -678,6 +686,11 @@ static int safexcel_ahash_export(struct ahash_request *areq, void *out) struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); struct safexcel_ahash_req *req = ahash_request_ctx(areq); struct safexcel_ahash_export_state *export = out; + u32 cache_sz; + + cache_sz = crypto_ahash_blocksize(ahash); + if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC) + cache_sz <<= 1; export->len[0] = req->len[0]; export->len[1] = req->len[1]; @@ -687,7 +700,7 @@ static int safexcel_ahash_export(struct ahash_request *areq, void *out) export->digest = req->digest; memcpy(export->state, req->state, req->state_sz); - memcpy(export->cache, req->cache, crypto_ahash_blocksize(ahash)); + memcpy(export->cache, req->cache, cache_sz); return 0; } @@ -697,12 +710,17 @@ static int safexcel_ahash_import(struct ahash_request *areq, const void *in) struct crypto_ahash *ahash = crypto_ahash_reqtfm(areq); struct safexcel_ahash_req *req = ahash_request_ctx(areq); const struct safexcel_ahash_export_state *export = in; + u32 cache_sz; int ret; ret = crypto_ahash_init(areq); if (ret) return ret; + cache_sz = crypto_ahash_blocksize(ahash); + if (req->digest == CONTEXT_CONTROL_DIGEST_HMAC) + cache_sz <<= 1; + req->len[0] = export->len[0]; req->len[1] = export->len[1]; req->processed[0] = export->processed[0]; @@ -710,7 +728,7 @@ static int safexcel_ahash_import(struct ahash_request *areq, const void *in) req->digest = export->digest; - memcpy(req->cache, export->cache, crypto_ahash_blocksize(ahash)); + memcpy(req->cache, export->cache, cache_sz); memcpy(req->state, export->state, req->state_sz); return 0; diff --git a/drivers/crypto/inside-secure/safexcel_ring.c b/drivers/crypto/inside-secure/safexcel_ring.c index eb75fa684876..142bc3f5c45c 100644 --- a/drivers/crypto/inside-secure/safexcel_ring.c +++ b/drivers/crypto/inside-secure/safexcel_ring.c @@ -145,6 +145,9 @@ struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *pr (lower_32_bits(context) & GENMASK(31, 2)) >> 2; cdesc->control_data.context_hi = upper_32_bits(context); + if (priv->version == EIP197B || priv->version == EIP197D) + cdesc->control_data.options |= EIP197_OPTION_RC_AUTO; + /* TODO: large xform HMAC with SHA-384/512 uses refresh = 3 */ cdesc->control_data.refresh = 2; diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c index e5cf3a59c420..acedafe3fa98 100644 --- a/drivers/crypto/ixp4xx_crypto.c +++ b/drivers/crypto/ixp4xx_crypto.c @@ -100,7 +100,7 @@ struct buffer_desc { u16 pkt_len; u16 buf_len; #endif - u32 phys_addr; + dma_addr_t phys_addr; u32 __reserved[4]; struct buffer_desc *next; enum dma_data_direction dir; @@ -117,9 +117,9 @@ struct crypt_ctl { u8 mode; /* NPE_OP_* operation mode */ #endif u8 iv[MAX_IVLEN]; /* IV for CBC mode or CTR IV for CTR mode */ - u32 icv_rev_aes; /* icv or rev aes */ - u32 src_buf; - u32 dst_buf; + dma_addr_t icv_rev_aes; /* icv or rev aes */ + dma_addr_t src_buf; + dma_addr_t dst_buf; #ifdef __ARMEB__ u16 auth_offs; /* Authentication start offset */ u16 auth_len; /* Authentication data length */ @@ -320,7 +320,8 @@ static struct crypt_ctl *get_crypt_desc_emerg(void) } } -static void free_buf_chain(struct device *dev, struct buffer_desc *buf,u32 phys) +static void free_buf_chain(struct device *dev, struct buffer_desc *buf, + dma_addr_t phys) { while (buf) { struct buffer_desc *buf1; @@ -602,7 +603,7 @@ static int register_chain_var(struct crypto_tfm *tfm, u8 xpad, u32 target, struct buffer_desc *buf; int i; u8 *pad; - u32 pad_phys, buf_phys; + dma_addr_t pad_phys, buf_phys; BUILD_BUG_ON(NPE_CTX_LEN < HMAC_PAD_BLOCKLEN); pad = dma_pool_alloc(ctx_pool, GFP_KERNEL, &pad_phys); @@ -787,7 +788,7 @@ static struct buffer_desc *chainup_buffers(struct device *dev, for (; nbytes > 0; sg = sg_next(sg)) { unsigned len = min(nbytes, sg->length); struct buffer_desc *next_buf; - u32 next_buf_phys; + dma_addr_t next_buf_phys; void *ptr; nbytes -= len; diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c index bdc4c42d3ac8..f1fa637cb029 100644 --- a/drivers/crypto/mxs-dcp.c +++ b/drivers/crypto/mxs-dcp.c @@ -986,8 +986,6 @@ static int mxs_dcp_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct dcp *sdcp = NULL; int i, ret; - - struct resource *iores; int dcp_vmi_irq, dcp_irq; if (global_sdcp) { @@ -995,7 +993,6 @@ static int mxs_dcp_probe(struct platform_device *pdev) return -ENODEV; } - iores = platform_get_resource(pdev, IORESOURCE_MEM, 0); dcp_vmi_irq = platform_get_irq(pdev, 0); if (dcp_vmi_irq < 0) { dev_err(dev, "Failed to get IRQ: (%d)!\n", dcp_vmi_irq); @@ -1013,7 +1010,7 @@ static int mxs_dcp_probe(struct platform_device *pdev) return -ENOMEM; sdcp->dev = dev; - sdcp->base = devm_ioremap_resource(dev, iores); + sdcp->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(sdcp->base)) return PTR_ERR(sdcp->base); diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c index 4acbc47973e9..e78ff5c65ed6 100644 --- a/drivers/crypto/nx/nx-842-powernv.c +++ b/drivers/crypto/nx/nx-842-powernv.c @@ -27,8 +27,6 @@ MODULE_ALIAS_CRYPTO("842-nx"); #define WORKMEM_ALIGN (CRB_ALIGN) #define CSB_WAIT_MAX (5000) /* ms */ #define VAS_RETRIES (10) -/* # of requests allowed per RxFIFO at a time. 0 for unlimited */ -#define MAX_CREDITS_PER_RXFIFO (1024) struct nx842_workmem { /* Below fields must be properly aligned */ @@ -812,7 +810,11 @@ static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id, rxattr.lnotify_lpid = lpid; rxattr.lnotify_pid = pid; rxattr.lnotify_tid = tid; - rxattr.wcreds_max = MAX_CREDITS_PER_RXFIFO; + /* + * Maximum RX window credits can not be more than #CRBs in + * RxFIFO. Otherwise, can get checkstop if RxFIFO overruns. + */ + rxattr.wcreds_max = fifo_size / CRB_SIZE; /* * Open a VAS receice window which is used to configure RxFIFO diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c index 428c273a1ab6..28817880c76d 100644 --- a/drivers/crypto/nx/nx.c +++ b/drivers/crypto/nx/nx.c @@ -569,9 +569,7 @@ static int nx_register_algs(void) memset(&nx_driver.stats, 0, sizeof(struct nx_stats)); - rc = NX_DEBUGFS_INIT(&nx_driver); - if (rc) - goto out; + NX_DEBUGFS_INIT(&nx_driver); nx_driver.of.status = NX_OKAY; diff --git a/drivers/crypto/nx/nx.h b/drivers/crypto/nx/nx.h index c3e54af18645..c6b5a3be02be 100644 --- a/drivers/crypto/nx/nx.h +++ b/drivers/crypto/nx/nx.h @@ -76,20 +76,12 @@ struct nx_stats { atomic_t last_error_pid; }; -struct nx_debugfs { - struct dentry *dfs_root; - struct dentry *dfs_aes_ops, *dfs_aes_bytes; - struct dentry *dfs_sha256_ops, *dfs_sha256_bytes; - struct dentry *dfs_sha512_ops, *dfs_sha512_bytes; - struct dentry *dfs_errors, *dfs_last_error, *dfs_last_error_pid; -}; - struct nx_crypto_driver { struct nx_stats stats; struct nx_of of; struct vio_dev *viodev; struct vio_driver viodriver; - struct nx_debugfs dfs; + struct dentry *dfs_root; }; #define NX_GCM4106_NONCE_LEN (4) @@ -177,7 +169,7 @@ struct nx_sg *nx_walk_and_build(struct nx_sg *, unsigned int, #define NX_DEBUGFS_INIT(drv) nx_debugfs_init(drv) #define NX_DEBUGFS_FINI(drv) nx_debugfs_fini(drv) -int nx_debugfs_init(struct nx_crypto_driver *); +void nx_debugfs_init(struct nx_crypto_driver *); void nx_debugfs_fini(struct nx_crypto_driver *); #else #define NX_DEBUGFS_INIT(drv) (0) diff --git a/drivers/crypto/nx/nx_debugfs.c b/drivers/crypto/nx/nx_debugfs.c index 03e4f0363c6a..e0d44a5512ab 100644 --- a/drivers/crypto/nx/nx_debugfs.c +++ b/drivers/crypto/nx/nx_debugfs.c @@ -30,62 +30,37 @@ * Documentation/ABI/testing/debugfs-pfo-nx-crypto */ -int nx_debugfs_init(struct nx_crypto_driver *drv) +void nx_debugfs_init(struct nx_crypto_driver *drv) { - struct nx_debugfs *dfs = &drv->dfs; + struct dentry *root; - dfs->dfs_root = debugfs_create_dir(NX_NAME, NULL); + root = debugfs_create_dir(NX_NAME, NULL); + drv->dfs_root = root; - dfs->dfs_aes_ops = - debugfs_create_u32("aes_ops", - S_IRUSR | S_IRGRP | S_IROTH, - dfs->dfs_root, (u32 *)&drv->stats.aes_ops); - dfs->dfs_sha256_ops = - debugfs_create_u32("sha256_ops", - S_IRUSR | S_IRGRP | S_IROTH, - dfs->dfs_root, - (u32 *)&drv->stats.sha256_ops); - dfs->dfs_sha512_ops = - debugfs_create_u32("sha512_ops", - S_IRUSR | S_IRGRP | S_IROTH, - dfs->dfs_root, - (u32 *)&drv->stats.sha512_ops); - dfs->dfs_aes_bytes = - debugfs_create_u64("aes_bytes", - S_IRUSR | S_IRGRP | S_IROTH, - dfs->dfs_root, - (u64 *)&drv->stats.aes_bytes); - dfs->dfs_sha256_bytes = - debugfs_create_u64("sha256_bytes", - S_IRUSR | S_IRGRP | S_IROTH, - dfs->dfs_root, - (u64 *)&drv->stats.sha256_bytes); - dfs->dfs_sha512_bytes = - debugfs_create_u64("sha512_bytes", - S_IRUSR | S_IRGRP | S_IROTH, - dfs->dfs_root, - (u64 *)&drv->stats.sha512_bytes); - dfs->dfs_errors = - debugfs_create_u32("errors", - S_IRUSR | S_IRGRP | S_IROTH, - dfs->dfs_root, (u32 *)&drv->stats.errors); - dfs->dfs_last_error = - debugfs_create_u32("last_error", - S_IRUSR | S_IRGRP | S_IROTH, - dfs->dfs_root, - (u32 *)&drv->stats.last_error); - dfs->dfs_last_error_pid = - debugfs_create_u32("last_error_pid", - S_IRUSR | S_IRGRP | S_IROTH, - dfs->dfs_root, - (u32 *)&drv->stats.last_error_pid); - return 0; + debugfs_create_u32("aes_ops", S_IRUSR | S_IRGRP | S_IROTH, + root, (u32 *)&drv->stats.aes_ops); + debugfs_create_u32("sha256_ops", S_IRUSR | S_IRGRP | S_IROTH, + root, (u32 *)&drv->stats.sha256_ops); + debugfs_create_u32("sha512_ops", S_IRUSR | S_IRGRP | S_IROTH, + root, (u32 *)&drv->stats.sha512_ops); + debugfs_create_u64("aes_bytes", S_IRUSR | S_IRGRP | S_IROTH, + root, (u64 *)&drv->stats.aes_bytes); + debugfs_create_u64("sha256_bytes", S_IRUSR | S_IRGRP | S_IROTH, + root, (u64 *)&drv->stats.sha256_bytes); + debugfs_create_u64("sha512_bytes", S_IRUSR | S_IRGRP | S_IROTH, + root, (u64 *)&drv->stats.sha512_bytes); + debugfs_create_u32("errors", S_IRUSR | S_IRGRP | S_IROTH, + root, (u32 *)&drv->stats.errors); + debugfs_create_u32("last_error", S_IRUSR | S_IRGRP | S_IROTH, + root, (u32 *)&drv->stats.last_error); + debugfs_create_u32("last_error_pid", S_IRUSR | S_IRGRP | S_IROTH, + root, (u32 *)&drv->stats.last_error_pid); } void nx_debugfs_fini(struct nx_crypto_driver *drv) { - debugfs_remove_recursive(drv->dfs.dfs_root); + debugfs_remove_recursive(drv->dfs_root); } #endif diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index c8d401646902..b50eb55f8f57 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -131,7 +131,6 @@ struct qat_alg_ablkcipher_ctx { struct icp_qat_fw_la_bulk_req dec_fw_req; struct qat_crypto_instance *inst; struct crypto_tfm *tfm; - spinlock_t lock; /* protects qat_alg_ablkcipher_ctx struct */ }; static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg) @@ -223,6 +222,9 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash, return -EFAULT; offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8); + if (offset < 0) + return -EFAULT; + hash_state_out = (__be32 *)(hash->sha.state1 + offset); hash512_state_out = (__be64 *)hash_state_out; @@ -253,7 +255,24 @@ static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash, return 0; } -static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header) +static void qat_alg_init_hdr_iv_updt(struct icp_qat_fw_comn_req_hdr *header) +{ + ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, + ICP_QAT_FW_CIPH_IV_64BIT_PTR); + ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_UPDATE_STATE); +} + +static void qat_alg_init_hdr_no_iv_updt(struct icp_qat_fw_comn_req_hdr *header) +{ + ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, + ICP_QAT_FW_CIPH_IV_16BYTE_DATA); + ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_NO_UPDATE_STATE); +} + +static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header, + int aead) { header->hdr_flags = ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET); @@ -263,12 +282,12 @@ static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header) QAT_COMN_PTR_TYPE_SGL); ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags, ICP_QAT_FW_LA_PARTIAL_NONE); - ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, - ICP_QAT_FW_CIPH_IV_16BYTE_DATA); + if (aead) + qat_alg_init_hdr_no_iv_updt(header); + else + qat_alg_init_hdr_iv_updt(header); ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags, ICP_QAT_FW_LA_NO_PROTO); - ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_NO_UPDATE_STATE); } static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm, @@ -303,7 +322,7 @@ static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm, return -EFAULT; /* Request setup */ - qat_alg_init_common_hdr(header); + qat_alg_init_common_hdr(header, 1); header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH; ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, ICP_QAT_FW_LA_DIGEST_IN_BUFFER); @@ -390,7 +409,7 @@ static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm, return -EFAULT; /* Request setup */ - qat_alg_init_common_hdr(header); + qat_alg_init_common_hdr(header, 1); header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER; ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, ICP_QAT_FW_LA_DIGEST_IN_BUFFER); @@ -454,7 +473,7 @@ static void qat_alg_ablkcipher_init_com(struct qat_alg_ablkcipher_ctx *ctx, struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl; memcpy(cd->aes.key, key, keylen); - qat_alg_init_common_hdr(header); + qat_alg_init_common_hdr(header, 0); header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER; cd_pars->u.s.content_desc_params_sz = sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3; @@ -576,45 +595,52 @@ bad_key: return -EINVAL; } -static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key, +static int qat_alg_aead_rekey(struct crypto_aead *tfm, const uint8_t *key, + unsigned int keylen) +{ + struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); + + memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd)); + memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd)); + memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req)); + memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req)); + + return qat_alg_aead_init_sessions(tfm, key, keylen, + ICP_QAT_HW_CIPHER_CBC_MODE); +} + +static int qat_alg_aead_newkey(struct crypto_aead *tfm, const uint8_t *key, unsigned int keylen) { struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); + struct qat_crypto_instance *inst = NULL; + int node = get_current_node(); struct device *dev; + int ret; - if (ctx->enc_cd) { - /* rekeying */ - dev = &GET_DEV(ctx->inst->accel_dev); - memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd)); - memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd)); - memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req)); - memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req)); - } else { - /* new key */ - int node = get_current_node(); - struct qat_crypto_instance *inst = - qat_crypto_get_instance_node(node); - if (!inst) { - return -EINVAL; - } - - dev = &GET_DEV(inst->accel_dev); - ctx->inst = inst; - ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd), - &ctx->enc_cd_paddr, - GFP_ATOMIC); - if (!ctx->enc_cd) { - return -ENOMEM; - } - ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd), - &ctx->dec_cd_paddr, - GFP_ATOMIC); - if (!ctx->dec_cd) { - goto out_free_enc; - } + inst = qat_crypto_get_instance_node(node); + if (!inst) + return -EINVAL; + dev = &GET_DEV(inst->accel_dev); + ctx->inst = inst; + ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd), + &ctx->enc_cd_paddr, + GFP_ATOMIC); + if (!ctx->enc_cd) { + ret = -ENOMEM; + goto out_free_inst; + } + ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd), + &ctx->dec_cd_paddr, + GFP_ATOMIC); + if (!ctx->dec_cd) { + ret = -ENOMEM; + goto out_free_enc; } - if (qat_alg_aead_init_sessions(tfm, key, keylen, - ICP_QAT_HW_CIPHER_CBC_MODE)) + + ret = qat_alg_aead_init_sessions(tfm, key, keylen, + ICP_QAT_HW_CIPHER_CBC_MODE); + if (ret) goto out_free_all; return 0; @@ -629,7 +655,21 @@ out_free_enc: dma_free_coherent(dev, sizeof(struct qat_alg_cd), ctx->enc_cd, ctx->enc_cd_paddr); ctx->enc_cd = NULL; - return -ENOMEM; +out_free_inst: + ctx->inst = NULL; + qat_crypto_put_instance(inst); + return ret; +} + +static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key, + unsigned int keylen) +{ + struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); + + if (ctx->enc_cd) + return qat_alg_aead_rekey(tfm, key, keylen); + else + return qat_alg_aead_newkey(tfm, key, keylen); } static void qat_alg_free_bufl(struct qat_crypto_instance *inst, @@ -677,8 +717,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, dma_addr_t blp; dma_addr_t bloutp = 0; struct scatterlist *sg; - size_t sz_out, sz = sizeof(struct qat_alg_buf_list) + - ((1 + n) * sizeof(struct qat_alg_buf)); + size_t sz_out, sz = struct_size(bufl, bufers, n + 1); if (unlikely(!n)) return -EINVAL; @@ -715,8 +754,7 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, struct qat_alg_buf *bufers; n = sg_nents(sglout); - sz_out = sizeof(struct qat_alg_buf_list) + - ((1 + n) * sizeof(struct qat_alg_buf)); + sz_out = struct_size(buflout, bufers, n + 1); sg_nctr = 0; buflout = kzalloc_node(sz_out, GFP_ATOMIC, dev_to_node(&GET_DEV(inst->accel_dev))); @@ -801,11 +839,17 @@ static void qat_ablkcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp, struct qat_crypto_instance *inst = ctx->inst; struct ablkcipher_request *areq = qat_req->ablkcipher_req; uint8_t stat_filed = qat_resp->comn_resp.comn_status; + struct device *dev = &GET_DEV(ctx->inst->accel_dev); int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed); qat_alg_free_bufl(inst, qat_req); if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK)) res = -EINVAL; + + memcpy(areq->info, qat_req->iv, AES_BLOCK_SIZE); + dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, + qat_req->iv_paddr); + areq->base.complete(&areq->base, res); } @@ -905,50 +949,49 @@ static int qat_alg_aead_enc(struct aead_request *areq) return -EINPROGRESS; } -static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm, +static int qat_alg_ablkcipher_rekey(struct qat_alg_ablkcipher_ctx *ctx, + const u8 *key, unsigned int keylen, + int mode) +{ + memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd)); + memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd)); + memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req)); + memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req)); + + return qat_alg_ablkcipher_init_sessions(ctx, key, keylen, mode); +} + +static int qat_alg_ablkcipher_newkey(struct qat_alg_ablkcipher_ctx *ctx, const u8 *key, unsigned int keylen, int mode) { - struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct qat_crypto_instance *inst = NULL; struct device *dev; + int node = get_current_node(); + int ret; - spin_lock(&ctx->lock); - if (ctx->enc_cd) { - /* rekeying */ - dev = &GET_DEV(ctx->inst->accel_dev); - memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd)); - memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd)); - memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req)); - memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req)); - } else { - /* new key */ - int node = get_current_node(); - struct qat_crypto_instance *inst = - qat_crypto_get_instance_node(node); - if (!inst) { - spin_unlock(&ctx->lock); - return -EINVAL; - } - - dev = &GET_DEV(inst->accel_dev); - ctx->inst = inst; - ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd), - &ctx->enc_cd_paddr, - GFP_ATOMIC); - if (!ctx->enc_cd) { - spin_unlock(&ctx->lock); - return -ENOMEM; - } - ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd), - &ctx->dec_cd_paddr, - GFP_ATOMIC); - if (!ctx->dec_cd) { - spin_unlock(&ctx->lock); - goto out_free_enc; - } + inst = qat_crypto_get_instance_node(node); + if (!inst) + return -EINVAL; + dev = &GET_DEV(inst->accel_dev); + ctx->inst = inst; + ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd), + &ctx->enc_cd_paddr, + GFP_ATOMIC); + if (!ctx->enc_cd) { + ret = -ENOMEM; + goto out_free_instance; + } + ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd), + &ctx->dec_cd_paddr, + GFP_ATOMIC); + if (!ctx->dec_cd) { + ret = -ENOMEM; + goto out_free_enc; } - spin_unlock(&ctx->lock); - if (qat_alg_ablkcipher_init_sessions(ctx, key, keylen, mode)) + + ret = qat_alg_ablkcipher_init_sessions(ctx, key, keylen, mode); + if (ret) goto out_free_all; return 0; @@ -963,7 +1006,22 @@ out_free_enc: dma_free_coherent(dev, sizeof(*ctx->enc_cd), ctx->enc_cd, ctx->enc_cd_paddr); ctx->enc_cd = NULL; - return -ENOMEM; +out_free_instance: + ctx->inst = NULL; + qat_crypto_put_instance(inst); + return ret; +} + +static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm, + const u8 *key, unsigned int keylen, + int mode) +{ + struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); + + if (ctx->enc_cd) + return qat_alg_ablkcipher_rekey(ctx, key, keylen, mode); + else + return qat_alg_ablkcipher_newkey(ctx, key, keylen, mode); } static int qat_alg_ablkcipher_cbc_setkey(struct crypto_ablkcipher *tfm, @@ -995,11 +1053,23 @@ static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req) struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req); struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_bulk_req *msg; + struct device *dev = &GET_DEV(ctx->inst->accel_dev); int ret, ctr = 0; + if (req->nbytes == 0) + return 0; + + qat_req->iv = dma_alloc_coherent(dev, AES_BLOCK_SIZE, + &qat_req->iv_paddr, GFP_ATOMIC); + if (!qat_req->iv) + return -ENOMEM; + ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req); - if (unlikely(ret)) + if (unlikely(ret)) { + dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, + qat_req->iv_paddr); return ret; + } msg = &qat_req->req; *msg = ctx->enc_fw_req; @@ -1012,18 +1082,29 @@ static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req) cipher_param = (void *)&qat_req->req.serv_specif_rqpars; cipher_param->cipher_length = req->nbytes; cipher_param->cipher_offset = 0; - memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE); + cipher_param->u.s.cipher_IV_ptr = qat_req->iv_paddr; + memcpy(qat_req->iv, req->info, AES_BLOCK_SIZE); do { ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg); } while (ret == -EAGAIN && ctr++ < 10); if (ret == -EAGAIN) { qat_alg_free_bufl(ctx->inst, qat_req); + dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, + qat_req->iv_paddr); return -EBUSY; } return -EINPROGRESS; } +static int qat_alg_ablkcipher_blk_encrypt(struct ablkcipher_request *req) +{ + if (req->nbytes % AES_BLOCK_SIZE != 0) + return -EINVAL; + + return qat_alg_ablkcipher_encrypt(req); +} + static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req) { struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req); @@ -1032,11 +1113,23 @@ static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req) struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req); struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_bulk_req *msg; + struct device *dev = &GET_DEV(ctx->inst->accel_dev); int ret, ctr = 0; + if (req->nbytes == 0) + return 0; + + qat_req->iv = dma_alloc_coherent(dev, AES_BLOCK_SIZE, + &qat_req->iv_paddr, GFP_ATOMIC); + if (!qat_req->iv) + return -ENOMEM; + ret = qat_alg_sgl_to_bufl(ctx->inst, req->src, req->dst, qat_req); - if (unlikely(ret)) + if (unlikely(ret)) { + dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, + qat_req->iv_paddr); return ret; + } msg = &qat_req->req; *msg = ctx->dec_fw_req; @@ -1049,18 +1142,28 @@ static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req) cipher_param = (void *)&qat_req->req.serv_specif_rqpars; cipher_param->cipher_length = req->nbytes; cipher_param->cipher_offset = 0; - memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE); + cipher_param->u.s.cipher_IV_ptr = qat_req->iv_paddr; + memcpy(qat_req->iv, req->info, AES_BLOCK_SIZE); do { ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg); } while (ret == -EAGAIN && ctr++ < 10); if (ret == -EAGAIN) { qat_alg_free_bufl(ctx->inst, qat_req); + dma_free_coherent(dev, AES_BLOCK_SIZE, qat_req->iv, + qat_req->iv_paddr); return -EBUSY; } return -EINPROGRESS; } +static int qat_alg_ablkcipher_blk_decrypt(struct ablkcipher_request *req) +{ + if (req->nbytes % AES_BLOCK_SIZE != 0) + return -EINVAL; + + return qat_alg_ablkcipher_decrypt(req); +} static int qat_alg_aead_init(struct crypto_aead *tfm, enum icp_qat_hw_auth_algo hash, const char *hash_name) @@ -1119,7 +1222,6 @@ static int qat_alg_ablkcipher_init(struct crypto_tfm *tfm) { struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm); - spin_lock_init(&ctx->lock); tfm->crt_ablkcipher.reqsize = sizeof(struct qat_crypto_request); ctx->tfm = tfm; return 0; @@ -1221,8 +1323,8 @@ static struct crypto_alg qat_algs[] = { { .cra_u = { .ablkcipher = { .setkey = qat_alg_ablkcipher_cbc_setkey, - .decrypt = qat_alg_ablkcipher_decrypt, - .encrypt = qat_alg_ablkcipher_encrypt, + .decrypt = qat_alg_ablkcipher_blk_decrypt, + .encrypt = qat_alg_ablkcipher_blk_encrypt, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, @@ -1233,7 +1335,7 @@ static struct crypto_alg qat_algs[] = { { .cra_driver_name = "qat_aes_ctr", .cra_priority = 4001, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, - .cra_blocksize = AES_BLOCK_SIZE, + .cra_blocksize = 1, .cra_ctxsize = sizeof(struct qat_alg_ablkcipher_ctx), .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, @@ -1265,8 +1367,8 @@ static struct crypto_alg qat_algs[] = { { .cra_u = { .ablkcipher = { .setkey = qat_alg_ablkcipher_xts_setkey, - .decrypt = qat_alg_ablkcipher_decrypt, - .encrypt = qat_alg_ablkcipher_encrypt, + .decrypt = qat_alg_ablkcipher_blk_decrypt, + .encrypt = qat_alg_ablkcipher_blk_encrypt, .min_keysize = 2 * AES_MIN_KEY_SIZE, .max_keysize = 2 * AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, diff --git a/drivers/crypto/qat/qat_common/qat_crypto.h b/drivers/crypto/qat/qat_common/qat_crypto.h index dc0273fe3620..c77a80020cde 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.h +++ b/drivers/crypto/qat/qat_common/qat_crypto.h @@ -88,6 +88,8 @@ struct qat_crypto_request { struct qat_crypto_request_buffs buf; void (*cb)(struct icp_qat_fw_la_resp *resp, struct qat_crypto_request *req); + void *iv; + dma_addr_t iv_paddr; }; #endif diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index 6b498a90181e..b0b8e3d48aef 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -1384,7 +1384,6 @@ MODULE_DEVICE_TABLE(of, sahara_dt_ids); static int sahara_probe(struct platform_device *pdev) { struct sahara_dev *dev; - struct resource *res; u32 version; int irq; int err; @@ -1398,8 +1397,7 @@ static int sahara_probe(struct platform_device *pdev) platform_set_drvdata(pdev, dev); /* Get the base address */ - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - dev->regs_base = devm_ioremap_resource(&pdev->dev, res); + dev->regs_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(dev->regs_base)) return PTR_ERR(dev->regs_base); diff --git a/drivers/crypto/stm32/Makefile b/drivers/crypto/stm32/Makefile index ce77e38c77e0..518e0e0b11a9 100644 --- a/drivers/crypto/stm32/Makefile +++ b/drivers/crypto/stm32/Makefile @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_CRYPTO_DEV_STM32_CRC) += stm32_crc32.o +obj-$(CONFIG_CRYPTO_DEV_STM32_CRC) += stm32-crc32.o obj-$(CONFIG_CRYPTO_DEV_STM32_HASH) += stm32-hash.o obj-$(CONFIG_CRYPTO_DEV_STM32_CRYP) += stm32-cryp.o diff --git a/drivers/crypto/stm32/stm32_crc32.c b/drivers/crypto/stm32/stm32-crc32.c index 440c9f1bd006..440c9f1bd006 100644 --- a/drivers/crypto/stm32/stm32_crc32.c +++ b/drivers/crypto/stm32/stm32-crc32.c diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c index 29519d1c403f..23061f2bc74b 100644 --- a/drivers/crypto/stm32/stm32-hash.c +++ b/drivers/crypto/stm32/stm32-hash.c @@ -349,7 +349,7 @@ static int stm32_hash_xmit_cpu(struct stm32_hash_dev *hdev, return -ETIMEDOUT; if ((hdev->flags & HASH_FLAGS_HMAC) && - (hdev->flags & ~HASH_FLAGS_HMAC_KEY)) { + (!(hdev->flags & HASH_FLAGS_HMAC_KEY))) { hdev->flags |= HASH_FLAGS_HMAC_KEY; stm32_hash_write_key(hdev); if (stm32_hash_wait_busy(hdev)) @@ -447,8 +447,8 @@ static int stm32_hash_xmit_dma(struct stm32_hash_dev *hdev, dma_async_issue_pending(hdev->dma_lch); - if (!wait_for_completion_interruptible_timeout(&hdev->dma_completion, - msecs_to_jiffies(100))) + if (!wait_for_completion_timeout(&hdev->dma_completion, + msecs_to_jiffies(100))) err = -ETIMEDOUT; if (dma_async_is_tx_complete(hdev->dma_lch, cookie, diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c index 7b0c42882830..4ab14d58e85b 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-cipher.c @@ -12,7 +12,7 @@ */ #include "sun4i-ss.h" -static int sun4i_ss_opti_poll(struct skcipher_request *areq) +static int noinline_for_stack sun4i_ss_opti_poll(struct skcipher_request *areq) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); @@ -114,6 +114,29 @@ release_ss: return err; } + +static int noinline_for_stack sun4i_ss_cipher_poll_fallback(struct skcipher_request *areq) +{ + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(areq); + struct sun4i_tfm_ctx *op = crypto_skcipher_ctx(tfm); + struct sun4i_cipher_req_ctx *ctx = skcipher_request_ctx(areq); + SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, op->fallback_tfm); + int err; + + skcipher_request_set_sync_tfm(subreq, op->fallback_tfm); + skcipher_request_set_callback(subreq, areq->base.flags, NULL, + NULL); + skcipher_request_set_crypt(subreq, areq->src, areq->dst, + areq->cryptlen, areq->iv); + if (ctx->mode & SS_DECRYPTION) + err = crypto_skcipher_decrypt(subreq); + else + err = crypto_skcipher_encrypt(subreq); + skcipher_request_zero(subreq); + + return err; +} + /* Generic function that support SG with size not multiple of 4 */ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) { @@ -140,8 +163,6 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) unsigned int todo; struct sg_mapping_iter mi, mo; unsigned int oi, oo; /* offset for in and out */ - char buf[4 * SS_RX_MAX];/* buffer for linearize SG src */ - char bufo[4 * SS_TX_MAX]; /* buffer for linearize SG dst */ unsigned int ob = 0; /* offset in buf */ unsigned int obo = 0; /* offset in bufo*/ unsigned int obl = 0; /* length of data in bufo */ @@ -178,20 +199,8 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) if (no_chunk == 1 && !need_fallback) return sun4i_ss_opti_poll(areq); - if (need_fallback) { - SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, op->fallback_tfm); - skcipher_request_set_sync_tfm(subreq, op->fallback_tfm); - skcipher_request_set_callback(subreq, areq->base.flags, NULL, - NULL); - skcipher_request_set_crypt(subreq, areq->src, areq->dst, - areq->cryptlen, areq->iv); - if (ctx->mode & SS_DECRYPTION) - err = crypto_skcipher_decrypt(subreq); - else - err = crypto_skcipher_encrypt(subreq); - skcipher_request_zero(subreq); - return err; - } + if (need_fallback) + return sun4i_ss_cipher_poll_fallback(areq); spin_lock_irqsave(&ss->slock, flags); @@ -224,6 +233,8 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) while (oleft) { if (ileft) { + char buf[4 * SS_RX_MAX];/* buffer for linearize SG src */ + /* * todo is the number of consecutive 4byte word that we * can read from current SG @@ -281,6 +292,8 @@ static int sun4i_ss_cipher_poll(struct skcipher_request *areq) oo = 0; } } else { + char bufo[4 * SS_TX_MAX]; /* buffer for linearize SG dst */ + /* * read obl bytes in bufo, we read at maximum for * emptying the device diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index fbc7bf9d7380..c9d686a0e805 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -265,11 +265,11 @@ static int init_device(struct device *dev) * callback must check err and feedback in descriptor header * for device processing status. */ -int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, - void (*callback)(struct device *dev, - struct talitos_desc *desc, - void *context, int error), - void *context) +static int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, + void (*callback)(struct device *dev, + struct talitos_desc *desc, + void *context, int error), + void *context) { struct talitos_private *priv = dev_get_drvdata(dev); struct talitos_request *request; @@ -319,7 +319,21 @@ int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, return -EINPROGRESS; } -EXPORT_SYMBOL(talitos_submit); + +static __be32 get_request_hdr(struct talitos_request *request, bool is_sec1) +{ + struct talitos_edesc *edesc; + + if (!is_sec1) + return request->desc->hdr; + + if (!request->desc->next_desc) + return request->desc->hdr1; + + edesc = container_of(request->desc, struct talitos_edesc, desc); + + return ((struct talitos_desc *)(edesc->buf + edesc->dma_len))->hdr1; +} /* * process what was done, notify callback of error if not @@ -342,12 +356,7 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch) /* descriptors with their done bits set don't get the error */ rmb(); - if (!is_sec1) - hdr = request->desc->hdr; - else if (request->desc->next_desc) - hdr = (request->desc + 1)->hdr1; - else - hdr = request->desc->hdr1; + hdr = get_request_hdr(request, is_sec1); if ((hdr & DESC_HDR_DONE) == DESC_HDR_DONE) status = 0; @@ -477,8 +486,14 @@ static u32 current_desc_hdr(struct device *dev, int ch) } } - if (priv->chan[ch].fifo[iter].desc->next_desc == cur_desc) - return (priv->chan[ch].fifo[iter].desc + 1)->hdr; + if (priv->chan[ch].fifo[iter].desc->next_desc == cur_desc) { + struct talitos_edesc *edesc; + + edesc = container_of(priv->chan[ch].fifo[iter].desc, + struct talitos_edesc, desc); + return ((struct talitos_desc *) + (edesc->buf + edesc->dma_len))->hdr; + } return priv->chan[ch].fifo[iter].desc->hdr; } @@ -824,7 +839,11 @@ static void talitos_unregister_rng(struct device *dev) * HMAC_SNOOP_NO_AFEA (HSNA) instead of type IPSEC_ESP */ #define TALITOS_CRA_PRIORITY_AEAD_HSNA (TALITOS_CRA_PRIORITY - 1) +#ifdef CONFIG_CRYPTO_DEV_TALITOS2 #define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA512_BLOCK_SIZE) +#else +#define TALITOS_MAX_KEY_SIZE (AES_MAX_KEY_SIZE + SHA256_BLOCK_SIZE) +#endif #define TALITOS_MAX_IV_LENGTH 16 /* max of AES_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE */ struct talitos_ctx { @@ -948,36 +967,6 @@ badkey: goto out; } -/* - * talitos_edesc - s/w-extended descriptor - * @src_nents: number of segments in input scatterlist - * @dst_nents: number of segments in output scatterlist - * @icv_ool: whether ICV is out-of-line - * @iv_dma: dma address of iv for checking continuity and link table - * @dma_len: length of dma mapped link_tbl space - * @dma_link_tbl: bus physical address of link_tbl/buf - * @desc: h/w descriptor - * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2) - * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1) - * - * if decrypting (with authcheck), or either one of src_nents or dst_nents - * is greater than 1, an integrity check value is concatenated to the end - * of link_tbl data - */ -struct talitos_edesc { - int src_nents; - int dst_nents; - bool icv_ool; - dma_addr_t iv_dma; - int dma_len; - dma_addr_t dma_link_tbl; - struct talitos_desc desc; - union { - struct talitos_ptr link_tbl[0]; - u8 buf[0]; - }; -}; - static void talitos_sg_unmap(struct device *dev, struct talitos_edesc *edesc, struct scatterlist *src, @@ -1008,11 +997,13 @@ static void talitos_sg_unmap(struct device *dev, static void ipsec_esp_unmap(struct device *dev, struct talitos_edesc *edesc, - struct aead_request *areq) + struct aead_request *areq, bool encrypt) { struct crypto_aead *aead = crypto_aead_reqtfm(areq); struct talitos_ctx *ctx = crypto_aead_ctx(aead); unsigned int ivsize = crypto_aead_ivsize(aead); + unsigned int authsize = crypto_aead_authsize(aead); + unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); bool is_ipsec_esp = edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP; struct talitos_ptr *civ_ptr = &edesc->desc.ptr[is_ipsec_esp ? 2 : 3]; @@ -1021,8 +1012,8 @@ static void ipsec_esp_unmap(struct device *dev, DMA_FROM_DEVICE); unmap_single_talitos_ptr(dev, civ_ptr, DMA_TO_DEVICE); - talitos_sg_unmap(dev, edesc, areq->src, areq->dst, areq->cryptlen, - areq->assoclen); + talitos_sg_unmap(dev, edesc, areq->src, areq->dst, + cryptlen + authsize, areq->assoclen); if (edesc->dma_len) dma_unmap_single(dev, edesc->dma_link_tbl, edesc->dma_len, @@ -1032,7 +1023,7 @@ static void ipsec_esp_unmap(struct device *dev, unsigned int dst_nents = edesc->dst_nents ? : 1; sg_pcopy_to_buffer(areq->dst, dst_nents, ctx->iv, ivsize, - areq->assoclen + areq->cryptlen - ivsize); + areq->assoclen + cryptlen - ivsize); } } @@ -1043,31 +1034,14 @@ static void ipsec_esp_encrypt_done(struct device *dev, struct talitos_desc *desc, void *context, int err) { - struct talitos_private *priv = dev_get_drvdata(dev); - bool is_sec1 = has_ftr_sec1(priv); struct aead_request *areq = context; struct crypto_aead *authenc = crypto_aead_reqtfm(areq); - unsigned int authsize = crypto_aead_authsize(authenc); unsigned int ivsize = crypto_aead_ivsize(authenc); struct talitos_edesc *edesc; - struct scatterlist *sg; - void *icvdata; edesc = container_of(desc, struct talitos_edesc, desc); - ipsec_esp_unmap(dev, edesc, areq); - - /* copy the generated ICV to dst */ - if (edesc->icv_ool) { - if (is_sec1) - icvdata = edesc->buf + areq->assoclen + areq->cryptlen; - else - icvdata = &edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 2]; - sg = sg_last(areq->dst, edesc->dst_nents); - memcpy((char *)sg_virt(sg) + sg->length - authsize, - icvdata, authsize); - } + ipsec_esp_unmap(dev, edesc, areq, true); dma_unmap_single(dev, edesc->iv_dma, ivsize, DMA_TO_DEVICE); @@ -1084,32 +1058,16 @@ static void ipsec_esp_decrypt_swauth_done(struct device *dev, struct crypto_aead *authenc = crypto_aead_reqtfm(req); unsigned int authsize = crypto_aead_authsize(authenc); struct talitos_edesc *edesc; - struct scatterlist *sg; char *oicv, *icv; - struct talitos_private *priv = dev_get_drvdata(dev); - bool is_sec1 = has_ftr_sec1(priv); edesc = container_of(desc, struct talitos_edesc, desc); - ipsec_esp_unmap(dev, edesc, req); + ipsec_esp_unmap(dev, edesc, req, false); if (!err) { /* auth check */ - sg = sg_last(req->dst, edesc->dst_nents ? : 1); - icv = (char *)sg_virt(sg) + sg->length - authsize; - - if (edesc->dma_len) { - if (is_sec1) - oicv = (char *)&edesc->dma_link_tbl + - req->assoclen + req->cryptlen; - else - oicv = (char *) - &edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 2]; - if (edesc->icv_ool) - icv = oicv + authsize; - } else - oicv = (char *)&edesc->link_tbl[0]; + oicv = edesc->buf + edesc->dma_len; + icv = oicv - authsize; err = crypto_memneq(oicv, icv, authsize) ? -EBADMSG : 0; } @@ -1128,7 +1086,7 @@ static void ipsec_esp_decrypt_hwauth_done(struct device *dev, edesc = container_of(desc, struct talitos_edesc, desc); - ipsec_esp_unmap(dev, edesc, req); + ipsec_esp_unmap(dev, edesc, req, false); /* check ICV auth status */ if (!err && ((desc->hdr_lo & DESC_HDR_LO_ICCR1_MASK) != @@ -1145,11 +1103,12 @@ static void ipsec_esp_decrypt_hwauth_done(struct device *dev, * stop at cryptlen bytes */ static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, - unsigned int offset, int cryptlen, + unsigned int offset, int datalen, int elen, struct talitos_ptr *link_tbl_ptr) { - int n_sg = sg_count; + int n_sg = elen ? sg_count + 1 : sg_count; int count = 0; + int cryptlen = datalen + elen; while (cryptlen && sg && n_sg--) { unsigned int len = sg_dma_len(sg); @@ -1164,11 +1123,20 @@ static int sg_to_link_tbl_offset(struct scatterlist *sg, int sg_count, if (len > cryptlen) len = cryptlen; + if (datalen > 0 && len > datalen) { + to_talitos_ptr(link_tbl_ptr + count, + sg_dma_address(sg) + offset, datalen, 0); + to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); + count++; + len -= datalen; + offset += datalen; + } to_talitos_ptr(link_tbl_ptr + count, sg_dma_address(sg) + offset, len, 0); to_talitos_ptr_ext_set(link_tbl_ptr + count, 0, 0); count++; cryptlen -= len; + datalen -= len; offset = 0; next: @@ -1178,7 +1146,7 @@ next: /* tag end of link table */ if (count > 0) to_talitos_ptr_ext_set(link_tbl_ptr + count - 1, - DESC_PTR_LNKTBL_RETURN, 0); + DESC_PTR_LNKTBL_RET, 0); return count; } @@ -1186,7 +1154,8 @@ next: static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src, unsigned int len, struct talitos_edesc *edesc, struct talitos_ptr *ptr, int sg_count, - unsigned int offset, int tbl_off, int elen) + unsigned int offset, int tbl_off, int elen, + bool force) { struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); @@ -1196,7 +1165,7 @@ static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src, return 1; } to_talitos_ptr_ext_set(ptr, elen, is_sec1); - if (sg_count == 1) { + if (sg_count == 1 && !force) { to_talitos_ptr(ptr, sg_dma_address(src) + offset, len, is_sec1); return sg_count; } @@ -1204,9 +1173,9 @@ static int talitos_sg_map_ext(struct device *dev, struct scatterlist *src, to_talitos_ptr(ptr, edesc->dma_link_tbl + offset, len, is_sec1); return sg_count; } - sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len + elen, + sg_count = sg_to_link_tbl_offset(src, sg_count, offset, len, elen, &edesc->link_tbl[tbl_off]); - if (sg_count == 1) { + if (sg_count == 1 && !force) { /* Only one segment now, so no link tbl needed*/ copy_talitos_ptr(ptr, &edesc->link_tbl[tbl_off], is_sec1); return sg_count; @@ -1224,13 +1193,14 @@ static int talitos_sg_map(struct device *dev, struct scatterlist *src, unsigned int offset, int tbl_off) { return talitos_sg_map_ext(dev, src, len, edesc, ptr, sg_count, offset, - tbl_off, 0); + tbl_off, 0, false); } /* * fill in and submit ipsec_esp descriptor */ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, + bool encrypt, void (*callback)(struct device *dev, struct talitos_desc *desc, void *context, int error)) @@ -1240,7 +1210,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, struct talitos_ctx *ctx = crypto_aead_ctx(aead); struct device *dev = ctx->dev; struct talitos_desc *desc = &edesc->desc; - unsigned int cryptlen = areq->cryptlen; + unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); unsigned int ivsize = crypto_aead_ivsize(aead); int tbl_off = 0; int sg_count, ret; @@ -1251,6 +1221,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, bool is_ipsec_esp = desc->hdr & DESC_HDR_TYPE_IPSEC_ESP; struct talitos_ptr *civ_ptr = &desc->ptr[is_ipsec_esp ? 2 : 3]; struct talitos_ptr *ckey_ptr = &desc->ptr[is_ipsec_esp ? 3 : 2]; + dma_addr_t dma_icv = edesc->dma_link_tbl + edesc->dma_len - authsize; /* hmac key */ to_talitos_ptr(&desc->ptr[0], ctx->dma_key, ctx->authkeylen, is_sec1); @@ -1290,7 +1261,8 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, elen = authsize; ret = talitos_sg_map_ext(dev, areq->src, cryptlen, edesc, &desc->ptr[4], - sg_count, areq->assoclen, tbl_off, elen); + sg_count, areq->assoclen, tbl_off, elen, + false); if (ret > 1) { tbl_off += ret; @@ -1304,55 +1276,32 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, dma_map_sg(dev, areq->dst, sg_count, DMA_FROM_DEVICE); } - ret = talitos_sg_map(dev, areq->dst, cryptlen, edesc, &desc->ptr[5], - sg_count, areq->assoclen, tbl_off); - - if (is_ipsec_esp) - to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1); - - /* ICV data */ - if (ret > 1) { - tbl_off += ret; - edesc->icv_ool = true; - sync_needed = true; - - if (is_ipsec_esp) { - struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; - int offset = (edesc->src_nents + edesc->dst_nents + 2) * - sizeof(struct talitos_ptr) + authsize; - - /* Add an entry to the link table for ICV data */ - to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1); - to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RETURN, - is_sec1); + if (is_ipsec_esp && encrypt) + elen = authsize; + else + elen = 0; + ret = talitos_sg_map_ext(dev, areq->dst, cryptlen, edesc, &desc->ptr[5], + sg_count, areq->assoclen, tbl_off, elen, + is_ipsec_esp && !encrypt); + tbl_off += ret; - /* icv data follows link tables */ - to_talitos_ptr(tbl_ptr, edesc->dma_link_tbl + offset, - authsize, is_sec1); - } else { - dma_addr_t addr = edesc->dma_link_tbl; + if (!encrypt && is_ipsec_esp) { + struct talitos_ptr *tbl_ptr = &edesc->link_tbl[tbl_off]; - if (is_sec1) - addr += areq->assoclen + cryptlen; - else - addr += sizeof(struct talitos_ptr) * tbl_off; + /* Add an entry to the link table for ICV data */ + to_talitos_ptr_ext_set(tbl_ptr - 1, 0, is_sec1); + to_talitos_ptr_ext_set(tbl_ptr, DESC_PTR_LNKTBL_RET, is_sec1); - to_talitos_ptr(&desc->ptr[6], addr, authsize, is_sec1); - } + /* icv data follows link tables */ + to_talitos_ptr(tbl_ptr, dma_icv, authsize, is_sec1); + to_talitos_ptr_ext_or(&desc->ptr[5], authsize, is_sec1); + sync_needed = true; + } else if (!encrypt) { + to_talitos_ptr(&desc->ptr[6], dma_icv, authsize, is_sec1); + sync_needed = true; } else if (!is_ipsec_esp) { - ret = talitos_sg_map(dev, areq->dst, authsize, edesc, - &desc->ptr[6], sg_count, areq->assoclen + - cryptlen, - tbl_off); - if (ret > 1) { - tbl_off += ret; - edesc->icv_ool = true; - sync_needed = true; - } else { - edesc->icv_ool = false; - } - } else { - edesc->icv_ool = false; + talitos_sg_map(dev, areq->dst, authsize, edesc, &desc->ptr[6], + sg_count, areq->assoclen + cryptlen, tbl_off); } /* iv out */ @@ -1367,7 +1316,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, ret = talitos_submit(dev, ctx->ch, desc, callback, areq); if (ret != -EINPROGRESS) { - ipsec_esp_unmap(dev, edesc, areq); + ipsec_esp_unmap(dev, edesc, areq, encrypt); kfree(edesc); } return ret; @@ -1435,18 +1384,18 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, * and space for two sets of ICVs (stashed and generated) */ alloc_len = sizeof(struct talitos_edesc); - if (src_nents || dst_nents) { + if (src_nents || dst_nents || !encrypt) { if (is_sec1) dma_len = (src_nents ? src_len : 0) + - (dst_nents ? dst_len : 0); + (dst_nents ? dst_len : 0) + authsize; else dma_len = (src_nents + dst_nents + 2) * - sizeof(struct talitos_ptr) + authsize * 2; + sizeof(struct talitos_ptr) + authsize; alloc_len += dma_len; } else { dma_len = 0; - alloc_len += icv_stashing ? authsize : 0; } + alloc_len += icv_stashing ? authsize : 0; /* if its a ahash, add space for a second desc next to the first one */ if (is_sec1 && !dst) @@ -1466,15 +1415,11 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, edesc->dst_nents = dst_nents; edesc->iv_dma = iv_dma; edesc->dma_len = dma_len; - if (dma_len) { - void *addr = &edesc->link_tbl[0]; - - if (is_sec1 && !dst) - addr += sizeof(struct talitos_desc); - edesc->dma_link_tbl = dma_map_single(dev, addr, + if (dma_len) + edesc->dma_link_tbl = dma_map_single(dev, &edesc->link_tbl[0], edesc->dma_len, DMA_BIDIRECTIONAL); - } + return edesc; } @@ -1485,9 +1430,10 @@ static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv, unsigned int authsize = crypto_aead_authsize(authenc); struct talitos_ctx *ctx = crypto_aead_ctx(authenc); unsigned int ivsize = crypto_aead_ivsize(authenc); + unsigned int cryptlen = areq->cryptlen - (encrypt ? 0 : authsize); return talitos_edesc_alloc(ctx->dev, areq->src, areq->dst, - iv, areq->assoclen, areq->cryptlen, + iv, areq->assoclen, cryptlen, authsize, ivsize, icv_stashing, areq->base.flags, encrypt); } @@ -1506,7 +1452,7 @@ static int aead_encrypt(struct aead_request *req) /* set encrypt */ edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_MODE0_ENCRYPT; - return ipsec_esp(edesc, req, ipsec_esp_encrypt_done); + return ipsec_esp(edesc, req, true, ipsec_esp_encrypt_done); } static int aead_decrypt(struct aead_request *req) @@ -1516,17 +1462,15 @@ static int aead_decrypt(struct aead_request *req) struct talitos_ctx *ctx = crypto_aead_ctx(authenc); struct talitos_private *priv = dev_get_drvdata(ctx->dev); struct talitos_edesc *edesc; - struct scatterlist *sg; void *icvdata; - req->cryptlen -= authsize; - /* allocate extended descriptor */ edesc = aead_edesc_alloc(req, req->iv, 1, false); if (IS_ERR(edesc)) return PTR_ERR(edesc); - if ((priv->features & TALITOS_FTR_HW_AUTH_CHECK) && + if ((edesc->desc.hdr & DESC_HDR_TYPE_IPSEC_ESP) && + (priv->features & TALITOS_FTR_HW_AUTH_CHECK) && ((!edesc->src_nents && !edesc->dst_nents) || priv->features & TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT)) { @@ -1537,24 +1481,20 @@ static int aead_decrypt(struct aead_request *req) /* reset integrity check result bits */ - return ipsec_esp(edesc, req, ipsec_esp_decrypt_hwauth_done); + return ipsec_esp(edesc, req, false, + ipsec_esp_decrypt_hwauth_done); } /* Have to check the ICV with software */ edesc->desc.hdr = ctx->desc_hdr_template | DESC_HDR_DIR_INBOUND; /* stash incoming ICV for later cmp with ICV generated by the h/w */ - if (edesc->dma_len) - icvdata = (char *)&edesc->link_tbl[edesc->src_nents + - edesc->dst_nents + 2]; - else - icvdata = &edesc->link_tbl[0]; + icvdata = edesc->buf + edesc->dma_len; - sg = sg_last(req->src, edesc->src_nents ? : 1); + sg_pcopy_to_buffer(req->src, edesc->src_nents ? : 1, icvdata, authsize, + req->assoclen + req->cryptlen - authsize); - memcpy(icvdata, (char *)sg_virt(sg) + sg->length - authsize, authsize); - - return ipsec_esp(edesc, req, ipsec_esp_decrypt_swauth_done); + return ipsec_esp(edesc, req, false, ipsec_esp_decrypt_swauth_done); } static int ablkcipher_setkey(struct crypto_ablkcipher *cipher, @@ -1605,6 +1545,18 @@ static int ablkcipher_des3_setkey(struct crypto_ablkcipher *cipher, return ablkcipher_setkey(cipher, key, keylen); } +static int ablkcipher_aes_setkey(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen) +{ + if (keylen == AES_KEYSIZE_128 || keylen == AES_KEYSIZE_192 || + keylen == AES_KEYSIZE_256) + return ablkcipher_setkey(cipher, key, keylen); + + crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + + return -EINVAL; +} + static void common_nonsnoop_unmap(struct device *dev, struct talitos_edesc *edesc, struct ablkcipher_request *areq) @@ -1624,11 +1576,15 @@ static void ablkcipher_done(struct device *dev, int err) { struct ablkcipher_request *areq = context; + struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); + struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); + unsigned int ivsize = crypto_ablkcipher_ivsize(cipher); struct talitos_edesc *edesc; edesc = container_of(desc, struct talitos_edesc, desc); common_nonsnoop_unmap(dev, edesc, areq); + memcpy(areq->info, ctx->iv, ivsize); kfree(edesc); @@ -1723,6 +1679,14 @@ static int ablkcipher_encrypt(struct ablkcipher_request *areq) struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); struct talitos_edesc *edesc; + unsigned int blocksize = + crypto_tfm_alg_blocksize(crypto_ablkcipher_tfm(cipher)); + + if (!areq->nbytes) + return 0; + + if (areq->nbytes % blocksize) + return -EINVAL; /* allocate extended descriptor */ edesc = ablkcipher_edesc_alloc(areq, true); @@ -1740,6 +1704,14 @@ static int ablkcipher_decrypt(struct ablkcipher_request *areq) struct crypto_ablkcipher *cipher = crypto_ablkcipher_reqtfm(areq); struct talitos_ctx *ctx = crypto_ablkcipher_ctx(cipher); struct talitos_edesc *edesc; + unsigned int blocksize = + crypto_tfm_alg_blocksize(crypto_ablkcipher_tfm(cipher)); + + if (!areq->nbytes) + return 0; + + if (areq->nbytes % blocksize) + return -EINVAL; /* allocate extended descriptor */ edesc = ablkcipher_edesc_alloc(areq, false); @@ -1759,14 +1731,16 @@ static void common_nonsnoop_hash_unmap(struct device *dev, struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); struct talitos_desc *desc = &edesc->desc; - struct talitos_desc *desc2 = desc + 1; + struct talitos_desc *desc2 = (struct talitos_desc *) + (edesc->buf + edesc->dma_len); unmap_single_talitos_ptr(dev, &edesc->desc.ptr[5], DMA_FROM_DEVICE); if (desc->next_desc && desc->ptr[5].ptr != desc2->ptr[5].ptr) unmap_single_talitos_ptr(dev, &desc2->ptr[5], DMA_FROM_DEVICE); - talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0); + if (req_ctx->psrc) + talitos_sg_unmap(dev, edesc, req_ctx->psrc, NULL, 0, 0); /* When using hashctx-in, must unmap it. */ if (from_talitos_ptr_len(&edesc->desc.ptr[1], is_sec1)) @@ -1833,7 +1807,6 @@ static void talitos_handle_buggy_hash(struct talitos_ctx *ctx, static int common_nonsnoop_hash(struct talitos_edesc *edesc, struct ahash_request *areq, unsigned int length, - unsigned int offset, void (*callback) (struct device *dev, struct talitos_desc *desc, void *context, int error)) @@ -1872,9 +1845,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, sg_count = edesc->src_nents ?: 1; if (is_sec1 && sg_count > 1) - sg_pcopy_to_buffer(req_ctx->psrc, sg_count, - edesc->buf + sizeof(struct talitos_desc), - length, req_ctx->nbuf); + sg_copy_to_buffer(req_ctx->psrc, sg_count, edesc->buf, length); else if (length) sg_count = dma_map_sg(dev, req_ctx->psrc, sg_count, DMA_TO_DEVICE); @@ -1887,7 +1858,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, DMA_TO_DEVICE); } else { sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, - &desc->ptr[3], sg_count, offset, 0); + &desc->ptr[3], sg_count, 0, 0); if (sg_count > 1) sync_needed = true; } @@ -1911,7 +1882,8 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, talitos_handle_buggy_hash(ctx, edesc, &desc->ptr[3]); if (is_sec1 && req_ctx->nbuf && length) { - struct talitos_desc *desc2 = desc + 1; + struct talitos_desc *desc2 = (struct talitos_desc *) + (edesc->buf + edesc->dma_len); dma_addr_t next_desc; memset(desc2, 0, sizeof(*desc2)); @@ -1932,7 +1904,7 @@ static int common_nonsnoop_hash(struct talitos_edesc *edesc, DMA_TO_DEVICE); copy_talitos_ptr(&desc2->ptr[2], &desc->ptr[2], is_sec1); sg_count = talitos_sg_map(dev, req_ctx->psrc, length, edesc, - &desc2->ptr[3], sg_count, offset, 0); + &desc2->ptr[3], sg_count, 0, 0); if (sg_count > 1) sync_needed = true; copy_talitos_ptr(&desc2->ptr[5], &desc->ptr[5], is_sec1); @@ -2043,7 +2015,6 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) struct device *dev = ctx->dev; struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); - int offset = 0; u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx]; if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) { @@ -2083,6 +2054,8 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) sg_chain(req_ctx->bufsl, 2, areq->src); req_ctx->psrc = req_ctx->bufsl; } else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) { + int offset; + if (nbytes_to_hash > blocksize) offset = blocksize - req_ctx->nbuf; else @@ -2095,7 +2068,8 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) sg_copy_to_buffer(areq->src, nents, ctx_buf + req_ctx->nbuf, offset); req_ctx->nbuf += offset; - req_ctx->psrc = areq->src; + req_ctx->psrc = scatterwalk_ffwd(req_ctx->bufsl, areq->src, + offset); } else req_ctx->psrc = areq->src; @@ -2135,8 +2109,7 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) if (ctx->keylen && (req_ctx->first || req_ctx->last)) edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC; - return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, offset, - ahash_done); + return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, ahash_done); } static int ahash_update(struct ahash_request *areq) @@ -2339,7 +2312,7 @@ static struct talitos_alg_template driver_algs[] = { .base = { .cra_name = "authenc(hmac(sha1),cbc(aes))", .cra_driver_name = "authenc-hmac-sha1-" - "cbc-aes-talitos", + "cbc-aes-talitos-hsna", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2384,7 +2357,7 @@ static struct talitos_alg_template driver_algs[] = { .cra_name = "authenc(hmac(sha1)," "cbc(des3_ede))", .cra_driver_name = "authenc-hmac-sha1-" - "cbc-3des-talitos", + "cbc-3des-talitos-hsna", .cra_blocksize = DES3_EDE_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2427,7 +2400,7 @@ static struct talitos_alg_template driver_algs[] = { .base = { .cra_name = "authenc(hmac(sha224),cbc(aes))", .cra_driver_name = "authenc-hmac-sha224-" - "cbc-aes-talitos", + "cbc-aes-talitos-hsna", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2472,7 +2445,7 @@ static struct talitos_alg_template driver_algs[] = { .cra_name = "authenc(hmac(sha224)," "cbc(des3_ede))", .cra_driver_name = "authenc-hmac-sha224-" - "cbc-3des-talitos", + "cbc-3des-talitos-hsna", .cra_blocksize = DES3_EDE_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2515,7 +2488,7 @@ static struct talitos_alg_template driver_algs[] = { .base = { .cra_name = "authenc(hmac(sha256),cbc(aes))", .cra_driver_name = "authenc-hmac-sha256-" - "cbc-aes-talitos", + "cbc-aes-talitos-hsna", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2560,7 +2533,7 @@ static struct talitos_alg_template driver_algs[] = { .cra_name = "authenc(hmac(sha256)," "cbc(des3_ede))", .cra_driver_name = "authenc-hmac-sha256-" - "cbc-3des-talitos", + "cbc-3des-talitos-hsna", .cra_blocksize = DES3_EDE_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2689,7 +2662,7 @@ static struct talitos_alg_template driver_algs[] = { .base = { .cra_name = "authenc(hmac(md5),cbc(aes))", .cra_driver_name = "authenc-hmac-md5-" - "cbc-aes-talitos", + "cbc-aes-talitos-hsna", .cra_blocksize = AES_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2732,7 +2705,7 @@ static struct talitos_alg_template driver_algs[] = { .base = { .cra_name = "authenc(hmac(md5),cbc(des3_ede))", .cra_driver_name = "authenc-hmac-md5-" - "cbc-3des-talitos", + "cbc-3des-talitos-hsna", .cra_blocksize = DES3_EDE_BLOCK_SIZE, .cra_flags = CRYPTO_ALG_ASYNC, }, @@ -2760,7 +2733,7 @@ static struct talitos_alg_template driver_algs[] = { .cra_ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, - .ivsize = AES_BLOCK_SIZE, + .setkey = ablkcipher_aes_setkey, } }, .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | @@ -2777,6 +2750,7 @@ static struct talitos_alg_template driver_algs[] = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, + .setkey = ablkcipher_aes_setkey, } }, .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | @@ -2787,13 +2761,14 @@ static struct talitos_alg_template driver_algs[] = { .alg.crypto = { .cra_name = "ctr(aes)", .cra_driver_name = "ctr-aes-talitos", - .cra_blocksize = AES_BLOCK_SIZE, + .cra_blocksize = 1, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .ivsize = AES_BLOCK_SIZE, + .setkey = ablkcipher_aes_setkey, } }, .desc_hdr_template = DESC_HDR_TYPE_AESU_CTR_NONSNOOP | @@ -2810,7 +2785,6 @@ static struct talitos_alg_template driver_algs[] = { .cra_ablkcipher = { .min_keysize = DES_KEY_SIZE, .max_keysize = DES_KEY_SIZE, - .ivsize = DES_BLOCK_SIZE, .setkey = ablkcipher_des_setkey, } }, @@ -2845,7 +2819,6 @@ static struct talitos_alg_template driver_algs[] = { .cra_ablkcipher = { .min_keysize = DES3_EDE_KEY_SIZE, .max_keysize = DES3_EDE_KEY_SIZE, - .ivsize = DES3_EDE_BLOCK_SIZE, .setkey = ablkcipher_des3_setkey, } }, @@ -3270,7 +3243,10 @@ static struct talitos_crypto_alg *talitos_alg_alloc(struct device *dev, alg->cra_priority = t_alg->algt.priority; else alg->cra_priority = TALITOS_CRA_PRIORITY; - alg->cra_alignmask = 0; + if (has_ftr_sec1(priv)) + alg->cra_alignmask = 3; + else + alg->cra_alignmask = 0; alg->cra_ctxsize = sizeof(struct talitos_ctx); alg->cra_flags |= CRYPTO_ALG_KERN_DRIVER_ONLY; @@ -3418,7 +3394,7 @@ static int talitos_probe(struct platform_device *ofdev) if (err) goto err_out; - if (of_device_is_compatible(np, "fsl,sec1.0")) { + if (has_ftr_sec1(priv)) { if (priv->num_channels == 1) tasklet_init(&priv->done_task[0], talitos1_done_ch0, (unsigned long)dev); diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h index a65a63e0d6c1..1469b956948a 100644 --- a/drivers/crypto/talitos.h +++ b/drivers/crypto/talitos.h @@ -1,31 +1,8 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ /* * Freescale SEC (talitos) device register and descriptor header defines * * Copyright (c) 2006-2011 Freescale Semiconductor, Inc. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * 3. The name of the author may not be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR - * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. - * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, - * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT - * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF - * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * */ #define TALITOS_TIMEOUT 100000 @@ -65,6 +42,34 @@ struct talitos_desc { #define TALITOS_DESC_SIZE (sizeof(struct talitos_desc) - sizeof(__be32)) +/* + * talitos_edesc - s/w-extended descriptor + * @src_nents: number of segments in input scatterlist + * @dst_nents: number of segments in output scatterlist + * @iv_dma: dma address of iv for checking continuity and link table + * @dma_len: length of dma mapped link_tbl space + * @dma_link_tbl: bus physical address of link_tbl/buf + * @desc: h/w descriptor + * @link_tbl: input and output h/w link tables (if {src,dst}_nents > 1) (SEC2) + * @buf: input and output buffeur (if {src,dst}_nents > 1) (SEC1) + * + * if decrypting (with authcheck), or either one of src_nents or dst_nents + * is greater than 1, an integrity check value is concatenated to the end + * of link_tbl data + */ +struct talitos_edesc { + int src_nents; + int dst_nents; + dma_addr_t iv_dma; + int dma_len; + dma_addr_t dma_link_tbl; + struct talitos_desc desc; + union { + struct talitos_ptr link_tbl[0]; + u8 buf[0]; + }; +}; + /** * talitos_request - descriptor submission request * @desc: descriptor pointer (kernel virtual) @@ -150,12 +155,6 @@ struct talitos_private { bool rng_registered; }; -extern int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, - void (*callback)(struct device *dev, - struct talitos_desc *desc, - void *context, int error), - void *context); - /* .features flag */ #define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001 #define TALITOS_FTR_HW_AUTH_CHECK 0x00000002 @@ -170,13 +169,11 @@ extern int talitos_submit(struct device *dev, int ch, struct talitos_desc *desc, */ static inline bool has_ftr_sec1(struct talitos_private *priv) { -#if defined(CONFIG_CRYPTO_DEV_TALITOS1) && defined(CONFIG_CRYPTO_DEV_TALITOS2) - return priv->features & TALITOS_FTR_SEC1 ? true : false; -#elif defined(CONFIG_CRYPTO_DEV_TALITOS1) - return true; -#else - return false; -#endif + if (IS_ENABLED(CONFIG_CRYPTO_DEV_TALITOS1) && + IS_ENABLED(CONFIG_CRYPTO_DEV_TALITOS2)) + return priv->features & TALITOS_FTR_SEC1; + + return IS_ENABLED(CONFIG_CRYPTO_DEV_TALITOS1); } /* @@ -412,5 +409,5 @@ static inline bool has_ftr_sec1(struct talitos_private *priv) /* link table extent field bits */ #define DESC_PTR_LNKTBL_JUMP 0x80 -#define DESC_PTR_LNKTBL_RETURN 0x02 +#define DESC_PTR_LNKTBL_RET 0x02 #define DESC_PTR_LNKTBL_NEXT 0x01 diff --git a/drivers/crypto/vmx/aes_cbc.c b/drivers/crypto/vmx/aes_cbc.c index c7e515a1bc97..d88084447f1c 100644 --- a/drivers/crypto/vmx/aes_cbc.c +++ b/drivers/crypto/vmx/aes_cbc.c @@ -7,64 +7,52 @@ * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com> */ -#include <linux/types.h> -#include <linux/err.h> -#include <linux/crypto.h> -#include <linux/delay.h> #include <asm/simd.h> #include <asm/switch_to.h> #include <crypto/aes.h> #include <crypto/internal/simd.h> -#include <crypto/scatterwalk.h> -#include <crypto/skcipher.h> +#include <crypto/internal/skcipher.h> #include "aesp8-ppc.h" struct p8_aes_cbc_ctx { - struct crypto_sync_skcipher *fallback; + struct crypto_skcipher *fallback; struct aes_key enc_key; struct aes_key dec_key; }; -static int p8_aes_cbc_init(struct crypto_tfm *tfm) +static int p8_aes_cbc_init(struct crypto_skcipher *tfm) { - const char *alg = crypto_tfm_alg_name(tfm); - struct crypto_sync_skcipher *fallback; - struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm); - - fallback = crypto_alloc_sync_skcipher(alg, 0, - CRYPTO_ALG_NEED_FALLBACK); + struct p8_aes_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); + struct crypto_skcipher *fallback; + fallback = crypto_alloc_skcipher("cbc(aes)", 0, + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC); if (IS_ERR(fallback)) { - printk(KERN_ERR - "Failed to allocate transformation for '%s': %ld\n", - alg, PTR_ERR(fallback)); + pr_err("Failed to allocate cbc(aes) fallback: %ld\n", + PTR_ERR(fallback)); return PTR_ERR(fallback); } - crypto_sync_skcipher_set_flags( - fallback, - crypto_skcipher_get_flags((struct crypto_skcipher *)tfm)); + crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) + + crypto_skcipher_reqsize(fallback)); ctx->fallback = fallback; - return 0; } -static void p8_aes_cbc_exit(struct crypto_tfm *tfm) +static void p8_aes_cbc_exit(struct crypto_skcipher *tfm) { - struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm); + struct p8_aes_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); - if (ctx->fallback) { - crypto_free_sync_skcipher(ctx->fallback); - ctx->fallback = NULL; - } + crypto_free_skcipher(ctx->fallback); } -static int p8_aes_cbc_setkey(struct crypto_tfm *tfm, const u8 *key, +static int p8_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { + struct p8_aes_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); int ret; - struct p8_aes_cbc_ctx *ctx = crypto_tfm_ctx(tfm); preempt_disable(); pagefault_disable(); @@ -75,108 +63,71 @@ static int p8_aes_cbc_setkey(struct crypto_tfm *tfm, const u8 *key, pagefault_enable(); preempt_enable(); - ret |= crypto_sync_skcipher_setkey(ctx->fallback, key, keylen); + ret |= crypto_skcipher_setkey(ctx->fallback, key, keylen); return ret ? -EINVAL : 0; } -static int p8_aes_cbc_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) +static int p8_aes_cbc_crypt(struct skcipher_request *req, int enc) { + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct p8_aes_cbc_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + unsigned int nbytes; int ret; - struct blkcipher_walk walk; - struct p8_aes_cbc_ctx *ctx = - crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); if (!crypto_simd_usable()) { - SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback); - skcipher_request_set_sync_tfm(req, ctx->fallback); - skcipher_request_set_callback(req, desc->flags, NULL, NULL); - skcipher_request_set_crypt(req, src, dst, nbytes, desc->info); - ret = crypto_skcipher_encrypt(req); - skcipher_request_zero(req); - } else { - blkcipher_walk_init(&walk, dst, src, nbytes); - ret = blkcipher_walk_virt(desc, &walk); - while ((nbytes = walk.nbytes)) { - preempt_disable(); - pagefault_disable(); - enable_kernel_vsx(); - aes_p8_cbc_encrypt(walk.src.virt.addr, - walk.dst.virt.addr, - nbytes & AES_BLOCK_MASK, - &ctx->enc_key, walk.iv, 1); - disable_kernel_vsx(); - pagefault_enable(); - preempt_enable(); - - nbytes &= AES_BLOCK_SIZE - 1; - ret = blkcipher_walk_done(desc, &walk, nbytes); - } + struct skcipher_request *subreq = skcipher_request_ctx(req); + + *subreq = *req; + skcipher_request_set_tfm(subreq, ctx->fallback); + return enc ? crypto_skcipher_encrypt(subreq) : + crypto_skcipher_decrypt(subreq); } + ret = skcipher_walk_virt(&walk, req, false); + while ((nbytes = walk.nbytes) != 0) { + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + aes_p8_cbc_encrypt(walk.src.virt.addr, + walk.dst.virt.addr, + round_down(nbytes, AES_BLOCK_SIZE), + enc ? &ctx->enc_key : &ctx->dec_key, + walk.iv, enc); + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + + ret = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE); + } return ret; } -static int p8_aes_cbc_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) +static int p8_aes_cbc_encrypt(struct skcipher_request *req) { - int ret; - struct blkcipher_walk walk; - struct p8_aes_cbc_ctx *ctx = - crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); - - if (!crypto_simd_usable()) { - SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback); - skcipher_request_set_sync_tfm(req, ctx->fallback); - skcipher_request_set_callback(req, desc->flags, NULL, NULL); - skcipher_request_set_crypt(req, src, dst, nbytes, desc->info); - ret = crypto_skcipher_decrypt(req); - skcipher_request_zero(req); - } else { - blkcipher_walk_init(&walk, dst, src, nbytes); - ret = blkcipher_walk_virt(desc, &walk); - while ((nbytes = walk.nbytes)) { - preempt_disable(); - pagefault_disable(); - enable_kernel_vsx(); - aes_p8_cbc_encrypt(walk.src.virt.addr, - walk.dst.virt.addr, - nbytes & AES_BLOCK_MASK, - &ctx->dec_key, walk.iv, 0); - disable_kernel_vsx(); - pagefault_enable(); - preempt_enable(); - - nbytes &= AES_BLOCK_SIZE - 1; - ret = blkcipher_walk_done(desc, &walk, nbytes); - } - } - - return ret; + return p8_aes_cbc_crypt(req, 1); } +static int p8_aes_cbc_decrypt(struct skcipher_request *req) +{ + return p8_aes_cbc_crypt(req, 0); +} -struct crypto_alg p8_aes_cbc_alg = { - .cra_name = "cbc(aes)", - .cra_driver_name = "p8_aes_cbc", - .cra_module = THIS_MODULE, - .cra_priority = 2000, - .cra_type = &crypto_blkcipher_type, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK, - .cra_alignmask = 0, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct p8_aes_cbc_ctx), - .cra_init = p8_aes_cbc_init, - .cra_exit = p8_aes_cbc_exit, - .cra_blkcipher = { - .ivsize = AES_BLOCK_SIZE, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = p8_aes_cbc_setkey, - .encrypt = p8_aes_cbc_encrypt, - .decrypt = p8_aes_cbc_decrypt, - }, +struct skcipher_alg p8_aes_cbc_alg = { + .base.cra_name = "cbc(aes)", + .base.cra_driver_name = "p8_aes_cbc", + .base.cra_module = THIS_MODULE, + .base.cra_priority = 2000, + .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct p8_aes_cbc_ctx), + .setkey = p8_aes_cbc_setkey, + .encrypt = p8_aes_cbc_encrypt, + .decrypt = p8_aes_cbc_decrypt, + .init = p8_aes_cbc_init, + .exit = p8_aes_cbc_exit, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, }; diff --git a/drivers/crypto/vmx/aes_ctr.c b/drivers/crypto/vmx/aes_ctr.c index dd017ef42fa9..79ba062ee1c1 100644 --- a/drivers/crypto/vmx/aes_ctr.c +++ b/drivers/crypto/vmx/aes_ctr.c @@ -7,62 +7,51 @@ * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com> */ -#include <linux/types.h> -#include <linux/err.h> -#include <linux/crypto.h> -#include <linux/delay.h> #include <asm/simd.h> #include <asm/switch_to.h> #include <crypto/aes.h> #include <crypto/internal/simd.h> -#include <crypto/scatterwalk.h> -#include <crypto/skcipher.h> +#include <crypto/internal/skcipher.h> #include "aesp8-ppc.h" struct p8_aes_ctr_ctx { - struct crypto_sync_skcipher *fallback; + struct crypto_skcipher *fallback; struct aes_key enc_key; }; -static int p8_aes_ctr_init(struct crypto_tfm *tfm) +static int p8_aes_ctr_init(struct crypto_skcipher *tfm) { - const char *alg = crypto_tfm_alg_name(tfm); - struct crypto_sync_skcipher *fallback; - struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm); + struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); + struct crypto_skcipher *fallback; - fallback = crypto_alloc_sync_skcipher(alg, 0, - CRYPTO_ALG_NEED_FALLBACK); + fallback = crypto_alloc_skcipher("ctr(aes)", 0, + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC); if (IS_ERR(fallback)) { - printk(KERN_ERR - "Failed to allocate transformation for '%s': %ld\n", - alg, PTR_ERR(fallback)); + pr_err("Failed to allocate ctr(aes) fallback: %ld\n", + PTR_ERR(fallback)); return PTR_ERR(fallback); } - crypto_sync_skcipher_set_flags( - fallback, - crypto_skcipher_get_flags((struct crypto_skcipher *)tfm)); + crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) + + crypto_skcipher_reqsize(fallback)); ctx->fallback = fallback; - return 0; } -static void p8_aes_ctr_exit(struct crypto_tfm *tfm) +static void p8_aes_ctr_exit(struct crypto_skcipher *tfm) { - struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm); + struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); - if (ctx->fallback) { - crypto_free_sync_skcipher(ctx->fallback); - ctx->fallback = NULL; - } + crypto_free_skcipher(ctx->fallback); } -static int p8_aes_ctr_setkey(struct crypto_tfm *tfm, const u8 *key, +static int p8_aes_ctr_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { + struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); int ret; - struct p8_aes_ctr_ctx *ctx = crypto_tfm_ctx(tfm); preempt_disable(); pagefault_disable(); @@ -72,13 +61,13 @@ static int p8_aes_ctr_setkey(struct crypto_tfm *tfm, const u8 *key, pagefault_enable(); preempt_enable(); - ret |= crypto_sync_skcipher_setkey(ctx->fallback, key, keylen); + ret |= crypto_skcipher_setkey(ctx->fallback, key, keylen); return ret ? -EINVAL : 0; } -static void p8_aes_ctr_final(struct p8_aes_ctr_ctx *ctx, - struct blkcipher_walk *walk) +static void p8_aes_ctr_final(const struct p8_aes_ctr_ctx *ctx, + struct skcipher_walk *walk) { u8 *ctrblk = walk->iv; u8 keystream[AES_BLOCK_SIZE]; @@ -98,77 +87,63 @@ static void p8_aes_ctr_final(struct p8_aes_ctr_ctx *ctx, crypto_inc(ctrblk, AES_BLOCK_SIZE); } -static int p8_aes_ctr_crypt(struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) +static int p8_aes_ctr_crypt(struct skcipher_request *req) { + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct p8_aes_ctr_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + unsigned int nbytes; int ret; - u64 inc; - struct blkcipher_walk walk; - struct p8_aes_ctr_ctx *ctx = - crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); if (!crypto_simd_usable()) { - SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback); - skcipher_request_set_sync_tfm(req, ctx->fallback); - skcipher_request_set_callback(req, desc->flags, NULL, NULL); - skcipher_request_set_crypt(req, src, dst, nbytes, desc->info); - ret = crypto_skcipher_encrypt(req); - skcipher_request_zero(req); - } else { - blkcipher_walk_init(&walk, dst, src, nbytes); - ret = blkcipher_walk_virt_block(desc, &walk, AES_BLOCK_SIZE); - while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { - preempt_disable(); - pagefault_disable(); - enable_kernel_vsx(); - aes_p8_ctr32_encrypt_blocks(walk.src.virt.addr, - walk.dst.virt.addr, - (nbytes & - AES_BLOCK_MASK) / - AES_BLOCK_SIZE, - &ctx->enc_key, - walk.iv); - disable_kernel_vsx(); - pagefault_enable(); - preempt_enable(); - - /* We need to update IV mostly for last bytes/round */ - inc = (nbytes & AES_BLOCK_MASK) / AES_BLOCK_SIZE; - if (inc > 0) - while (inc--) - crypto_inc(walk.iv, AES_BLOCK_SIZE); - - nbytes &= AES_BLOCK_SIZE - 1; - ret = blkcipher_walk_done(desc, &walk, nbytes); - } - if (walk.nbytes) { - p8_aes_ctr_final(ctx, &walk); - ret = blkcipher_walk_done(desc, &walk, 0); - } + struct skcipher_request *subreq = skcipher_request_ctx(req); + + *subreq = *req; + skcipher_request_set_tfm(subreq, ctx->fallback); + return crypto_skcipher_encrypt(subreq); } + ret = skcipher_walk_virt(&walk, req, false); + while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); + aes_p8_ctr32_encrypt_blocks(walk.src.virt.addr, + walk.dst.virt.addr, + nbytes / AES_BLOCK_SIZE, + &ctx->enc_key, walk.iv); + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + + do { + crypto_inc(walk.iv, AES_BLOCK_SIZE); + } while ((nbytes -= AES_BLOCK_SIZE) >= AES_BLOCK_SIZE); + + ret = skcipher_walk_done(&walk, nbytes); + } + if (nbytes) { + p8_aes_ctr_final(ctx, &walk); + ret = skcipher_walk_done(&walk, 0); + } return ret; } -struct crypto_alg p8_aes_ctr_alg = { - .cra_name = "ctr(aes)", - .cra_driver_name = "p8_aes_ctr", - .cra_module = THIS_MODULE, - .cra_priority = 2000, - .cra_type = &crypto_blkcipher_type, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK, - .cra_alignmask = 0, - .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct p8_aes_ctr_ctx), - .cra_init = p8_aes_ctr_init, - .cra_exit = p8_aes_ctr_exit, - .cra_blkcipher = { - .ivsize = AES_BLOCK_SIZE, - .min_keysize = AES_MIN_KEY_SIZE, - .max_keysize = AES_MAX_KEY_SIZE, - .setkey = p8_aes_ctr_setkey, - .encrypt = p8_aes_ctr_crypt, - .decrypt = p8_aes_ctr_crypt, - }, +struct skcipher_alg p8_aes_ctr_alg = { + .base.cra_name = "ctr(aes)", + .base.cra_driver_name = "p8_aes_ctr", + .base.cra_module = THIS_MODULE, + .base.cra_priority = 2000, + .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct p8_aes_ctr_ctx), + .setkey = p8_aes_ctr_setkey, + .encrypt = p8_aes_ctr_crypt, + .decrypt = p8_aes_ctr_crypt, + .init = p8_aes_ctr_init, + .exit = p8_aes_ctr_exit, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .chunksize = AES_BLOCK_SIZE, }; diff --git a/drivers/crypto/vmx/aes_xts.c b/drivers/crypto/vmx/aes_xts.c index 536167e737a0..49f7258045fa 100644 --- a/drivers/crypto/vmx/aes_xts.c +++ b/drivers/crypto/vmx/aes_xts.c @@ -7,67 +7,56 @@ * Author: Leonidas S. Barbosa <leosilva@linux.vnet.ibm.com> */ -#include <linux/types.h> -#include <linux/err.h> -#include <linux/crypto.h> -#include <linux/delay.h> #include <asm/simd.h> #include <asm/switch_to.h> #include <crypto/aes.h> #include <crypto/internal/simd.h> -#include <crypto/scatterwalk.h> +#include <crypto/internal/skcipher.h> #include <crypto/xts.h> -#include <crypto/skcipher.h> #include "aesp8-ppc.h" struct p8_aes_xts_ctx { - struct crypto_sync_skcipher *fallback; + struct crypto_skcipher *fallback; struct aes_key enc_key; struct aes_key dec_key; struct aes_key tweak_key; }; -static int p8_aes_xts_init(struct crypto_tfm *tfm) +static int p8_aes_xts_init(struct crypto_skcipher *tfm) { - const char *alg = crypto_tfm_alg_name(tfm); - struct crypto_sync_skcipher *fallback; - struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm); + struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); + struct crypto_skcipher *fallback; - fallback = crypto_alloc_sync_skcipher(alg, 0, - CRYPTO_ALG_NEED_FALLBACK); + fallback = crypto_alloc_skcipher("xts(aes)", 0, + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC); if (IS_ERR(fallback)) { - printk(KERN_ERR - "Failed to allocate transformation for '%s': %ld\n", - alg, PTR_ERR(fallback)); + pr_err("Failed to allocate xts(aes) fallback: %ld\n", + PTR_ERR(fallback)); return PTR_ERR(fallback); } - crypto_sync_skcipher_set_flags( - fallback, - crypto_skcipher_get_flags((struct crypto_skcipher *)tfm)); + crypto_skcipher_set_reqsize(tfm, sizeof(struct skcipher_request) + + crypto_skcipher_reqsize(fallback)); ctx->fallback = fallback; - return 0; } -static void p8_aes_xts_exit(struct crypto_tfm *tfm) +static void p8_aes_xts_exit(struct crypto_skcipher *tfm) { - struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm); + struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); - if (ctx->fallback) { - crypto_free_sync_skcipher(ctx->fallback); - ctx->fallback = NULL; - } + crypto_free_skcipher(ctx->fallback); } -static int p8_aes_xts_setkey(struct crypto_tfm *tfm, const u8 *key, +static int p8_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { + struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); int ret; - struct p8_aes_xts_ctx *ctx = crypto_tfm_ctx(tfm); - ret = xts_check_key(tfm, key, keylen); + ret = xts_verify_key(tfm, key, keylen); if (ret) return ret; @@ -81,100 +70,90 @@ static int p8_aes_xts_setkey(struct crypto_tfm *tfm, const u8 *key, pagefault_enable(); preempt_enable(); - ret |= crypto_sync_skcipher_setkey(ctx->fallback, key, keylen); + ret |= crypto_skcipher_setkey(ctx->fallback, key, keylen); return ret ? -EINVAL : 0; } -static int p8_aes_xts_crypt(struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, - unsigned int nbytes, int enc) +static int p8_aes_xts_crypt(struct skcipher_request *req, int enc) { - int ret; + struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); + const struct p8_aes_xts_ctx *ctx = crypto_skcipher_ctx(tfm); + struct skcipher_walk walk; + unsigned int nbytes; u8 tweak[AES_BLOCK_SIZE]; - u8 *iv; - struct blkcipher_walk walk; - struct p8_aes_xts_ctx *ctx = - crypto_tfm_ctx(crypto_blkcipher_tfm(desc->tfm)); + int ret; if (!crypto_simd_usable()) { - SYNC_SKCIPHER_REQUEST_ON_STACK(req, ctx->fallback); - skcipher_request_set_sync_tfm(req, ctx->fallback); - skcipher_request_set_callback(req, desc->flags, NULL, NULL); - skcipher_request_set_crypt(req, src, dst, nbytes, desc->info); - ret = enc? crypto_skcipher_encrypt(req) : crypto_skcipher_decrypt(req); - skcipher_request_zero(req); - } else { - blkcipher_walk_init(&walk, dst, src, nbytes); + struct skcipher_request *subreq = skcipher_request_ctx(req); + + *subreq = *req; + skcipher_request_set_tfm(subreq, ctx->fallback); + return enc ? crypto_skcipher_encrypt(subreq) : + crypto_skcipher_decrypt(subreq); + } + + ret = skcipher_walk_virt(&walk, req, false); + if (ret) + return ret; + + preempt_disable(); + pagefault_disable(); + enable_kernel_vsx(); - ret = blkcipher_walk_virt(desc, &walk); + aes_p8_encrypt(walk.iv, tweak, &ctx->tweak_key); + + disable_kernel_vsx(); + pagefault_enable(); + preempt_enable(); + while ((nbytes = walk.nbytes) != 0) { preempt_disable(); pagefault_disable(); enable_kernel_vsx(); - - iv = walk.iv; - memset(tweak, 0, AES_BLOCK_SIZE); - aes_p8_encrypt(iv, tweak, &ctx->tweak_key); - + if (enc) + aes_p8_xts_encrypt(walk.src.virt.addr, + walk.dst.virt.addr, + round_down(nbytes, AES_BLOCK_SIZE), + &ctx->enc_key, NULL, tweak); + else + aes_p8_xts_decrypt(walk.src.virt.addr, + walk.dst.virt.addr, + round_down(nbytes, AES_BLOCK_SIZE), + &ctx->dec_key, NULL, tweak); disable_kernel_vsx(); pagefault_enable(); preempt_enable(); - while ((nbytes = walk.nbytes)) { - preempt_disable(); - pagefault_disable(); - enable_kernel_vsx(); - if (enc) - aes_p8_xts_encrypt(walk.src.virt.addr, walk.dst.virt.addr, - nbytes & AES_BLOCK_MASK, &ctx->enc_key, NULL, tweak); - else - aes_p8_xts_decrypt(walk.src.virt.addr, walk.dst.virt.addr, - nbytes & AES_BLOCK_MASK, &ctx->dec_key, NULL, tweak); - disable_kernel_vsx(); - pagefault_enable(); - preempt_enable(); - - nbytes &= AES_BLOCK_SIZE - 1; - ret = blkcipher_walk_done(desc, &walk, nbytes); - } + ret = skcipher_walk_done(&walk, nbytes % AES_BLOCK_SIZE); } return ret; } -static int p8_aes_xts_encrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) +static int p8_aes_xts_encrypt(struct skcipher_request *req) { - return p8_aes_xts_crypt(desc, dst, src, nbytes, 1); + return p8_aes_xts_crypt(req, 1); } -static int p8_aes_xts_decrypt(struct blkcipher_desc *desc, - struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) +static int p8_aes_xts_decrypt(struct skcipher_request *req) { - return p8_aes_xts_crypt(desc, dst, src, nbytes, 0); + return p8_aes_xts_crypt(req, 0); } -struct crypto_alg p8_aes_xts_alg = { - .cra_name = "xts(aes)", - .cra_driver_name = "p8_aes_xts", - .cra_module = THIS_MODULE, - .cra_priority = 2000, - .cra_type = &crypto_blkcipher_type, - .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER | CRYPTO_ALG_NEED_FALLBACK, - .cra_alignmask = 0, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct p8_aes_xts_ctx), - .cra_init = p8_aes_xts_init, - .cra_exit = p8_aes_xts_exit, - .cra_blkcipher = { - .ivsize = AES_BLOCK_SIZE, - .min_keysize = 2 * AES_MIN_KEY_SIZE, - .max_keysize = 2 * AES_MAX_KEY_SIZE, - .setkey = p8_aes_xts_setkey, - .encrypt = p8_aes_xts_encrypt, - .decrypt = p8_aes_xts_decrypt, - } +struct skcipher_alg p8_aes_xts_alg = { + .base.cra_name = "xts(aes)", + .base.cra_driver_name = "p8_aes_xts", + .base.cra_module = THIS_MODULE, + .base.cra_priority = 2000, + .base.cra_flags = CRYPTO_ALG_NEED_FALLBACK, + .base.cra_blocksize = AES_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct p8_aes_xts_ctx), + .setkey = p8_aes_xts_setkey, + .encrypt = p8_aes_xts_encrypt, + .decrypt = p8_aes_xts_decrypt, + .init = p8_aes_xts_init, + .exit = p8_aes_xts_exit, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, }; diff --git a/drivers/crypto/vmx/aesp8-ppc.h b/drivers/crypto/vmx/aesp8-ppc.h index 349646b73754..01774a4d26a2 100644 --- a/drivers/crypto/vmx/aesp8-ppc.h +++ b/drivers/crypto/vmx/aesp8-ppc.h @@ -2,8 +2,6 @@ #include <linux/types.h> #include <crypto/aes.h> -#define AES_BLOCK_MASK (~(AES_BLOCK_SIZE-1)) - struct aes_key { u8 key[AES_MAX_KEYLENGTH]; int rounds; diff --git a/drivers/crypto/vmx/aesp8-ppc.pl b/drivers/crypto/vmx/aesp8-ppc.pl index 9c6b5c1d6a1a..db874367b602 100644 --- a/drivers/crypto/vmx/aesp8-ppc.pl +++ b/drivers/crypto/vmx/aesp8-ppc.pl @@ -1286,6 +1286,24 @@ ___ ######################################################################### {{{ # CTR procedure[s] # + +####################### WARNING: Here be dragons! ####################### +# +# This code is written as 'ctr32', based on a 32-bit counter used +# upstream. The kernel does *not* use a 32-bit counter. The kernel uses +# a 128-bit counter. +# +# This leads to subtle changes from the upstream code: the counter +# is incremented with vaddu_q_m rather than vaddu_w_m. This occurs in +# both the bulk (8 blocks at a time) path, and in the individual block +# path. Be aware of this when doing updates. +# +# See: +# 1d4aa0b4c181 ("crypto: vmx - Fixing AES-CTR counter bug") +# 009b30ac7444 ("crypto: vmx - CTR: always increment IV as quadword") +# https://github.com/openssl/openssl/pull/8942 +# +######################################################################### my ($inp,$out,$len,$key,$ivp,$x10,$rounds,$idx)=map("r$_",(3..10)); my ($rndkey0,$rndkey1,$inout,$tmp)= map("v$_",(0..3)); my ($ivec,$inptail,$inpperm,$outhead,$outperm,$outmask,$keyperm,$one)= @@ -1357,7 +1375,7 @@ Loop_ctr32_enc: addi $idx,$idx,16 bdnz Loop_ctr32_enc - vadduqm $ivec,$ivec,$one + vadduqm $ivec,$ivec,$one # Kernel change for 128-bit vmr $dat,$inptail lvx $inptail,0,$inp addi $inp,$inp,16 @@ -1501,7 +1519,7 @@ Load_ctr32_enc_key: $SHL $len,$len,4 vadduqm $out1,$ivec,$one # counter values ... - vadduqm $out2,$ivec,$two + vadduqm $out2,$ivec,$two # (do all ctr adds as 128-bit) vxor $out0,$ivec,$rndkey0 # ... xored with rndkey[0] le?li $idx,8 vadduqm $out3,$out1,$two diff --git a/drivers/crypto/vmx/vmx.c b/drivers/crypto/vmx/vmx.c index 6c4c77f4e159..3e0335fb406c 100644 --- a/drivers/crypto/vmx/vmx.c +++ b/drivers/crypto/vmx/vmx.c @@ -15,54 +15,58 @@ #include <linux/crypto.h> #include <asm/cputable.h> #include <crypto/internal/hash.h> +#include <crypto/internal/skcipher.h> extern struct shash_alg p8_ghash_alg; extern struct crypto_alg p8_aes_alg; -extern struct crypto_alg p8_aes_cbc_alg; -extern struct crypto_alg p8_aes_ctr_alg; -extern struct crypto_alg p8_aes_xts_alg; -static struct crypto_alg *algs[] = { - &p8_aes_alg, - &p8_aes_cbc_alg, - &p8_aes_ctr_alg, - &p8_aes_xts_alg, - NULL, -}; +extern struct skcipher_alg p8_aes_cbc_alg; +extern struct skcipher_alg p8_aes_ctr_alg; +extern struct skcipher_alg p8_aes_xts_alg; static int __init p8_init(void) { - int ret = 0; - struct crypto_alg **alg_it; + int ret; - for (alg_it = algs; *alg_it; alg_it++) { - ret = crypto_register_alg(*alg_it); - printk(KERN_INFO "crypto_register_alg '%s' = %d\n", - (*alg_it)->cra_name, ret); - if (ret) { - for (alg_it--; alg_it >= algs; alg_it--) - crypto_unregister_alg(*alg_it); - break; - } - } + ret = crypto_register_shash(&p8_ghash_alg); if (ret) - return ret; + goto err; - ret = crypto_register_shash(&p8_ghash_alg); - if (ret) { - for (alg_it = algs; *alg_it; alg_it++) - crypto_unregister_alg(*alg_it); - } + ret = crypto_register_alg(&p8_aes_alg); + if (ret) + goto err_unregister_ghash; + + ret = crypto_register_skcipher(&p8_aes_cbc_alg); + if (ret) + goto err_unregister_aes; + + ret = crypto_register_skcipher(&p8_aes_ctr_alg); + if (ret) + goto err_unregister_aes_cbc; + + ret = crypto_register_skcipher(&p8_aes_xts_alg); + if (ret) + goto err_unregister_aes_ctr; + + return 0; + +err_unregister_aes_ctr: + crypto_unregister_skcipher(&p8_aes_ctr_alg); +err_unregister_aes_cbc: + crypto_unregister_skcipher(&p8_aes_cbc_alg); +err_unregister_aes: + crypto_unregister_alg(&p8_aes_alg); +err_unregister_ghash: + crypto_unregister_shash(&p8_ghash_alg); +err: return ret; } static void __exit p8_exit(void) { - struct crypto_alg **alg_it; - - for (alg_it = algs; *alg_it; alg_it++) { - printk(KERN_INFO "Removing '%s'\n", (*alg_it)->cra_name); - crypto_unregister_alg(*alg_it); - } + crypto_unregister_skcipher(&p8_aes_xts_alg); + crypto_unregister_skcipher(&p8_aes_ctr_alg); + crypto_unregister_skcipher(&p8_aes_cbc_alg); + crypto_unregister_alg(&p8_aes_alg); crypto_unregister_shash(&p8_ghash_alg); } diff --git a/drivers/i2c/i2c-core-acpi.c b/drivers/i2c/i2c-core-acpi.c index d84095591e45..f86065e16772 100644 --- a/drivers/i2c/i2c-core-acpi.c +++ b/drivers/i2c/i2c-core-acpi.c @@ -111,8 +111,7 @@ static int i2c_acpi_do_lookup(struct acpi_device *adev, struct list_head resource_list; int ret; - if (acpi_bus_get_status(adev) || !adev->status.present || - acpi_device_enumerated(adev)) + if (acpi_bus_get_status(adev) || !adev->status.present) return -EINVAL; if (acpi_match_device_ids(adev, i2c_acpi_ignored_device_ids) == 0) @@ -147,6 +146,9 @@ static int i2c_acpi_get_info(struct acpi_device *adev, lookup.info = info; lookup.index = -1; + if (acpi_device_enumerated(adev)) + return -EINVAL; + ret = i2c_acpi_do_lookup(adev, &lookup); if (ret) return ret; diff --git a/drivers/net/ppp/Kconfig b/drivers/net/ppp/Kconfig index bf395df3bb37..1a2e2f7629f3 100644 --- a/drivers/net/ppp/Kconfig +++ b/drivers/net/ppp/Kconfig @@ -87,8 +87,7 @@ config PPP_MPPE depends on PPP select CRYPTO select CRYPTO_SHA1 - select CRYPTO_ARC4 - select CRYPTO_ECB + select CRYPTO_LIB_ARC4 ---help--- Support for the MPPE Encryption protocol, as employed by the Microsoft Point-to-Point Tunneling Protocol. diff --git a/drivers/net/ppp/ppp_mppe.c b/drivers/net/ppp/ppp_mppe.c index 66c8e65f6872..bd3c80b0bc77 100644 --- a/drivers/net/ppp/ppp_mppe.c +++ b/drivers/net/ppp/ppp_mppe.c @@ -42,9 +42,10 @@ * deprecated in 2.6 */ +#include <crypto/arc4.h> #include <crypto/hash.h> -#include <crypto/skcipher.h> #include <linux/err.h> +#include <linux/fips.h> #include <linux/module.h> #include <linux/kernel.h> #include <linux/init.h> @@ -66,13 +67,6 @@ MODULE_ALIAS("ppp-compress-" __stringify(CI_MPPE)); MODULE_SOFTDEP("pre: arc4"); MODULE_VERSION("1.0.2"); -static unsigned int -setup_sg(struct scatterlist *sg, const void *address, unsigned int length) -{ - sg_set_buf(sg, address, length); - return length; -} - #define SHA1_PAD_SIZE 40 /* @@ -96,7 +90,7 @@ static inline void sha_pad_init(struct sha_pad *shapad) * State for an MPPE (de)compressor. */ struct ppp_mppe_state { - struct crypto_sync_skcipher *arc4; + struct arc4_ctx arc4; struct shash_desc *sha1; unsigned char *sha1_digest; unsigned char master_key[MPPE_MAX_KEY_LEN]; @@ -155,24 +149,11 @@ static void get_new_key_from_sha(struct ppp_mppe_state * state) */ static void mppe_rekey(struct ppp_mppe_state * state, int initial_key) { - struct scatterlist sg_in[1], sg_out[1]; - SYNC_SKCIPHER_REQUEST_ON_STACK(req, state->arc4); - - skcipher_request_set_sync_tfm(req, state->arc4); - skcipher_request_set_callback(req, 0, NULL, NULL); - get_new_key_from_sha(state); if (!initial_key) { - crypto_sync_skcipher_setkey(state->arc4, state->sha1_digest, - state->keylen); - sg_init_table(sg_in, 1); - sg_init_table(sg_out, 1); - setup_sg(sg_in, state->sha1_digest, state->keylen); - setup_sg(sg_out, state->session_key, state->keylen); - skcipher_request_set_crypt(req, sg_in, sg_out, state->keylen, - NULL); - if (crypto_skcipher_encrypt(req)) - printk(KERN_WARNING "mppe_rekey: cipher_encrypt failed\n"); + arc4_setkey(&state->arc4, state->sha1_digest, state->keylen); + arc4_crypt(&state->arc4, state->session_key, state->sha1_digest, + state->keylen); } else { memcpy(state->session_key, state->sha1_digest, state->keylen); } @@ -182,9 +163,7 @@ static void mppe_rekey(struct ppp_mppe_state * state, int initial_key) state->session_key[1] = 0x26; state->session_key[2] = 0x9e; } - crypto_sync_skcipher_setkey(state->arc4, state->session_key, - state->keylen); - skcipher_request_zero(req); + arc4_setkey(&state->arc4, state->session_key, state->keylen); } /* @@ -197,7 +176,8 @@ static void *mppe_alloc(unsigned char *options, int optlen) unsigned int digestsize; if (optlen != CILEN_MPPE + sizeof(state->master_key) || - options[0] != CI_MPPE || options[1] != CILEN_MPPE) + options[0] != CI_MPPE || options[1] != CILEN_MPPE || + fips_enabled) goto out; state = kzalloc(sizeof(*state), GFP_KERNEL); @@ -205,12 +185,6 @@ static void *mppe_alloc(unsigned char *options, int optlen) goto out; - state->arc4 = crypto_alloc_sync_skcipher("ecb(arc4)", 0, 0); - if (IS_ERR(state->arc4)) { - state->arc4 = NULL; - goto out_free; - } - shash = crypto_alloc_shash("sha1", 0, 0); if (IS_ERR(shash)) goto out_free; @@ -251,7 +225,6 @@ out_free: crypto_free_shash(state->sha1->tfm); kzfree(state->sha1); } - crypto_free_sync_skcipher(state->arc4); kfree(state); out: return NULL; @@ -267,8 +240,7 @@ static void mppe_free(void *arg) kfree(state->sha1_digest); crypto_free_shash(state->sha1->tfm); kzfree(state->sha1); - crypto_free_sync_skcipher(state->arc4); - kfree(state); + kzfree(state); } } @@ -367,10 +339,7 @@ mppe_compress(void *arg, unsigned char *ibuf, unsigned char *obuf, int isize, int osize) { struct ppp_mppe_state *state = (struct ppp_mppe_state *) arg; - SYNC_SKCIPHER_REQUEST_ON_STACK(req, state->arc4); int proto; - int err; - struct scatterlist sg_in[1], sg_out[1]; /* * Check that the protocol is in the range we handle. @@ -421,21 +390,7 @@ mppe_compress(void *arg, unsigned char *ibuf, unsigned char *obuf, ibuf += 2; /* skip to proto field */ isize -= 2; - /* Encrypt packet */ - sg_init_table(sg_in, 1); - sg_init_table(sg_out, 1); - setup_sg(sg_in, ibuf, isize); - setup_sg(sg_out, obuf, osize); - - skcipher_request_set_sync_tfm(req, state->arc4); - skcipher_request_set_callback(req, 0, NULL, NULL); - skcipher_request_set_crypt(req, sg_in, sg_out, isize, NULL); - err = crypto_skcipher_encrypt(req); - skcipher_request_zero(req); - if (err) { - printk(KERN_DEBUG "crypto_cypher_encrypt failed\n"); - return -1; - } + arc4_crypt(&state->arc4, obuf, ibuf, isize); state->stats.unc_bytes += isize; state->stats.unc_packets++; @@ -481,10 +436,8 @@ mppe_decompress(void *arg, unsigned char *ibuf, int isize, unsigned char *obuf, int osize) { struct ppp_mppe_state *state = (struct ppp_mppe_state *) arg; - SYNC_SKCIPHER_REQUEST_ON_STACK(req, state->arc4); unsigned ccount; int flushed = MPPE_BITS(ibuf) & MPPE_BIT_FLUSHED; - struct scatterlist sg_in[1], sg_out[1]; if (isize <= PPP_HDRLEN + MPPE_OVHD) { if (state->debug) @@ -611,19 +564,7 @@ mppe_decompress(void *arg, unsigned char *ibuf, int isize, unsigned char *obuf, * Decrypt the first byte in order to check if it is * a compressed or uncompressed protocol field. */ - sg_init_table(sg_in, 1); - sg_init_table(sg_out, 1); - setup_sg(sg_in, ibuf, 1); - setup_sg(sg_out, obuf, 1); - - skcipher_request_set_sync_tfm(req, state->arc4); - skcipher_request_set_callback(req, 0, NULL, NULL); - skcipher_request_set_crypt(req, sg_in, sg_out, 1, NULL); - if (crypto_skcipher_decrypt(req)) { - printk(KERN_DEBUG "crypto_cypher_decrypt failed\n"); - osize = DECOMP_ERROR; - goto out_zap_req; - } + arc4_crypt(&state->arc4, obuf, ibuf, 1); /* * Do PFC decompression. @@ -638,14 +579,7 @@ mppe_decompress(void *arg, unsigned char *ibuf, int isize, unsigned char *obuf, } /* And finally, decrypt the rest of the packet. */ - setup_sg(sg_in, ibuf + 1, isize - 1); - setup_sg(sg_out, obuf + 1, osize - 1); - skcipher_request_set_crypt(req, sg_in, sg_out, isize - 1, NULL); - if (crypto_skcipher_decrypt(req)) { - printk(KERN_DEBUG "crypto_cypher_decrypt failed\n"); - osize = DECOMP_ERROR; - goto out_zap_req; - } + arc4_crypt(&state->arc4, obuf + 1, ibuf + 1, isize - 1); state->stats.unc_bytes += osize; state->stats.unc_packets++; @@ -655,8 +589,6 @@ mppe_decompress(void *arg, unsigned char *ibuf, int isize, unsigned char *obuf, /* good packet credit */ state->sanity_errors >>= 1; -out_zap_req: - skcipher_request_zero(req); return osize; sanity_error: @@ -729,8 +661,7 @@ static struct compressor ppp_mppe = { static int __init ppp_mppe_init(void) { int answer; - if (!(crypto_has_skcipher("ecb(arc4)", 0, CRYPTO_ALG_ASYNC) && - crypto_has_ahash("sha1", 0, CRYPTO_ALG_ASYNC))) + if (fips_enabled || !crypto_has_ahash("sha1", 0, CRYPTO_ALG_ASYNC)) return -ENODEV; sha_pad = kmalloc(sizeof(struct sha_pad), GFP_KERNEL); diff --git a/fs/cifs/Kconfig b/fs/cifs/Kconfig index aae2b8b2adf5..523e9ea78a28 100644 --- a/fs/cifs/Kconfig +++ b/fs/cifs/Kconfig @@ -10,7 +10,7 @@ config CIFS select CRYPTO_SHA512 select CRYPTO_CMAC select CRYPTO_HMAC - select CRYPTO_ARC4 + select CRYPTO_LIB_ARC4 select CRYPTO_AEAD2 select CRYPTO_CCM select CRYPTO_ECB diff --git a/fs/cifs/cifsencrypt.c b/fs/cifs/cifsencrypt.c index d2a05e46d6f5..97b7497c13ef 100644 --- a/fs/cifs/cifsencrypt.c +++ b/fs/cifs/cifsencrypt.c @@ -33,7 +33,8 @@ #include <linux/ctype.h> #include <linux/random.h> #include <linux/highmem.h> -#include <crypto/skcipher.h> +#include <linux/fips.h> +#include <crypto/arc4.h> #include <crypto/aead.h> int __cifs_calc_signature(struct smb_rqst *rqst, @@ -772,63 +773,32 @@ setup_ntlmv2_rsp_ret: int calc_seckey(struct cifs_ses *ses) { - int rc; - struct crypto_skcipher *tfm_arc4; - struct scatterlist sgin, sgout; - struct skcipher_request *req; - unsigned char *sec_key; + unsigned char sec_key[CIFS_SESS_KEY_SIZE]; /* a nonce */ + struct arc4_ctx *ctx_arc4; - sec_key = kmalloc(CIFS_SESS_KEY_SIZE, GFP_KERNEL); - if (sec_key == NULL) - return -ENOMEM; + if (fips_enabled) + return -ENODEV; get_random_bytes(sec_key, CIFS_SESS_KEY_SIZE); - tfm_arc4 = crypto_alloc_skcipher("ecb(arc4)", 0, CRYPTO_ALG_ASYNC); - if (IS_ERR(tfm_arc4)) { - rc = PTR_ERR(tfm_arc4); - cifs_dbg(VFS, "could not allocate crypto API arc4\n"); - goto out; - } - - rc = crypto_skcipher_setkey(tfm_arc4, ses->auth_key.response, - CIFS_SESS_KEY_SIZE); - if (rc) { - cifs_dbg(VFS, "%s: Could not set response as a key\n", - __func__); - goto out_free_cipher; - } - - req = skcipher_request_alloc(tfm_arc4, GFP_KERNEL); - if (!req) { - rc = -ENOMEM; - cifs_dbg(VFS, "could not allocate crypto API arc4 request\n"); - goto out_free_cipher; + ctx_arc4 = kmalloc(sizeof(*ctx_arc4), GFP_KERNEL); + if (!ctx_arc4) { + cifs_dbg(VFS, "could not allocate arc4 context\n"); + return -ENOMEM; } - sg_init_one(&sgin, sec_key, CIFS_SESS_KEY_SIZE); - sg_init_one(&sgout, ses->ntlmssp->ciphertext, CIFS_CPHTXT_SIZE); - - skcipher_request_set_callback(req, 0, NULL, NULL); - skcipher_request_set_crypt(req, &sgin, &sgout, CIFS_CPHTXT_SIZE, NULL); - - rc = crypto_skcipher_encrypt(req); - skcipher_request_free(req); - if (rc) { - cifs_dbg(VFS, "could not encrypt session key rc: %d\n", rc); - goto out_free_cipher; - } + arc4_setkey(ctx_arc4, ses->auth_key.response, CIFS_SESS_KEY_SIZE); + arc4_crypt(ctx_arc4, ses->ntlmssp->ciphertext, sec_key, + CIFS_CPHTXT_SIZE); /* make secondary_key/nonce as session key */ memcpy(ses->auth_key.response, sec_key, CIFS_SESS_KEY_SIZE); /* and make len as that of session key only */ ses->auth_key.len = CIFS_SESS_KEY_SIZE; -out_free_cipher: - crypto_free_skcipher(tfm_arc4); -out: - kfree(sec_key); - return rc; + memzero_explicit(sec_key, CIFS_SESS_KEY_SIZE); + kzfree(ctx_arc4); + return 0; } void diff --git a/fs/cifs/cifsfs.c b/fs/cifs/cifsfs.c index 65d9771e49f9..72db1c89bf5a 100644 --- a/fs/cifs/cifsfs.c +++ b/fs/cifs/cifsfs.c @@ -1591,7 +1591,6 @@ MODULE_DESCRIPTION ("VFS to access SMB3 servers e.g. Samba, Macs, Azure and Windows (and " "also older servers complying with the SNIA CIFS Specification)"); MODULE_VERSION(CIFS_VERSION); -MODULE_SOFTDEP("pre: arc4"); MODULE_SOFTDEP("pre: des"); MODULE_SOFTDEP("pre: ecb"); MODULE_SOFTDEP("pre: hmac"); diff --git a/include/crypto/aead.h b/include/crypto/aead.h index 61bb10490492..3c245b1859e7 100644 --- a/include/crypto/aead.h +++ b/include/crypto/aead.h @@ -317,21 +317,7 @@ static inline struct crypto_aead *crypto_aead_reqtfm(struct aead_request *req) * * Return: 0 if the cipher operation was successful; < 0 if an error occurred */ -static inline int crypto_aead_encrypt(struct aead_request *req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct crypto_alg *alg = aead->base.__crt_alg; - unsigned int cryptlen = req->cryptlen; - int ret; - - crypto_stats_get(alg); - if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY) - ret = -ENOKEY; - else - ret = crypto_aead_alg(aead)->encrypt(req); - crypto_stats_aead_encrypt(cryptlen, alg, ret); - return ret; -} +int crypto_aead_encrypt(struct aead_request *req); /** * crypto_aead_decrypt() - decrypt ciphertext @@ -355,23 +341,7 @@ static inline int crypto_aead_encrypt(struct aead_request *req) * integrity of the ciphertext or the associated data was violated); * < 0 if an error occurred. */ -static inline int crypto_aead_decrypt(struct aead_request *req) -{ - struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct crypto_alg *alg = aead->base.__crt_alg; - unsigned int cryptlen = req->cryptlen; - int ret; - - crypto_stats_get(alg); - if (crypto_aead_get_flags(aead) & CRYPTO_TFM_NEED_KEY) - ret = -ENOKEY; - else if (req->cryptlen < crypto_aead_authsize(aead)) - ret = -EINVAL; - else - ret = crypto_aead_alg(aead)->decrypt(req); - crypto_stats_aead_decrypt(cryptlen, alg, ret); - return ret; -} +int crypto_aead_decrypt(struct aead_request *req); /** * DOC: Asynchronous AEAD Request Handle diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index 743d626479ef..dc1106af95c3 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -189,7 +189,6 @@ void crypto_init_queue(struct crypto_queue *queue, unsigned int max_qlen); int crypto_enqueue_request(struct crypto_queue *queue, struct crypto_async_request *request); struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue); -int crypto_tfm_in_queue(struct crypto_queue *queue, struct crypto_tfm *tfm); static inline unsigned int crypto_queue_len(struct crypto_queue *queue) { return queue->qlen; @@ -371,12 +370,6 @@ static inline void *ablkcipher_request_ctx(struct ablkcipher_request *req) return req->__ctx; } -static inline int ablkcipher_tfm_in_queue(struct crypto_queue *queue, - struct crypto_ablkcipher *tfm) -{ - return crypto_tfm_in_queue(queue, crypto_ablkcipher_tfm(tfm)); -} - static inline struct crypto_alg *crypto_get_attr_alg(struct rtattr **tb, u32 type, u32 mask) { diff --git a/include/crypto/arc4.h b/include/crypto/arc4.h index 5b2c24ab0139..f3c22fe01704 100644 --- a/include/crypto/arc4.h +++ b/include/crypto/arc4.h @@ -6,8 +6,18 @@ #ifndef _CRYPTO_ARC4_H #define _CRYPTO_ARC4_H +#include <linux/types.h> + #define ARC4_MIN_KEY_SIZE 1 #define ARC4_MAX_KEY_SIZE 256 #define ARC4_BLOCK_SIZE 1 +struct arc4_ctx { + u32 S[256]; + u32 x, y; +}; + +int arc4_setkey(struct arc4_ctx *ctx, const u8 *in_key, unsigned int key_len); +void arc4_crypt(struct arc4_ctx *ctx, u8 *out, const u8 *in, unsigned int len); + #endif /* _CRYPTO_ARC4_H */ diff --git a/include/crypto/chacha.h b/include/crypto/chacha.h index 1fc70a69d550..d1e723c6a37d 100644 --- a/include/crypto/chacha.h +++ b/include/crypto/chacha.h @@ -41,7 +41,7 @@ static inline void chacha20_block(u32 *state, u8 *stream) } void hchacha_block(const u32 *in, u32 *out, int nrounds); -void crypto_chacha_init(u32 *state, struct chacha_ctx *ctx, u8 *iv); +void crypto_chacha_init(u32 *state, const struct chacha_ctx *ctx, const u8 *iv); int crypto_chacha20_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keysize); diff --git a/include/crypto/crypto_wq.h b/include/crypto/crypto_wq.h deleted file mode 100644 index 23114746ac08..000000000000 --- a/include/crypto/crypto_wq.h +++ /dev/null @@ -1,8 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef CRYPTO_WQ_H -#define CRYPTO_WQ_H - -#include <linux/workqueue.h> - -extern struct workqueue_struct *kcrypto_wq; -#endif diff --git a/include/crypto/drbg.h b/include/crypto/drbg.h index 3fb581bf3b87..8c9af21efce1 100644 --- a/include/crypto/drbg.h +++ b/include/crypto/drbg.h @@ -129,6 +129,8 @@ struct drbg_state { bool seeded; /* DRBG fully seeded? */ bool pr; /* Prediction resistance enabled? */ + bool fips_primed; /* Continuous test primed? */ + unsigned char *prev; /* FIPS 140-2 continuous test value */ struct work_struct seed_work; /* asynchronous seeding support */ struct crypto_rng *jent; const struct drbg_state_ops *d_ops; diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index 31e0662fa429..bfc9db7b100d 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -196,12 +196,6 @@ static inline struct ahash_request *ahash_dequeue_request( return ahash_request_cast(crypto_dequeue_request(queue)); } -static inline int ahash_tfm_in_queue(struct crypto_queue *queue, - struct crypto_ahash *tfm) -{ - return crypto_tfm_in_queue(queue, crypto_ahash_tfm(tfm)); -} - static inline void *crypto_shash_ctx(struct crypto_shash *tfm) { return crypto_tfm_ctx(&tfm->base); diff --git a/include/crypto/internal/skcipher.h b/include/crypto/internal/skcipher.h index fe0376d5a471..d68faa5759ad 100644 --- a/include/crypto/internal/skcipher.h +++ b/include/crypto/internal/skcipher.h @@ -200,6 +200,66 @@ static inline unsigned int crypto_skcipher_alg_max_keysize( return alg->max_keysize; } +static inline unsigned int crypto_skcipher_alg_chunksize( + struct skcipher_alg *alg) +{ + if ((alg->base.cra_flags & CRYPTO_ALG_TYPE_MASK) == + CRYPTO_ALG_TYPE_BLKCIPHER) + return alg->base.cra_blocksize; + + if (alg->base.cra_ablkcipher.encrypt) + return alg->base.cra_blocksize; + + return alg->chunksize; +} + +static inline unsigned int crypto_skcipher_alg_walksize( + struct skcipher_alg *alg) +{ + if ((alg->base.cra_flags & CRYPTO_ALG_TYPE_MASK) == + CRYPTO_ALG_TYPE_BLKCIPHER) + return alg->base.cra_blocksize; + + if (alg->base.cra_ablkcipher.encrypt) + return alg->base.cra_blocksize; + + return alg->walksize; +} + +/** + * crypto_skcipher_chunksize() - obtain chunk size + * @tfm: cipher handle + * + * The block size is set to one for ciphers such as CTR. However, + * you still need to provide incremental updates in multiples of + * the underlying block size as the IV does not have sub-block + * granularity. This is known in this API as the chunk size. + * + * Return: chunk size in bytes + */ +static inline unsigned int crypto_skcipher_chunksize( + struct crypto_skcipher *tfm) +{ + return crypto_skcipher_alg_chunksize(crypto_skcipher_alg(tfm)); +} + +/** + * crypto_skcipher_walksize() - obtain walk size + * @tfm: cipher handle + * + * In some cases, algorithms can only perform optimally when operating on + * multiple blocks in parallel. This is reflected by the walksize, which + * must be a multiple of the chunksize (or equal if the concern does not + * apply) + * + * Return: walk size in bytes + */ +static inline unsigned int crypto_skcipher_walksize( + struct crypto_skcipher *tfm) +{ + return crypto_skcipher_alg_walksize(crypto_skcipher_alg(tfm)); +} + /* Helpers for simple block cipher modes of operation */ struct skcipher_ctx_simple { struct crypto_cipher *cipher; /* underlying block cipher */ diff --git a/include/crypto/skcipher.h b/include/crypto/skcipher.h index ce7fa0973580..37c164234d97 100644 --- a/include/crypto/skcipher.h +++ b/include/crypto/skcipher.h @@ -288,66 +288,6 @@ static inline unsigned int crypto_sync_skcipher_ivsize( return crypto_skcipher_ivsize(&tfm->base); } -static inline unsigned int crypto_skcipher_alg_chunksize( - struct skcipher_alg *alg) -{ - if ((alg->base.cra_flags & CRYPTO_ALG_TYPE_MASK) == - CRYPTO_ALG_TYPE_BLKCIPHER) - return alg->base.cra_blocksize; - - if (alg->base.cra_ablkcipher.encrypt) - return alg->base.cra_blocksize; - - return alg->chunksize; -} - -static inline unsigned int crypto_skcipher_alg_walksize( - struct skcipher_alg *alg) -{ - if ((alg->base.cra_flags & CRYPTO_ALG_TYPE_MASK) == - CRYPTO_ALG_TYPE_BLKCIPHER) - return alg->base.cra_blocksize; - - if (alg->base.cra_ablkcipher.encrypt) - return alg->base.cra_blocksize; - - return alg->walksize; -} - -/** - * crypto_skcipher_chunksize() - obtain chunk size - * @tfm: cipher handle - * - * The block size is set to one for ciphers such as CTR. However, - * you still need to provide incremental updates in multiples of - * the underlying block size as the IV does not have sub-block - * granularity. This is known in this API as the chunk size. - * - * Return: chunk size in bytes - */ -static inline unsigned int crypto_skcipher_chunksize( - struct crypto_skcipher *tfm) -{ - return crypto_skcipher_alg_chunksize(crypto_skcipher_alg(tfm)); -} - -/** - * crypto_skcipher_walksize() - obtain walk size - * @tfm: cipher handle - * - * In some cases, algorithms can only perform optimally when operating on - * multiple blocks in parallel. This is reflected by the walksize, which - * must be a multiple of the chunksize (or equal if the concern does not - * apply) - * - * Return: walk size in bytes - */ -static inline unsigned int crypto_skcipher_walksize( - struct crypto_skcipher *tfm) -{ - return crypto_skcipher_alg_walksize(crypto_skcipher_alg(tfm)); -} - /** * crypto_skcipher_blocksize() - obtain block size of cipher * @tfm: cipher handle @@ -479,21 +419,7 @@ static inline struct crypto_sync_skcipher *crypto_sync_skcipher_reqtfm( * * Return: 0 if the cipher operation was successful; < 0 if an error occurred */ -static inline int crypto_skcipher_encrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct crypto_alg *alg = tfm->base.__crt_alg; - unsigned int cryptlen = req->cryptlen; - int ret; - - crypto_stats_get(alg); - if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) - ret = -ENOKEY; - else - ret = tfm->encrypt(req); - crypto_stats_skcipher_encrypt(cryptlen, ret, alg); - return ret; -} +int crypto_skcipher_encrypt(struct skcipher_request *req); /** * crypto_skcipher_decrypt() - decrypt ciphertext @@ -506,21 +432,7 @@ static inline int crypto_skcipher_encrypt(struct skcipher_request *req) * * Return: 0 if the cipher operation was successful; < 0 if an error occurred */ -static inline int crypto_skcipher_decrypt(struct skcipher_request *req) -{ - struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req); - struct crypto_alg *alg = tfm->base.__crt_alg; - unsigned int cryptlen = req->cryptlen; - int ret; - - crypto_stats_get(alg); - if (crypto_skcipher_get_flags(tfm) & CRYPTO_TFM_NEED_KEY) - ret = -ENOKEY; - else - ret = tfm->decrypt(req); - crypto_stats_skcipher_decrypt(cryptlen, ret, alg); - return ret; -} +int crypto_skcipher_decrypt(struct skcipher_request *req); /** * DOC: Symmetric Key Cipher Request Handle diff --git a/include/linux/crypto.h b/include/linux/crypto.h index 9cf8f3ce0e50..19ea3a371d7b 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -49,7 +49,6 @@ #define CRYPTO_ALG_TYPE_SCOMPRESS 0x0000000b #define CRYPTO_ALG_TYPE_RNG 0x0000000c #define CRYPTO_ALG_TYPE_AKCIPHER 0x0000000d -#define CRYPTO_ALG_TYPE_DIGEST 0x0000000e #define CRYPTO_ALG_TYPE_HASH 0x0000000e #define CRYPTO_ALG_TYPE_SHASH 0x0000000e #define CRYPTO_ALG_TYPE_AHASH 0x0000000f @@ -323,6 +322,17 @@ struct cipher_alg { void (*cia_decrypt)(struct crypto_tfm *tfm, u8 *dst, const u8 *src); }; +/** + * struct compress_alg - compression/decompression algorithm + * @coa_compress: Compress a buffer of specified length, storing the resulting + * data in the specified buffer. Return the length of the + * compressed data in dlen. + * @coa_decompress: Decompress the source buffer, storing the uncompressed + * data in the specified buffer. The length of the data is + * returned in dlen. + * + * All fields are mandatory. + */ struct compress_alg { int (*coa_compress)(struct crypto_tfm *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen); diff --git a/lib/Makefile b/lib/Makefile index fb7697031a79..d3daedf93c5a 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -102,7 +102,7 @@ endif obj-$(CONFIG_DEBUG_INFO_REDUCED) += debug_info.o CFLAGS_debug_info.o += $(call cc-option, -femit-struct-debug-detailed=any) -obj-y += math/ +obj-y += math/ crypto/ obj-$(CONFIG_GENERIC_IOMAP) += iomap.o obj-$(CONFIG_GENERIC_PCI_IOMAP) += pci_iomap.o diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile new file mode 100644 index 000000000000..88195c34932d --- /dev/null +++ b/lib/crypto/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_CRYPTO_LIB_ARC4) += libarc4.o +libarc4-y := arc4.o diff --git a/lib/crypto/arc4.c b/lib/crypto/arc4.c new file mode 100644 index 000000000000..c2020f19c652 --- /dev/null +++ b/lib/crypto/arc4.c @@ -0,0 +1,74 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Cryptographic API + * + * ARC4 Cipher Algorithm + * + * Jon Oberheide <jon@oberheide.org> + */ + +#include <crypto/arc4.h> +#include <linux/module.h> + +int arc4_setkey(struct arc4_ctx *ctx, const u8 *in_key, unsigned int key_len) +{ + int i, j = 0, k = 0; + + ctx->x = 1; + ctx->y = 0; + + for (i = 0; i < 256; i++) + ctx->S[i] = i; + + for (i = 0; i < 256; i++) { + u32 a = ctx->S[i]; + + j = (j + in_key[k] + a) & 0xff; + ctx->S[i] = ctx->S[j]; + ctx->S[j] = a; + if (++k >= key_len) + k = 0; + } + + return 0; +} +EXPORT_SYMBOL(arc4_setkey); + +void arc4_crypt(struct arc4_ctx *ctx, u8 *out, const u8 *in, unsigned int len) +{ + u32 *const S = ctx->S; + u32 x, y, a, b; + u32 ty, ta, tb; + + if (len == 0) + return; + + x = ctx->x; + y = ctx->y; + + a = S[x]; + y = (y + a) & 0xff; + b = S[y]; + + do { + S[y] = a; + a = (a + b) & 0xff; + S[x] = b; + x = (x + 1) & 0xff; + ta = S[x]; + ty = (y + ta) & 0xff; + tb = S[ty]; + *out++ = *in++ ^ S[a]; + if (--len == 0) + break; + y = ty; + a = ta; + b = tb; + } while (true); + + ctx->x = x; + ctx->y = y; +} +EXPORT_SYMBOL(arc4_crypt); + +MODULE_LICENSE("GPL"); diff --git a/lib/scatterlist.c b/lib/scatterlist.c index 2882d9ba6607..eacb82468437 100644 --- a/lib/scatterlist.c +++ b/lib/scatterlist.c @@ -676,17 +676,18 @@ static bool sg_miter_get_next_page(struct sg_mapping_iter *miter) { if (!miter->__remaining) { struct scatterlist *sg; - unsigned long pgoffset; if (!__sg_page_iter_next(&miter->piter)) return false; sg = miter->piter.sg; - pgoffset = miter->piter.sg_pgoffset; - miter->__offset = pgoffset ? 0 : sg->offset; + miter->__offset = miter->piter.sg_pgoffset ? 0 : sg->offset; + miter->piter.sg_pgoffset += miter->__offset >> PAGE_SHIFT; + miter->__offset &= PAGE_SIZE - 1; miter->__remaining = sg->offset + sg->length - - (pgoffset << PAGE_SHIFT) - miter->__offset; + (miter->piter.sg_pgoffset << PAGE_SHIFT) - + miter->__offset; miter->__remaining = min_t(unsigned long, miter->__remaining, PAGE_SIZE - miter->__offset); } diff --git a/net/mac80211/Kconfig b/net/mac80211/Kconfig index 0227cce9685e..0c93b1b7a826 100644 --- a/net/mac80211/Kconfig +++ b/net/mac80211/Kconfig @@ -3,7 +3,7 @@ config MAC80211 tristate "Generic IEEE 802.11 Networking Stack (mac80211)" depends on CFG80211 select CRYPTO - select CRYPTO_ARC4 + select CRYPTO_LIB_ARC4 select CRYPTO_AES select CRYPTO_CCM select CRYPTO_GCM diff --git a/net/mac80211/cfg.c b/net/mac80211/cfg.c index a1973a26c7fc..3fae902937fd 100644 --- a/net/mac80211/cfg.c +++ b/net/mac80211/cfg.c @@ -14,6 +14,7 @@ #include <linux/slab.h> #include <net/net_namespace.h> #include <linux/rcupdate.h> +#include <linux/fips.h> #include <linux/if_ether.h> #include <net/cfg80211.h> #include "ieee80211_i.h" @@ -402,9 +403,8 @@ static int ieee80211_add_key(struct wiphy *wiphy, struct net_device *dev, case WLAN_CIPHER_SUITE_WEP40: case WLAN_CIPHER_SUITE_TKIP: case WLAN_CIPHER_SUITE_WEP104: - if (IS_ERR(local->wep_tx_tfm)) + if (WARN_ON_ONCE(fips_enabled)) return -EINVAL; - break; case WLAN_CIPHER_SUITE_CCMP: case WLAN_CIPHER_SUITE_CCMP_256: case WLAN_CIPHER_SUITE_AES_CMAC: diff --git a/net/mac80211/ieee80211_i.h b/net/mac80211/ieee80211_i.h index 6396d46a9a71..004e2e3adb88 100644 --- a/net/mac80211/ieee80211_i.h +++ b/net/mac80211/ieee80211_i.h @@ -1255,8 +1255,8 @@ struct ieee80211_local { struct rate_control_ref *rate_ctrl; - struct crypto_cipher *wep_tx_tfm; - struct crypto_cipher *wep_rx_tfm; + struct arc4_ctx wep_tx_ctx; + struct arc4_ctx wep_rx_ctx; u32 wep_iv; /* see iface.c */ diff --git a/net/mac80211/key.h b/net/mac80211/key.h index be118c39433f..b8b9cd743bf4 100644 --- a/net/mac80211/key.h +++ b/net/mac80211/key.h @@ -11,6 +11,7 @@ #include <linux/list.h> #include <linux/crypto.h> #include <linux/rcupdate.h> +#include <crypto/arc4.h> #include <net/mac80211.h> #define NUM_DEFAULT_KEYS 4 diff --git a/net/mac80211/main.c b/net/mac80211/main.c index 55583b71ffaf..087c90e461b5 100644 --- a/net/mac80211/main.c +++ b/net/mac80211/main.c @@ -10,6 +10,7 @@ #include <net/mac80211.h> #include <linux/module.h> +#include <linux/fips.h> #include <linux/init.h> #include <linux/netdevice.h> #include <linux/types.h> @@ -730,8 +731,7 @@ EXPORT_SYMBOL(ieee80211_alloc_hw_nm); static int ieee80211_init_cipher_suites(struct ieee80211_local *local) { - bool have_wep = !(IS_ERR(local->wep_tx_tfm) || - IS_ERR(local->wep_rx_tfm)); + bool have_wep = !fips_enabled; /* FIPS does not permit the use of RC4 */ bool have_mfp = ieee80211_hw_check(&local->hw, MFP_CAPABLE); int n_suites = 0, r = 0, w = 0; u32 *suites; @@ -1298,7 +1298,6 @@ int ieee80211_register_hw(struct ieee80211_hw *hw) fail_rate: rtnl_unlock(); ieee80211_led_exit(local); - ieee80211_wep_free(local); fail_flows: destroy_workqueue(local->workqueue); fail_workqueue: @@ -1355,7 +1354,6 @@ void ieee80211_unregister_hw(struct ieee80211_hw *hw) destroy_workqueue(local->workqueue); wiphy_unregister(local->hw.wiphy); - ieee80211_wep_free(local); ieee80211_led_exit(local); kfree(local->int_scan_req); } diff --git a/net/mac80211/mlme.c b/net/mac80211/mlme.c index 379d2ab6d327..a00b703bfdfd 100644 --- a/net/mac80211/mlme.c +++ b/net/mac80211/mlme.c @@ -12,6 +12,7 @@ */ #include <linux/delay.h> +#include <linux/fips.h> #include <linux/if_ether.h> #include <linux/skbuff.h> #include <linux/if_arp.h> @@ -5045,7 +5046,7 @@ int ieee80211_mgd_auth(struct ieee80211_sub_if_data *sdata, auth_alg = WLAN_AUTH_OPEN; break; case NL80211_AUTHTYPE_SHARED_KEY: - if (IS_ERR(local->wep_tx_tfm)) + if (fips_enabled) return -EOPNOTSUPP; auth_alg = WLAN_AUTH_SHARED_KEY; break; diff --git a/net/mac80211/tkip.c b/net/mac80211/tkip.c index 7914b8e3ce8c..727dc9f3f3b3 100644 --- a/net/mac80211/tkip.c +++ b/net/mac80211/tkip.c @@ -219,7 +219,7 @@ EXPORT_SYMBOL(ieee80211_get_tkip_p2k); * @payload_len is the length of payload (_not_ including IV/ICV length). * @ta is the transmitter addresses. */ -int ieee80211_tkip_encrypt_data(struct crypto_cipher *tfm, +int ieee80211_tkip_encrypt_data(struct arc4_ctx *ctx, struct ieee80211_key *key, struct sk_buff *skb, u8 *payload, size_t payload_len) @@ -228,7 +228,7 @@ int ieee80211_tkip_encrypt_data(struct crypto_cipher *tfm, ieee80211_get_tkip_p2k(&key->conf, skb, rc4key); - return ieee80211_wep_encrypt_data(tfm, rc4key, 16, + return ieee80211_wep_encrypt_data(ctx, rc4key, 16, payload, payload_len); } @@ -236,7 +236,7 @@ int ieee80211_tkip_encrypt_data(struct crypto_cipher *tfm, * beginning of the buffer containing IEEE 802.11 header payload, i.e., * including IV, Ext. IV, real data, Michael MIC, ICV. @payload_len is the * length of payload, including IV, Ext. IV, MIC, ICV. */ -int ieee80211_tkip_decrypt_data(struct crypto_cipher *tfm, +int ieee80211_tkip_decrypt_data(struct arc4_ctx *ctx, struct ieee80211_key *key, u8 *payload, size_t payload_len, u8 *ta, u8 *ra, int only_iv, int queue, @@ -294,7 +294,7 @@ int ieee80211_tkip_decrypt_data(struct crypto_cipher *tfm, tkip_mixing_phase2(tk, &rx_ctx->ctx, iv16, rc4key); - res = ieee80211_wep_decrypt_data(tfm, rc4key, 16, pos, payload_len - 12); + res = ieee80211_wep_decrypt_data(ctx, rc4key, 16, pos, payload_len - 12); done: if (res == TKIP_DECRYPT_OK) { /* diff --git a/net/mac80211/tkip.h b/net/mac80211/tkip.h index 676a7babdf5d..9d2f8bd36cc7 100644 --- a/net/mac80211/tkip.h +++ b/net/mac80211/tkip.h @@ -10,7 +10,7 @@ #include <linux/crypto.h> #include "key.h" -int ieee80211_tkip_encrypt_data(struct crypto_cipher *tfm, +int ieee80211_tkip_encrypt_data(struct arc4_ctx *ctx, struct ieee80211_key *key, struct sk_buff *skb, u8 *payload, size_t payload_len); @@ -21,7 +21,7 @@ enum { TKIP_DECRYPT_INVALID_KEYIDX = -2, TKIP_DECRYPT_REPLAY = -3, }; -int ieee80211_tkip_decrypt_data(struct crypto_cipher *tfm, +int ieee80211_tkip_decrypt_data(struct arc4_ctx *ctx, struct ieee80211_key *key, u8 *payload, size_t payload_len, u8 *ta, u8 *ra, int only_iv, int queue, diff --git a/net/mac80211/wep.c b/net/mac80211/wep.c index 3d9e92867ef0..b75c2c54e665 100644 --- a/net/mac80211/wep.c +++ b/net/mac80211/wep.c @@ -27,30 +27,9 @@ int ieee80211_wep_init(struct ieee80211_local *local) /* start WEP IV from a random value */ get_random_bytes(&local->wep_iv, IEEE80211_WEP_IV_LEN); - local->wep_tx_tfm = crypto_alloc_cipher("arc4", 0, 0); - if (IS_ERR(local->wep_tx_tfm)) { - local->wep_rx_tfm = ERR_PTR(-EINVAL); - return PTR_ERR(local->wep_tx_tfm); - } - - local->wep_rx_tfm = crypto_alloc_cipher("arc4", 0, 0); - if (IS_ERR(local->wep_rx_tfm)) { - crypto_free_cipher(local->wep_tx_tfm); - local->wep_tx_tfm = ERR_PTR(-EINVAL); - return PTR_ERR(local->wep_rx_tfm); - } - return 0; } -void ieee80211_wep_free(struct ieee80211_local *local) -{ - if (!IS_ERR(local->wep_tx_tfm)) - crypto_free_cipher(local->wep_tx_tfm); - if (!IS_ERR(local->wep_rx_tfm)) - crypto_free_cipher(local->wep_rx_tfm); -} - static inline bool ieee80211_wep_weak_iv(u32 iv, int keylen) { /* @@ -128,21 +107,17 @@ static void ieee80211_wep_remove_iv(struct ieee80211_local *local, /* Perform WEP encryption using given key. data buffer must have tailroom * for 4-byte ICV. data_len must not include this ICV. Note: this function * does _not_ add IV. data = RC4(data | CRC32(data)) */ -int ieee80211_wep_encrypt_data(struct crypto_cipher *tfm, u8 *rc4key, +int ieee80211_wep_encrypt_data(struct arc4_ctx *ctx, u8 *rc4key, size_t klen, u8 *data, size_t data_len) { __le32 icv; - int i; - - if (IS_ERR(tfm)) - return -1; icv = cpu_to_le32(~crc32_le(~0, data, data_len)); put_unaligned(icv, (__le32 *)(data + data_len)); - crypto_cipher_setkey(tfm, rc4key, klen); - for (i = 0; i < data_len + IEEE80211_WEP_ICV_LEN; i++) - crypto_cipher_encrypt_one(tfm, data + i, data + i); + arc4_setkey(ctx, rc4key, klen); + arc4_crypt(ctx, data, data, data_len + IEEE80211_WEP_ICV_LEN); + memzero_explicit(ctx, sizeof(*ctx)); return 0; } @@ -181,7 +156,7 @@ int ieee80211_wep_encrypt(struct ieee80211_local *local, /* Add room for ICV */ skb_put(skb, IEEE80211_WEP_ICV_LEN); - return ieee80211_wep_encrypt_data(local->wep_tx_tfm, rc4key, keylen + 3, + return ieee80211_wep_encrypt_data(&local->wep_tx_ctx, rc4key, keylen + 3, iv + IEEE80211_WEP_IV_LEN, len); } @@ -189,18 +164,14 @@ int ieee80211_wep_encrypt(struct ieee80211_local *local, /* Perform WEP decryption using given key. data buffer includes encrypted * payload, including 4-byte ICV, but _not_ IV. data_len must not include ICV. * Return 0 on success and -1 on ICV mismatch. */ -int ieee80211_wep_decrypt_data(struct crypto_cipher *tfm, u8 *rc4key, +int ieee80211_wep_decrypt_data(struct arc4_ctx *ctx, u8 *rc4key, size_t klen, u8 *data, size_t data_len) { __le32 crc; - int i; - - if (IS_ERR(tfm)) - return -1; - crypto_cipher_setkey(tfm, rc4key, klen); - for (i = 0; i < data_len + IEEE80211_WEP_ICV_LEN; i++) - crypto_cipher_decrypt_one(tfm, data + i, data + i); + arc4_setkey(ctx, rc4key, klen); + arc4_crypt(ctx, data, data, data_len + IEEE80211_WEP_ICV_LEN); + memzero_explicit(ctx, sizeof(*ctx)); crc = cpu_to_le32(~crc32_le(~0, data, data_len)); if (memcmp(&crc, data + data_len, IEEE80211_WEP_ICV_LEN) != 0) @@ -253,7 +224,7 @@ static int ieee80211_wep_decrypt(struct ieee80211_local *local, /* Copy rest of the WEP key (the secret part) */ memcpy(rc4key + 3, key->conf.key, key->conf.keylen); - if (ieee80211_wep_decrypt_data(local->wep_rx_tfm, rc4key, klen, + if (ieee80211_wep_decrypt_data(&local->wep_rx_ctx, rc4key, klen, skb->data + hdrlen + IEEE80211_WEP_IV_LEN, len)) ret = -1; diff --git a/net/mac80211/wep.h b/net/mac80211/wep.h index 866a6798c9ef..997a034233c2 100644 --- a/net/mac80211/wep.h +++ b/net/mac80211/wep.h @@ -14,13 +14,12 @@ #include "key.h" int ieee80211_wep_init(struct ieee80211_local *local); -void ieee80211_wep_free(struct ieee80211_local *local); -int ieee80211_wep_encrypt_data(struct crypto_cipher *tfm, u8 *rc4key, +int ieee80211_wep_encrypt_data(struct arc4_ctx *ctx, u8 *rc4key, size_t klen, u8 *data, size_t data_len); int ieee80211_wep_encrypt(struct ieee80211_local *local, struct sk_buff *skb, const u8 *key, int keylen, int keyidx); -int ieee80211_wep_decrypt_data(struct crypto_cipher *tfm, u8 *rc4key, +int ieee80211_wep_decrypt_data(struct arc4_ctx *ctx, u8 *rc4key, size_t klen, u8 *data, size_t data_len); ieee80211_rx_result diff --git a/net/mac80211/wpa.c b/net/mac80211/wpa.c index a51c7909366e..ee72779729e5 100644 --- a/net/mac80211/wpa.c +++ b/net/mac80211/wpa.c @@ -239,7 +239,7 @@ static int tkip_encrypt_skb(struct ieee80211_tx_data *tx, struct sk_buff *skb) /* Add room for ICV */ skb_put(skb, IEEE80211_TKIP_ICV_LEN); - return ieee80211_tkip_encrypt_data(tx->local->wep_tx_tfm, + return ieee80211_tkip_encrypt_data(&tx->local->wep_tx_ctx, key, skb, pos, len); } @@ -290,7 +290,7 @@ ieee80211_crypto_tkip_decrypt(struct ieee80211_rx_data *rx) if (status->flag & RX_FLAG_DECRYPTED) hwaccel = 1; - res = ieee80211_tkip_decrypt_data(rx->local->wep_rx_tfm, + res = ieee80211_tkip_decrypt_data(&rx->local->wep_rx_ctx, key, skb->data + hdrlen, skb->len - hdrlen, rx->sta->sta.addr, hdr->addr1, hwaccel, rx->security_idx, diff --git a/net/wireless/Kconfig b/net/wireless/Kconfig index 6310ddede220..578cce4fbe6c 100644 --- a/net/wireless/Kconfig +++ b/net/wireless/Kconfig @@ -213,12 +213,14 @@ config LIB80211 config LIB80211_CRYPT_WEP tristate + select CRYPTO_LIB_ARC4 config LIB80211_CRYPT_CCMP tristate config LIB80211_CRYPT_TKIP tristate + select CRYPTO_LIB_ARC4 config LIB80211_DEBUG bool "lib80211 debugging messages" diff --git a/net/wireless/lib80211_crypt_tkip.c b/net/wireless/lib80211_crypt_tkip.c index 62edf5b01953..f5e842ba7673 100644 --- a/net/wireless/lib80211_crypt_tkip.c +++ b/net/wireless/lib80211_crypt_tkip.c @@ -9,6 +9,7 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt #include <linux/err.h> +#include <linux/fips.h> #include <linux/module.h> #include <linux/init.h> #include <linux/slab.h> @@ -25,6 +26,7 @@ #include <linux/ieee80211.h> #include <net/iw_handler.h> +#include <crypto/arc4.h> #include <crypto/hash.h> #include <linux/crypto.h> #include <linux/crc32.h> @@ -60,9 +62,9 @@ struct lib80211_tkip_data { int key_idx; - struct crypto_cipher *rx_tfm_arc4; + struct arc4_ctx rx_ctx_arc4; + struct arc4_ctx tx_ctx_arc4; struct crypto_shash *rx_tfm_michael; - struct crypto_cipher *tx_tfm_arc4; struct crypto_shash *tx_tfm_michael; /* scratch buffers for virt_to_page() (crypto API) */ @@ -89,30 +91,21 @@ static void *lib80211_tkip_init(int key_idx) { struct lib80211_tkip_data *priv; + if (fips_enabled) + return NULL; + priv = kzalloc(sizeof(*priv), GFP_ATOMIC); if (priv == NULL) goto fail; priv->key_idx = key_idx; - priv->tx_tfm_arc4 = crypto_alloc_cipher("arc4", 0, 0); - if (IS_ERR(priv->tx_tfm_arc4)) { - priv->tx_tfm_arc4 = NULL; - goto fail; - } - priv->tx_tfm_michael = crypto_alloc_shash("michael_mic", 0, 0); if (IS_ERR(priv->tx_tfm_michael)) { priv->tx_tfm_michael = NULL; goto fail; } - priv->rx_tfm_arc4 = crypto_alloc_cipher("arc4", 0, 0); - if (IS_ERR(priv->rx_tfm_arc4)) { - priv->rx_tfm_arc4 = NULL; - goto fail; - } - priv->rx_tfm_michael = crypto_alloc_shash("michael_mic", 0, 0); if (IS_ERR(priv->rx_tfm_michael)) { priv->rx_tfm_michael = NULL; @@ -124,9 +117,7 @@ static void *lib80211_tkip_init(int key_idx) fail: if (priv) { crypto_free_shash(priv->tx_tfm_michael); - crypto_free_cipher(priv->tx_tfm_arc4); crypto_free_shash(priv->rx_tfm_michael); - crypto_free_cipher(priv->rx_tfm_arc4); kfree(priv); } @@ -138,11 +129,9 @@ static void lib80211_tkip_deinit(void *priv) struct lib80211_tkip_data *_priv = priv; if (_priv) { crypto_free_shash(_priv->tx_tfm_michael); - crypto_free_cipher(_priv->tx_tfm_arc4); crypto_free_shash(_priv->rx_tfm_michael); - crypto_free_cipher(_priv->rx_tfm_arc4); } - kfree(priv); + kzfree(priv); } static inline u16 RotR1(u16 val) @@ -341,7 +330,6 @@ static int lib80211_tkip_encrypt(struct sk_buff *skb, int hdr_len, void *priv) int len; u8 rc4key[16], *pos, *icv; u32 crc; - int i; if (tkey->flags & IEEE80211_CRYPTO_TKIP_COUNTERMEASURES) { struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data; @@ -366,9 +354,9 @@ static int lib80211_tkip_encrypt(struct sk_buff *skb, int hdr_len, void *priv) icv[2] = crc >> 16; icv[3] = crc >> 24; - crypto_cipher_setkey(tkey->tx_tfm_arc4, rc4key, 16); - for (i = 0; i < len + 4; i++) - crypto_cipher_encrypt_one(tkey->tx_tfm_arc4, pos + i, pos + i); + arc4_setkey(&tkey->tx_ctx_arc4, rc4key, 16); + arc4_crypt(&tkey->tx_ctx_arc4, pos, pos, len + 4); + return 0; } @@ -396,7 +384,6 @@ static int lib80211_tkip_decrypt(struct sk_buff *skb, int hdr_len, void *priv) u8 icv[4]; u32 crc; int plen; - int i; hdr = (struct ieee80211_hdr *)skb->data; @@ -449,9 +436,8 @@ static int lib80211_tkip_decrypt(struct sk_buff *skb, int hdr_len, void *priv) plen = skb->len - hdr_len - 12; - crypto_cipher_setkey(tkey->rx_tfm_arc4, rc4key, 16); - for (i = 0; i < plen + 4; i++) - crypto_cipher_decrypt_one(tkey->rx_tfm_arc4, pos + i, pos + i); + arc4_setkey(&tkey->rx_ctx_arc4, rc4key, 16); + arc4_crypt(&tkey->rx_ctx_arc4, pos, pos, plen + 4); crc = ~crc32_le(~0, pos, plen); icv[0] = crc; @@ -636,17 +622,17 @@ static int lib80211_tkip_set_key(void *key, int len, u8 * seq, void *priv) struct lib80211_tkip_data *tkey = priv; int keyidx; struct crypto_shash *tfm = tkey->tx_tfm_michael; - struct crypto_cipher *tfm2 = tkey->tx_tfm_arc4; + struct arc4_ctx *tfm2 = &tkey->tx_ctx_arc4; struct crypto_shash *tfm3 = tkey->rx_tfm_michael; - struct crypto_cipher *tfm4 = tkey->rx_tfm_arc4; + struct arc4_ctx *tfm4 = &tkey->rx_ctx_arc4; keyidx = tkey->key_idx; memset(tkey, 0, sizeof(*tkey)); tkey->key_idx = keyidx; tkey->tx_tfm_michael = tfm; - tkey->tx_tfm_arc4 = tfm2; + tkey->tx_ctx_arc4 = *tfm2; tkey->rx_tfm_michael = tfm3; - tkey->rx_tfm_arc4 = tfm4; + tkey->rx_ctx_arc4 = *tfm4; if (len == TKIP_KEY_LEN) { memcpy(tkey->key, key, TKIP_KEY_LEN); tkey->key_set = 1; diff --git a/net/wireless/lib80211_crypt_wep.c b/net/wireless/lib80211_crypt_wep.c index e127b6f7fc9f..dafc6f3571db 100644 --- a/net/wireless/lib80211_crypt_wep.c +++ b/net/wireless/lib80211_crypt_wep.c @@ -7,6 +7,7 @@ */ #include <linux/err.h> +#include <linux/fips.h> #include <linux/module.h> #include <linux/init.h> #include <linux/slab.h> @@ -18,7 +19,7 @@ #include <net/lib80211.h> -#include <linux/crypto.h> +#include <crypto/arc4.h> #include <linux/crc32.h> MODULE_AUTHOR("Jouni Malinen"); @@ -31,52 +32,31 @@ struct lib80211_wep_data { u8 key[WEP_KEY_LEN + 1]; u8 key_len; u8 key_idx; - struct crypto_cipher *tx_tfm; - struct crypto_cipher *rx_tfm; + struct arc4_ctx tx_ctx; + struct arc4_ctx rx_ctx; }; static void *lib80211_wep_init(int keyidx) { struct lib80211_wep_data *priv; + if (fips_enabled) + return NULL; + priv = kzalloc(sizeof(*priv), GFP_ATOMIC); if (priv == NULL) - goto fail; + return NULL; priv->key_idx = keyidx; - priv->tx_tfm = crypto_alloc_cipher("arc4", 0, 0); - if (IS_ERR(priv->tx_tfm)) { - priv->tx_tfm = NULL; - goto fail; - } - - priv->rx_tfm = crypto_alloc_cipher("arc4", 0, 0); - if (IS_ERR(priv->rx_tfm)) { - priv->rx_tfm = NULL; - goto fail; - } /* start WEP IV from a random value */ get_random_bytes(&priv->iv, 4); return priv; - - fail: - if (priv) { - crypto_free_cipher(priv->tx_tfm); - crypto_free_cipher(priv->rx_tfm); - kfree(priv); - } - return NULL; } static void lib80211_wep_deinit(void *priv) { - struct lib80211_wep_data *_priv = priv; - if (_priv) { - crypto_free_cipher(_priv->tx_tfm); - crypto_free_cipher(_priv->rx_tfm); - } - kfree(priv); + kzfree(priv); } /* Add WEP IV/key info to a frame that has at least 4 bytes of headroom */ @@ -128,7 +108,6 @@ static int lib80211_wep_encrypt(struct sk_buff *skb, int hdr_len, void *priv) u32 crc, klen, len; u8 *pos, *icv; u8 key[WEP_KEY_LEN + 3]; - int i; /* other checks are in lib80211_wep_build_iv */ if (skb_tailroom(skb) < 4) @@ -156,10 +135,8 @@ static int lib80211_wep_encrypt(struct sk_buff *skb, int hdr_len, void *priv) icv[2] = crc >> 16; icv[3] = crc >> 24; - crypto_cipher_setkey(wep->tx_tfm, key, klen); - - for (i = 0; i < len + 4; i++) - crypto_cipher_encrypt_one(wep->tx_tfm, pos + i, pos + i); + arc4_setkey(&wep->tx_ctx, key, klen); + arc4_crypt(&wep->tx_ctx, pos, pos, len + 4); return 0; } @@ -177,7 +154,6 @@ static int lib80211_wep_decrypt(struct sk_buff *skb, int hdr_len, void *priv) u32 crc, klen, plen; u8 key[WEP_KEY_LEN + 3]; u8 keyidx, *pos, icv[4]; - int i; if (skb->len < hdr_len + 8) return -1; @@ -198,9 +174,8 @@ static int lib80211_wep_decrypt(struct sk_buff *skb, int hdr_len, void *priv) /* Apply RC4 to data and compute CRC32 over decrypted data */ plen = skb->len - hdr_len - 8; - crypto_cipher_setkey(wep->rx_tfm, key, klen); - for (i = 0; i < plen + 4; i++) - crypto_cipher_decrypt_one(wep->rx_tfm, pos + i, pos + i); + arc4_setkey(&wep->rx_ctx, key, klen); + arc4_crypt(&wep->rx_ctx, pos, pos, plen + 4); crc = ~crc32_le(~0, pos, plen); icv[0] = crc; |