diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2016-01-12 18:51:14 -0800 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2016-01-12 18:51:14 -0800 |
| commit | c597b6bcd5c624534afc3df65cdc42bb05173bca (patch) | |
| tree | 8fedd26c5dc0357a10db08a6bef31085e2508280 /drivers/crypto | |
| parent | 60b7eca1dc2ec066916b3b7ac6ad89bea13cb9af (diff) | |
| parent | 48d627648141479c8be8acd110191072e24eba25 (diff) | |
Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6
Pull crypto update from Herbert Xu:
"Algorithms:
- Add RSA padding algorithm
Drivers:
- Add GCM mode support to atmel
- Add atmel support for SAMA5D2 devices
- Add cipher modes to talitos
- Add rockchip driver for rk3288
- Add qat support for C3XXX and C62X"
* 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (103 commits)
crypto: hifn_795x, picoxcell - use ablkcipher_request_cast
crypto: qat - fix SKU definiftion for c3xxx dev
crypto: qat - Fix random config build issue
crypto: ccp - use to_pci_dev and to_platform_device
crypto: qat - Rename dh895xcc mmp firmware
crypto: 842 - remove WARN inside printk
crypto: atmel-aes - add debug facilities to monitor register accesses.
crypto: atmel-aes - add support to GCM mode
crypto: atmel-aes - change the DMA threshold
crypto: atmel-aes - fix the counter overflow in CTR mode
crypto: atmel-aes - fix atmel-ctr-aes driver for RFC 3686
crypto: atmel-aes - create sections to regroup functions by usage
crypto: atmel-aes - fix typo and indentation
crypto: atmel-aes - use SIZE_IN_WORDS() helper macro
crypto: atmel-aes - improve performances of data transfer
crypto: atmel-aes - fix atmel_aes_remove()
crypto: atmel-aes - remove useless AES_FLAGS_DMA flag
crypto: atmel-aes - reduce latency of DMA completion
crypto: atmel-aes - remove unused 'err' member of struct atmel_aes_dev
crypto: atmel-aes - rework crypto request completion
...
Diffstat (limited to 'drivers/crypto')
84 files changed, 6420 insertions, 1531 deletions
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 2569e043317e..3dd69df9c970 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -194,6 +194,9 @@ config CRYPTO_DEV_NIAGARA2 select CRYPTO_DES select CRYPTO_BLKCIPHER select CRYPTO_HASH + select CRYPTO_MD5 + select CRYPTO_SHA1 + select CRYPTO_SHA256 depends on SPARC64 help Each core of a Niagara2 processor contains a Stream @@ -378,10 +381,10 @@ config CRYPTO_DEV_BFIN_CRC config CRYPTO_DEV_ATMEL_AES tristate "Support for Atmel AES hw accelerator" - depends on ARCH_AT91 + depends on AT_XDMAC || AT_HDMAC || COMPILE_TEST select CRYPTO_AES + select CRYPTO_AEAD select CRYPTO_BLKCIPHER - select AT_HDMAC help Some Atmel processors have AES hw accelerator. Select this if you want to use the Atmel module for @@ -498,4 +501,15 @@ config CRYPTO_DEV_SUN4I_SS To compile this driver as a module, choose M here: the module will be called sun4i-ss. +config CRYPTO_DEV_ROCKCHIP + tristate "Rockchip's Cryptographic Engine driver" + depends on OF && ARCH_ROCKCHIP + select CRYPTO_AES + select CRYPTO_DES + select CRYPTO_BLKCIPHER + + help + This driver interfaces with the hardware crypto accelerator. + Supporting cbc/ecb chainmode, and aes/des/des3_ede cipher mode. + endif # CRYPTO_HW diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index c3ced6fbd1b8..713de9d11148 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -29,3 +29,4 @@ obj-$(CONFIG_CRYPTO_DEV_QAT) += qat/ obj-$(CONFIG_CRYPTO_DEV_QCE) += qce/ obj-$(CONFIG_CRYPTO_DEV_VMX) += vmx/ obj-$(CONFIG_CRYPTO_DEV_SUN4I_SS) += sunxi-ss/ +obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP) += rockchip/ diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 58a630e55d5d..62134c8a2260 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -781,6 +781,10 @@ u32 crypto4xx_build_pd(struct crypto_async_request *req, /* figure how many gd is needed */ num_gd = sg_nents_for_len(src, datalen); + if ((int)num_gd < 0) { + dev_err(dev->core_dev->device, "Invalid number of src SG.\n"); + return -EINVAL; + } if (num_gd == 1) num_gd = 0; diff --git a/drivers/crypto/atmel-aes-regs.h b/drivers/crypto/atmel-aes-regs.h index 2786bb1a5aa0..6c2951bb70b1 100644 --- a/drivers/crypto/atmel-aes-regs.h +++ b/drivers/crypto/atmel-aes-regs.h @@ -9,6 +9,7 @@ #define AES_MR 0x04 #define AES_MR_CYPHER_DEC (0 << 0) #define AES_MR_CYPHER_ENC (1 << 0) +#define AES_MR_GTAGEN (1 << 1) #define AES_MR_DUALBUFF (1 << 3) #define AES_MR_PROCDLY_MASK (0xF << 4) #define AES_MR_PROCDLY_OFFSET 4 @@ -26,6 +27,7 @@ #define AES_MR_OPMOD_OFB (0x2 << 12) #define AES_MR_OPMOD_CFB (0x3 << 12) #define AES_MR_OPMOD_CTR (0x4 << 12) +#define AES_MR_OPMOD_GCM (0x5 << 12) #define AES_MR_LOD (0x1 << 15) #define AES_MR_CFBS_MASK (0x7 << 16) #define AES_MR_CFBS_128b (0x0 << 16) @@ -44,6 +46,7 @@ #define AES_ISR 0x1C #define AES_INT_DATARDY (1 << 0) #define AES_INT_URAD (1 << 8) +#define AES_INT_TAGRDY (1 << 16) #define AES_ISR_URAT_MASK (0xF << 12) #define AES_ISR_URAT_IDR_WR_PROC (0x0 << 12) #define AES_ISR_URAT_ODR_RD_PROC (0x1 << 12) @@ -57,6 +60,13 @@ #define AES_ODATAR(x) (0x50 + ((x) * 0x04)) #define AES_IVR(x) (0x60 + ((x) * 0x04)) +#define AES_AADLENR 0x70 +#define AES_CLENR 0x74 +#define AES_GHASHR(x) (0x78 + ((x) * 0x04)) +#define AES_TAGR(x) (0x88 + ((x) * 0x04)) +#define AES_CTRR 0x98 +#define AES_GCMHR(x) (0x9c + ((x) * 0x04)) + #define AES_HW_VERSION 0xFC #endif /* __ATMEL_AES_REGS_H__ */ diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index fb16d812c8f5..5621612ee921 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -33,68 +33,118 @@ #include <linux/of_device.h> #include <linux/delay.h> #include <linux/crypto.h> -#include <linux/cryptohash.h> #include <crypto/scatterwalk.h> #include <crypto/algapi.h> #include <crypto/aes.h> -#include <crypto/hash.h> -#include <crypto/internal/hash.h> +#include <crypto/internal/aead.h> #include <linux/platform_data/crypto-atmel.h> #include <dt-bindings/dma/at91.h> #include "atmel-aes-regs.h" +#define ATMEL_AES_PRIORITY 300 + +#define ATMEL_AES_BUFFER_ORDER 2 +#define ATMEL_AES_BUFFER_SIZE (PAGE_SIZE << ATMEL_AES_BUFFER_ORDER) + #define CFB8_BLOCK_SIZE 1 #define CFB16_BLOCK_SIZE 2 #define CFB32_BLOCK_SIZE 4 #define CFB64_BLOCK_SIZE 8 +#define SIZE_IN_WORDS(x) ((x) >> 2) + /* AES flags */ -#define AES_FLAGS_MODE_MASK 0x03ff -#define AES_FLAGS_ENCRYPT BIT(0) -#define AES_FLAGS_CBC BIT(1) -#define AES_FLAGS_CFB BIT(2) -#define AES_FLAGS_CFB8 BIT(3) -#define AES_FLAGS_CFB16 BIT(4) -#define AES_FLAGS_CFB32 BIT(5) -#define AES_FLAGS_CFB64 BIT(6) -#define AES_FLAGS_CFB128 BIT(7) -#define AES_FLAGS_OFB BIT(8) -#define AES_FLAGS_CTR BIT(9) - -#define AES_FLAGS_INIT BIT(16) -#define AES_FLAGS_DMA BIT(17) -#define AES_FLAGS_BUSY BIT(18) -#define AES_FLAGS_FAST BIT(19) +/* Reserve bits [18:16] [14:12] [1:0] for mode (same as for AES_MR) */ +#define AES_FLAGS_ENCRYPT AES_MR_CYPHER_ENC +#define AES_FLAGS_GTAGEN AES_MR_GTAGEN +#define AES_FLAGS_OPMODE_MASK (AES_MR_OPMOD_MASK | AES_MR_CFBS_MASK) +#define AES_FLAGS_ECB AES_MR_OPMOD_ECB +#define AES_FLAGS_CBC AES_MR_OPMOD_CBC +#define AES_FLAGS_OFB AES_MR_OPMOD_OFB +#define AES_FLAGS_CFB128 (AES_MR_OPMOD_CFB | AES_MR_CFBS_128b) +#define AES_FLAGS_CFB64 (AES_MR_OPMOD_CFB | AES_MR_CFBS_64b) +#define AES_FLAGS_CFB32 (AES_MR_OPMOD_CFB | AES_MR_CFBS_32b) +#define AES_FLAGS_CFB16 (AES_MR_OPMOD_CFB | AES_MR_CFBS_16b) +#define AES_FLAGS_CFB8 (AES_MR_OPMOD_CFB | AES_MR_CFBS_8b) +#define AES_FLAGS_CTR AES_MR_OPMOD_CTR +#define AES_FLAGS_GCM AES_MR_OPMOD_GCM + +#define AES_FLAGS_MODE_MASK (AES_FLAGS_OPMODE_MASK | \ + AES_FLAGS_ENCRYPT | \ + AES_FLAGS_GTAGEN) + +#define AES_FLAGS_INIT BIT(2) +#define AES_FLAGS_BUSY BIT(3) +#define AES_FLAGS_DUMP_REG BIT(4) + +#define AES_FLAGS_PERSISTENT (AES_FLAGS_INIT | AES_FLAGS_BUSY) #define ATMEL_AES_QUEUE_LENGTH 50 -#define ATMEL_AES_DMA_THRESHOLD 16 +#define ATMEL_AES_DMA_THRESHOLD 256 struct atmel_aes_caps { - bool has_dualbuff; - bool has_cfb64; - u32 max_burst_size; + bool has_dualbuff; + bool has_cfb64; + bool has_ctr32; + bool has_gcm; + u32 max_burst_size; }; struct atmel_aes_dev; + +typedef int (*atmel_aes_fn_t)(struct atmel_aes_dev *); + + +struct atmel_aes_base_ctx { + struct atmel_aes_dev *dd; + atmel_aes_fn_t start; + int keylen; + u32 key[AES_KEYSIZE_256 / sizeof(u32)]; + u16 block_size; +}; + struct atmel_aes_ctx { - struct atmel_aes_dev *dd; + struct atmel_aes_base_ctx base; +}; + +struct atmel_aes_ctr_ctx { + struct atmel_aes_base_ctx base; + + u32 iv[AES_BLOCK_SIZE / sizeof(u32)]; + size_t offset; + struct scatterlist src[2]; + struct scatterlist dst[2]; +}; - int keylen; - u32 key[AES_KEYSIZE_256 / sizeof(u32)]; +struct atmel_aes_gcm_ctx { + struct atmel_aes_base_ctx base; - u16 block_size; + struct scatterlist src[2]; + struct scatterlist dst[2]; + + u32 j0[AES_BLOCK_SIZE / sizeof(u32)]; + u32 tag[AES_BLOCK_SIZE / sizeof(u32)]; + u32 ghash[AES_BLOCK_SIZE / sizeof(u32)]; + size_t textlen; + + const u32 *ghash_in; + u32 *ghash_out; + atmel_aes_fn_t ghash_resume; }; struct atmel_aes_reqctx { - unsigned long mode; + unsigned long mode; }; struct atmel_aes_dma { - struct dma_chan *chan; - struct dma_slave_config dma_conf; + struct dma_chan *chan; + struct scatterlist *sg; + int nents; + unsigned int remainder; + unsigned int sg_len; }; struct atmel_aes_dev { @@ -102,13 +152,18 @@ struct atmel_aes_dev { unsigned long phys_base; void __iomem *io_base; - struct atmel_aes_ctx *ctx; + struct crypto_async_request *areq; + struct atmel_aes_base_ctx *ctx; + + bool is_async; + atmel_aes_fn_t resume; + atmel_aes_fn_t cpu_transfer_complete; + struct device *dev; struct clk *iclk; - int irq; + int irq; unsigned long flags; - int err; spinlock_t lock; struct crypto_queue queue; @@ -116,33 +171,21 @@ struct atmel_aes_dev { struct tasklet_struct done_task; struct tasklet_struct queue_task; - struct ablkcipher_request *req; - size_t total; + size_t total; + size_t datalen; + u32 *data; - struct scatterlist *in_sg; - unsigned int nb_in_sg; - size_t in_offset; - struct scatterlist *out_sg; - unsigned int nb_out_sg; - size_t out_offset; + struct atmel_aes_dma src; + struct atmel_aes_dma dst; - size_t bufcnt; - size_t buflen; - size_t dma_size; - - void *buf_in; - int dma_in; - dma_addr_t dma_addr_in; - struct atmel_aes_dma dma_lch_in; - - void *buf_out; - int dma_out; - dma_addr_t dma_addr_out; - struct atmel_aes_dma dma_lch_out; + size_t buflen; + void *buf; + struct scatterlist aligned_sg; + struct scatterlist *real_dst; struct atmel_aes_caps caps; - u32 hw_version; + u32 hw_version; }; struct atmel_aes_drv { @@ -155,71 +198,128 @@ static struct atmel_aes_drv atmel_aes = { .lock = __SPIN_LOCK_UNLOCKED(atmel_aes.lock), }; -static int atmel_aes_sg_length(struct ablkcipher_request *req, - struct scatterlist *sg) +#ifdef VERBOSE_DEBUG +static const char *atmel_aes_reg_name(u32 offset, char *tmp, size_t sz) { - unsigned int total = req->nbytes; - int sg_nb; - unsigned int len; - struct scatterlist *sg_list; - - sg_nb = 0; - sg_list = sg; - total = req->nbytes; + switch (offset) { + case AES_CR: + return "CR"; + + case AES_MR: + return "MR"; + + case AES_ISR: + return "ISR"; + + case AES_IMR: + return "IMR"; + + case AES_IER: + return "IER"; + + case AES_IDR: + return "IDR"; + + case AES_KEYWR(0): + case AES_KEYWR(1): + case AES_KEYWR(2): + case AES_KEYWR(3): + case AES_KEYWR(4): + case AES_KEYWR(5): + case AES_KEYWR(6): + case AES_KEYWR(7): + snprintf(tmp, sz, "KEYWR[%u]", (offset - AES_KEYWR(0)) >> 2); + break; - while (total) { - len = min(sg_list->length, total); + case AES_IDATAR(0): + case AES_IDATAR(1): + case AES_IDATAR(2): + case AES_IDATAR(3): + snprintf(tmp, sz, "IDATAR[%u]", (offset - AES_IDATAR(0)) >> 2); + break; - sg_nb++; - total -= len; + case AES_ODATAR(0): + case AES_ODATAR(1): + case AES_ODATAR(2): + case AES_ODATAR(3): + snprintf(tmp, sz, "ODATAR[%u]", (offset - AES_ODATAR(0)) >> 2); + break; - sg_list = sg_next(sg_list); - if (!sg_list) - total = 0; - } + case AES_IVR(0): + case AES_IVR(1): + case AES_IVR(2): + case AES_IVR(3): + snprintf(tmp, sz, "IVR[%u]", (offset - AES_IVR(0)) >> 2); + break; - return sg_nb; -} + case AES_AADLENR: + return "AADLENR"; -static int atmel_aes_sg_copy(struct scatterlist **sg, size_t *offset, - void *buf, size_t buflen, size_t total, int out) -{ - unsigned int count, off = 0; + case AES_CLENR: + return "CLENR"; - while (buflen && total) { - count = min((*sg)->length - *offset, total); - count = min(count, buflen); + case AES_GHASHR(0): + case AES_GHASHR(1): + case AES_GHASHR(2): + case AES_GHASHR(3): + snprintf(tmp, sz, "GHASHR[%u]", (offset - AES_GHASHR(0)) >> 2); + break; - if (!count) - return off; + case AES_TAGR(0): + case AES_TAGR(1): + case AES_TAGR(2): + case AES_TAGR(3): + snprintf(tmp, sz, "TAGR[%u]", (offset - AES_TAGR(0)) >> 2); + break; - scatterwalk_map_and_copy(buf + off, *sg, *offset, count, out); + case AES_CTRR: + return "CTRR"; - off += count; - buflen -= count; - *offset += count; - total -= count; + case AES_GCMHR(0): + case AES_GCMHR(1): + case AES_GCMHR(2): + case AES_GCMHR(3): + snprintf(tmp, sz, "GCMHR[%u]", (offset - AES_GCMHR(0)) >> 2); - if (*offset == (*sg)->length) { - *sg = sg_next(*sg); - if (*sg) - *offset = 0; - else - total = 0; - } + default: + snprintf(tmp, sz, "0x%02x", offset); + break; } - return off; + return tmp; } +#endif /* VERBOSE_DEBUG */ + +/* Shared functions */ static inline u32 atmel_aes_read(struct atmel_aes_dev *dd, u32 offset) { - return readl_relaxed(dd->io_base + offset); + u32 value = readl_relaxed(dd->io_base + offset); + +#ifdef VERBOSE_DEBUG + if (dd->flags & AES_FLAGS_DUMP_REG) { + char tmp[16]; + + dev_vdbg(dd->dev, "read 0x%08x from %s\n", value, + atmel_aes_reg_name(offset, tmp, sizeof(tmp))); + } +#endif /* VERBOSE_DEBUG */ + + return value; } static inline void atmel_aes_write(struct atmel_aes_dev *dd, u32 offset, u32 value) { +#ifdef VERBOSE_DEBUG + if (dd->flags & AES_FLAGS_DUMP_REG) { + char tmp[16]; + + dev_vdbg(dd->dev, "write 0x%08x into %s\n", value, + atmel_aes_reg_name(offset, tmp)); + } +#endif /* VERBOSE_DEBUG */ + writel_relaxed(value, dd->io_base + offset); } @@ -231,13 +331,50 @@ static void atmel_aes_read_n(struct atmel_aes_dev *dd, u32 offset, } static void atmel_aes_write_n(struct atmel_aes_dev *dd, u32 offset, - u32 *value, int count) + const u32 *value, int count) { for (; count--; value++, offset += 4) atmel_aes_write(dd, offset, *value); } -static struct atmel_aes_dev *atmel_aes_find_dev(struct atmel_aes_ctx *ctx) +static inline void atmel_aes_read_block(struct atmel_aes_dev *dd, u32 offset, + u32 *value) +{ + atmel_aes_read_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE)); +} + +static inline void atmel_aes_write_block(struct atmel_aes_dev *dd, u32 offset, + const u32 *value) +{ + atmel_aes_write_n(dd, offset, value, SIZE_IN_WORDS(AES_BLOCK_SIZE)); +} + +static inline int atmel_aes_wait_for_data_ready(struct atmel_aes_dev *dd, + atmel_aes_fn_t resume) +{ + u32 isr = atmel_aes_read(dd, AES_ISR); + + if (unlikely(isr & AES_INT_DATARDY)) + return resume(dd); + + dd->resume = resume; + atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); + return -EINPROGRESS; +} + +static inline size_t atmel_aes_padlen(size_t len, size_t block_size) +{ + len &= block_size - 1; + return len ? block_size - len : 0; +} + +static inline struct aead_request * +aead_request_cast(struct crypto_async_request *req) +{ + return container_of(req, struct aead_request, base); +} + +static struct atmel_aes_dev *atmel_aes_find_dev(struct atmel_aes_base_ctx *ctx) { struct atmel_aes_dev *aes_dd = NULL; struct atmel_aes_dev *tmp; @@ -270,7 +407,6 @@ static int atmel_aes_hw_init(struct atmel_aes_dev *dd) atmel_aes_write(dd, AES_CR, AES_CR_SWRST); atmel_aes_write(dd, AES_MR, 0xE << AES_MR_CKEY_OFFSET); dd->flags |= AES_FLAGS_INIT; - dd->err = 0; } return 0; @@ -281,552 +417,643 @@ static inline unsigned int atmel_aes_get_version(struct atmel_aes_dev *dd) return atmel_aes_read(dd, AES_HW_VERSION) & 0x00000fff; } -static void atmel_aes_hw_version_init(struct atmel_aes_dev *dd) +static int atmel_aes_hw_version_init(struct atmel_aes_dev *dd) { - atmel_aes_hw_init(dd); + int err; + + err = atmel_aes_hw_init(dd); + if (err) + return err; dd->hw_version = atmel_aes_get_version(dd); - dev_info(dd->dev, - "version: 0x%x\n", dd->hw_version); + dev_info(dd->dev, "version: 0x%x\n", dd->hw_version); clk_disable_unprepare(dd->iclk); + return 0; } -static void atmel_aes_finish_req(struct atmel_aes_dev *dd, int err) +static inline void atmel_aes_set_mode(struct atmel_aes_dev *dd, + const struct atmel_aes_reqctx *rctx) { - struct ablkcipher_request *req = dd->req; + /* Clear all but persistent flags and set request flags. */ + dd->flags = (dd->flags & AES_FLAGS_PERSISTENT) | rctx->mode; +} +static inline bool atmel_aes_is_encrypt(const struct atmel_aes_dev *dd) +{ + return (dd->flags & AES_FLAGS_ENCRYPT); +} + +static inline int atmel_aes_complete(struct atmel_aes_dev *dd, int err) +{ clk_disable_unprepare(dd->iclk); dd->flags &= ~AES_FLAGS_BUSY; - req->base.complete(&req->base, err); -} + if (dd->is_async) + dd->areq->complete(dd->areq, err); -static void atmel_aes_dma_callback(void *data) -{ - struct atmel_aes_dev *dd = data; + tasklet_schedule(&dd->queue_task); - /* dma_lch_out - completed */ - tasklet_schedule(&dd->done_task); + return err; } -static int atmel_aes_crypt_dma(struct atmel_aes_dev *dd, - dma_addr_t dma_addr_in, dma_addr_t dma_addr_out, int length) +static void atmel_aes_write_ctrl(struct atmel_aes_dev *dd, bool use_dma, + const u32 *iv) { - struct scatterlist sg[2]; - struct dma_async_tx_descriptor *in_desc, *out_desc; + u32 valmr = 0; - dd->dma_size = length; + /* MR register must be set before IV registers */ + if (dd->ctx->keylen == AES_KEYSIZE_128) + valmr |= AES_MR_KEYSIZE_128; + else if (dd->ctx->keylen == AES_KEYSIZE_192) + valmr |= AES_MR_KEYSIZE_192; + else + valmr |= AES_MR_KEYSIZE_256; - dma_sync_single_for_device(dd->dev, dma_addr_in, length, - DMA_TO_DEVICE); - dma_sync_single_for_device(dd->dev, dma_addr_out, length, - DMA_FROM_DEVICE); + valmr |= dd->flags & AES_FLAGS_MODE_MASK; - if (dd->flags & AES_FLAGS_CFB8) { - dd->dma_lch_in.dma_conf.dst_addr_width = - DMA_SLAVE_BUSWIDTH_1_BYTE; - dd->dma_lch_out.dma_conf.src_addr_width = - DMA_SLAVE_BUSWIDTH_1_BYTE; - } else if (dd->flags & AES_FLAGS_CFB16) { - dd->dma_lch_in.dma_conf.dst_addr_width = - DMA_SLAVE_BUSWIDTH_2_BYTES; - dd->dma_lch_out.dma_conf.src_addr_width = - DMA_SLAVE_BUSWIDTH_2_BYTES; + if (use_dma) { + valmr |= AES_MR_SMOD_IDATAR0; + if (dd->caps.has_dualbuff) + valmr |= AES_MR_DUALBUFF; } else { - dd->dma_lch_in.dma_conf.dst_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_out.dma_conf.src_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; + valmr |= AES_MR_SMOD_AUTO; } - if (dd->flags & (AES_FLAGS_CFB8 | AES_FLAGS_CFB16 | - AES_FLAGS_CFB32 | AES_FLAGS_CFB64)) { - dd->dma_lch_in.dma_conf.src_maxburst = 1; - dd->dma_lch_in.dma_conf.dst_maxburst = 1; - dd->dma_lch_out.dma_conf.src_maxburst = 1; - dd->dma_lch_out.dma_conf.dst_maxburst = 1; - } else { - dd->dma_lch_in.dma_conf.src_maxburst = dd->caps.max_burst_size; - dd->dma_lch_in.dma_conf.dst_maxburst = dd->caps.max_burst_size; - dd->dma_lch_out.dma_conf.src_maxburst = dd->caps.max_burst_size; - dd->dma_lch_out.dma_conf.dst_maxburst = dd->caps.max_burst_size; - } + atmel_aes_write(dd, AES_MR, valmr); - dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf); - dmaengine_slave_config(dd->dma_lch_out.chan, &dd->dma_lch_out.dma_conf); + atmel_aes_write_n(dd, AES_KEYWR(0), dd->ctx->key, + SIZE_IN_WORDS(dd->ctx->keylen)); - dd->flags |= AES_FLAGS_DMA; + if (iv && (valmr & AES_MR_OPMOD_MASK) != AES_MR_OPMOD_ECB) + atmel_aes_write_block(dd, AES_IVR(0), iv); +} - sg_init_table(&sg[0], 1); - sg_dma_address(&sg[0]) = dma_addr_in; - sg_dma_len(&sg[0]) = length; - sg_init_table(&sg[1], 1); - sg_dma_address(&sg[1]) = dma_addr_out; - sg_dma_len(&sg[1]) = length; +/* CPU transfer */ - in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, &sg[0], - 1, DMA_MEM_TO_DEV, - DMA_PREP_INTERRUPT | DMA_CTRL_ACK); - if (!in_desc) - return -EINVAL; +static int atmel_aes_cpu_transfer(struct atmel_aes_dev *dd) +{ + int err = 0; + u32 isr; - out_desc = dmaengine_prep_slave_sg(dd->dma_lch_out.chan, &sg[1], - 1, DMA_DEV_TO_MEM, - DMA_PREP_INTERRUPT | DMA_CTRL_ACK); - if (!out_desc) - return -EINVAL; + for (;;) { + atmel_aes_read_block(dd, AES_ODATAR(0), dd->data); + dd->data += 4; + dd->datalen -= AES_BLOCK_SIZE; - out_desc->callback = atmel_aes_dma_callback; - out_desc->callback_param = dd; + if (dd->datalen < AES_BLOCK_SIZE) + break; - dmaengine_submit(out_desc); - dma_async_issue_pending(dd->dma_lch_out.chan); + atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); - dmaengine_submit(in_desc); - dma_async_issue_pending(dd->dma_lch_in.chan); + isr = atmel_aes_read(dd, AES_ISR); + if (!(isr & AES_INT_DATARDY)) { + dd->resume = atmel_aes_cpu_transfer; + atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); + return -EINPROGRESS; + } + } - return 0; + if (!sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst), + dd->buf, dd->total)) + err = -EINVAL; + + if (err) + return atmel_aes_complete(dd, err); + + return dd->cpu_transfer_complete(dd); } -static int atmel_aes_crypt_cpu_start(struct atmel_aes_dev *dd) +static int atmel_aes_cpu_start(struct atmel_aes_dev *dd, + struct scatterlist *src, + struct scatterlist *dst, + size_t len, + atmel_aes_fn_t resume) { - dd->flags &= ~AES_FLAGS_DMA; + size_t padlen = atmel_aes_padlen(len, AES_BLOCK_SIZE); - dma_sync_single_for_cpu(dd->dev, dd->dma_addr_in, - dd->dma_size, DMA_TO_DEVICE); - dma_sync_single_for_cpu(dd->dev, dd->dma_addr_out, - dd->dma_size, DMA_FROM_DEVICE); - - /* use cache buffers */ - dd->nb_in_sg = atmel_aes_sg_length(dd->req, dd->in_sg); - if (!dd->nb_in_sg) + if (unlikely(len == 0)) return -EINVAL; - dd->nb_out_sg = atmel_aes_sg_length(dd->req, dd->out_sg); - if (!dd->nb_out_sg) - return -EINVAL; + sg_copy_to_buffer(src, sg_nents(src), dd->buf, len); - dd->bufcnt = sg_copy_to_buffer(dd->in_sg, dd->nb_in_sg, - dd->buf_in, dd->total); + dd->total = len; + dd->real_dst = dst; + dd->cpu_transfer_complete = resume; + dd->datalen = len + padlen; + dd->data = (u32 *)dd->buf; + atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); + return atmel_aes_wait_for_data_ready(dd, atmel_aes_cpu_transfer); +} - if (!dd->bufcnt) - return -EINVAL; - dd->total -= dd->bufcnt; +/* DMA transfer */ - atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); - atmel_aes_write_n(dd, AES_IDATAR(0), (u32 *) dd->buf_in, - dd->bufcnt >> 2); +static void atmel_aes_dma_callback(void *data); - return 0; +static bool atmel_aes_check_aligned(struct atmel_aes_dev *dd, + struct scatterlist *sg, + size_t len, + struct atmel_aes_dma *dma) +{ + int nents; + + if (!IS_ALIGNED(len, dd->ctx->block_size)) + return false; + + for (nents = 0; sg; sg = sg_next(sg), ++nents) { + if (!IS_ALIGNED(sg->offset, sizeof(u32))) + return false; + + if (len <= sg->length) { + if (!IS_ALIGNED(len, dd->ctx->block_size)) + return false; + + dma->nents = nents+1; + dma->remainder = sg->length - len; + sg->length = len; + return true; + } + + if (!IS_ALIGNED(sg->length, dd->ctx->block_size)) + return false; + + len -= sg->length; + } + + return false; } -static int atmel_aes_crypt_dma_start(struct atmel_aes_dev *dd) +static inline void atmel_aes_restore_sg(const struct atmel_aes_dma *dma) { - int err, fast = 0, in, out; - size_t count; - dma_addr_t addr_in, addr_out; + struct scatterlist *sg = dma->sg; + int nents = dma->nents; - if ((!dd->in_offset) && (!dd->out_offset)) { - /* check for alignment */ - in = IS_ALIGNED((u32)dd->in_sg->offset, sizeof(u32)) && - IS_ALIGNED(dd->in_sg->length, dd->ctx->block_size); - out = IS_ALIGNED((u32)dd->out_sg->offset, sizeof(u32)) && - IS_ALIGNED(dd->out_sg->length, dd->ctx->block_size); - fast = in && out; + if (!dma->remainder) + return; - if (sg_dma_len(dd->in_sg) != sg_dma_len(dd->out_sg)) - fast = 0; - } + while (--nents > 0 && sg) + sg = sg_next(sg); + if (!sg) + return; - if (fast) { - count = min(dd->total, sg_dma_len(dd->in_sg)); - count = min(count, sg_dma_len(dd->out_sg)); + sg->length += dma->remainder; +} - err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); - if (!err) { - dev_err(dd->dev, "dma_map_sg() error\n"); - return -EINVAL; - } +static int atmel_aes_map(struct atmel_aes_dev *dd, + struct scatterlist *src, + struct scatterlist *dst, + size_t len) +{ + bool src_aligned, dst_aligned; + size_t padlen; + + dd->total = len; + dd->src.sg = src; + dd->dst.sg = dst; + dd->real_dst = dst; - err = dma_map_sg(dd->dev, dd->out_sg, 1, - DMA_FROM_DEVICE); - if (!err) { - dev_err(dd->dev, "dma_map_sg() error\n"); - dma_unmap_sg(dd->dev, dd->in_sg, 1, - DMA_TO_DEVICE); - return -EINVAL; + src_aligned = atmel_aes_check_aligned(dd, src, len, &dd->src); + if (src == dst) + dst_aligned = src_aligned; + else + dst_aligned = atmel_aes_check_aligned(dd, dst, len, &dd->dst); + if (!src_aligned || !dst_aligned) { + padlen = atmel_aes_padlen(len, dd->ctx->block_size); + + if (dd->buflen < len + padlen) + return -ENOMEM; + + if (!src_aligned) { + sg_copy_to_buffer(src, sg_nents(src), dd->buf, len); + dd->src.sg = &dd->aligned_sg; + dd->src.nents = 1; + dd->src.remainder = 0; } - addr_in = sg_dma_address(dd->in_sg); - addr_out = sg_dma_address(dd->out_sg); + if (!dst_aligned) { + dd->dst.sg = &dd->aligned_sg; + dd->dst.nents = 1; + dd->dst.remainder = 0; + } - dd->flags |= AES_FLAGS_FAST; + sg_init_table(&dd->aligned_sg, 1); + sg_set_buf(&dd->aligned_sg, dd->buf, len + padlen); + } + if (dd->src.sg == dd->dst.sg) { + dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents, + DMA_BIDIRECTIONAL); + dd->dst.sg_len = dd->src.sg_len; + if (!dd->src.sg_len) + return -EFAULT; } else { - dma_sync_single_for_cpu(dd->dev, dd->dma_addr_in, - dd->dma_size, DMA_TO_DEVICE); + dd->src.sg_len = dma_map_sg(dd->dev, dd->src.sg, dd->src.nents, + DMA_TO_DEVICE); + if (!dd->src.sg_len) + return -EFAULT; + + dd->dst.sg_len = dma_map_sg(dd->dev, dd->dst.sg, dd->dst.nents, + DMA_FROM_DEVICE); + if (!dd->dst.sg_len) { + dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents, + DMA_TO_DEVICE); + return -EFAULT; + } + } - /* use cache buffers */ - count = atmel_aes_sg_copy(&dd->in_sg, &dd->in_offset, - dd->buf_in, dd->buflen, dd->total, 0); + return 0; +} - addr_in = dd->dma_addr_in; - addr_out = dd->dma_addr_out; +static void atmel_aes_unmap(struct atmel_aes_dev *dd) +{ + if (dd->src.sg == dd->dst.sg) { + dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents, + DMA_BIDIRECTIONAL); - dd->flags &= ~AES_FLAGS_FAST; - } + if (dd->src.sg != &dd->aligned_sg) + atmel_aes_restore_sg(&dd->src); + } else { + dma_unmap_sg(dd->dev, dd->dst.sg, dd->dst.nents, + DMA_FROM_DEVICE); - dd->total -= count; + if (dd->dst.sg != &dd->aligned_sg) + atmel_aes_restore_sg(&dd->dst); - err = atmel_aes_crypt_dma(dd, addr_in, addr_out, count); + dma_unmap_sg(dd->dev, dd->src.sg, dd->src.nents, + DMA_TO_DEVICE); - if (err && (dd->flags & AES_FLAGS_FAST)) { - dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); - dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE); + if (dd->src.sg != &dd->aligned_sg) + atmel_aes_restore_sg(&dd->src); } - return err; + if (dd->dst.sg == &dd->aligned_sg) + sg_copy_from_buffer(dd->real_dst, sg_nents(dd->real_dst), + dd->buf, dd->total); } -static int atmel_aes_write_ctrl(struct atmel_aes_dev *dd) +static int atmel_aes_dma_transfer_start(struct atmel_aes_dev *dd, + enum dma_slave_buswidth addr_width, + enum dma_transfer_direction dir, + u32 maxburst) { + struct dma_async_tx_descriptor *desc; + struct dma_slave_config config; + dma_async_tx_callback callback; + struct atmel_aes_dma *dma; int err; - u32 valcr = 0, valmr = 0; - err = atmel_aes_hw_init(dd); + memset(&config, 0, sizeof(config)); + config.direction = dir; + config.src_addr_width = addr_width; + config.dst_addr_width = addr_width; + config.src_maxburst = maxburst; + config.dst_maxburst = maxburst; + + switch (dir) { + case DMA_MEM_TO_DEV: + dma = &dd->src; + callback = NULL; + config.dst_addr = dd->phys_base + AES_IDATAR(0); + break; + + case DMA_DEV_TO_MEM: + dma = &dd->dst; + callback = atmel_aes_dma_callback; + config.src_addr = dd->phys_base + AES_ODATAR(0); + break; + default: + return -EINVAL; + } + + err = dmaengine_slave_config(dma->chan, &config); if (err) return err; - /* MR register must be set before IV registers */ - if (dd->ctx->keylen == AES_KEYSIZE_128) - valmr |= AES_MR_KEYSIZE_128; - else if (dd->ctx->keylen == AES_KEYSIZE_192) - valmr |= AES_MR_KEYSIZE_192; - else - valmr |= AES_MR_KEYSIZE_256; + desc = dmaengine_prep_slave_sg(dma->chan, dma->sg, dma->sg_len, dir, + DMA_PREP_INTERRUPT | DMA_CTRL_ACK); + if (!desc) + return -ENOMEM; - if (dd->flags & AES_FLAGS_CBC) { - valmr |= AES_MR_OPMOD_CBC; - } else if (dd->flags & AES_FLAGS_CFB) { - valmr |= AES_MR_OPMOD_CFB; - if (dd->flags & AES_FLAGS_CFB8) - valmr |= AES_MR_CFBS_8b; - else if (dd->flags & AES_FLAGS_CFB16) - valmr |= AES_MR_CFBS_16b; - else if (dd->flags & AES_FLAGS_CFB32) - valmr |= AES_MR_CFBS_32b; - else if (dd->flags & AES_FLAGS_CFB64) - valmr |= AES_MR_CFBS_64b; - else if (dd->flags & AES_FLAGS_CFB128) - valmr |= AES_MR_CFBS_128b; - } else if (dd->flags & AES_FLAGS_OFB) { - valmr |= AES_MR_OPMOD_OFB; - } else if (dd->flags & AES_FLAGS_CTR) { - valmr |= AES_MR_OPMOD_CTR; - } else { - valmr |= AES_MR_OPMOD_ECB; - } + desc->callback = callback; + desc->callback_param = dd; + dmaengine_submit(desc); + dma_async_issue_pending(dma->chan); - if (dd->flags & AES_FLAGS_ENCRYPT) - valmr |= AES_MR_CYPHER_ENC; + return 0; +} - if (dd->total > ATMEL_AES_DMA_THRESHOLD) { - valmr |= AES_MR_SMOD_IDATAR0; - if (dd->caps.has_dualbuff) - valmr |= AES_MR_DUALBUFF; - } else { - valmr |= AES_MR_SMOD_AUTO; +static void atmel_aes_dma_transfer_stop(struct atmel_aes_dev *dd, + enum dma_transfer_direction dir) +{ + struct atmel_aes_dma *dma; + + switch (dir) { + case DMA_MEM_TO_DEV: + dma = &dd->src; + break; + + case DMA_DEV_TO_MEM: + dma = &dd->dst; + break; + + default: + return; } - atmel_aes_write(dd, AES_CR, valcr); - atmel_aes_write(dd, AES_MR, valmr); + dmaengine_terminate_all(dma->chan); +} - atmel_aes_write_n(dd, AES_KEYWR(0), dd->ctx->key, - dd->ctx->keylen >> 2); +static int atmel_aes_dma_start(struct atmel_aes_dev *dd, + struct scatterlist *src, + struct scatterlist *dst, + size_t len, + atmel_aes_fn_t resume) +{ + enum dma_slave_buswidth addr_width; + u32 maxburst; + int err; + + switch (dd->ctx->block_size) { + case CFB8_BLOCK_SIZE: + addr_width = DMA_SLAVE_BUSWIDTH_1_BYTE; + maxburst = 1; + break; + + case CFB16_BLOCK_SIZE: + addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES; + maxburst = 1; + break; + + case CFB32_BLOCK_SIZE: + case CFB64_BLOCK_SIZE: + addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + maxburst = 1; + break; + + case AES_BLOCK_SIZE: + addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; + maxburst = dd->caps.max_burst_size; + break; - if (((dd->flags & AES_FLAGS_CBC) || (dd->flags & AES_FLAGS_CFB) || - (dd->flags & AES_FLAGS_OFB) || (dd->flags & AES_FLAGS_CTR)) && - dd->req->info) { - atmel_aes_write_n(dd, AES_IVR(0), dd->req->info, 4); + default: + err = -EINVAL; + goto exit; } - return 0; + err = atmel_aes_map(dd, src, dst, len); + if (err) + goto exit; + + dd->resume = resume; + + /* Set output DMA transfer first */ + err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_DEV_TO_MEM, + maxburst); + if (err) + goto unmap; + + /* Then set input DMA transfer */ + err = atmel_aes_dma_transfer_start(dd, addr_width, DMA_MEM_TO_DEV, + maxburst); + if (err) + goto output_transfer_stop; + + return -EINPROGRESS; + +output_transfer_stop: + atmel_aes_dma_transfer_stop(dd, DMA_DEV_TO_MEM); +unmap: + atmel_aes_unmap(dd); +exit: + return atmel_aes_complete(dd, err); +} + +static void atmel_aes_dma_stop(struct atmel_aes_dev *dd) +{ + atmel_aes_dma_transfer_stop(dd, DMA_MEM_TO_DEV); + atmel_aes_dma_transfer_stop(dd, DMA_DEV_TO_MEM); + atmel_aes_unmap(dd); +} + +static void atmel_aes_dma_callback(void *data) +{ + struct atmel_aes_dev *dd = data; + + atmel_aes_dma_stop(dd); + dd->is_async = true; + (void)dd->resume(dd); } static int atmel_aes_handle_queue(struct atmel_aes_dev *dd, - struct ablkcipher_request *req) + struct crypto_async_request *new_areq) { - struct crypto_async_request *async_req, *backlog; - struct atmel_aes_ctx *ctx; - struct atmel_aes_reqctx *rctx; + struct crypto_async_request *areq, *backlog; + struct atmel_aes_base_ctx *ctx; unsigned long flags; int err, ret = 0; spin_lock_irqsave(&dd->lock, flags); - if (req) - ret = ablkcipher_enqueue_request(&dd->queue, req); + if (new_areq) + ret = crypto_enqueue_request(&dd->queue, new_areq); if (dd->flags & AES_FLAGS_BUSY) { spin_unlock_irqrestore(&dd->lock, flags); return ret; } backlog = crypto_get_backlog(&dd->queue); - async_req = crypto_dequeue_request(&dd->queue); - if (async_req) + areq = crypto_dequeue_request(&dd->queue); + if (areq) dd->flags |= AES_FLAGS_BUSY; spin_unlock_irqrestore(&dd->lock, flags); - if (!async_req) + if (!areq) return ret; if (backlog) backlog->complete(backlog, -EINPROGRESS); - req = ablkcipher_request_cast(async_req); + ctx = crypto_tfm_ctx(areq->tfm); - /* assign new request to device */ - dd->req = req; - dd->total = req->nbytes; - dd->in_offset = 0; - dd->in_sg = req->src; - dd->out_offset = 0; - dd->out_sg = req->dst; - - rctx = ablkcipher_request_ctx(req); - ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)); - rctx->mode &= AES_FLAGS_MODE_MASK; - dd->flags = (dd->flags & ~AES_FLAGS_MODE_MASK) | rctx->mode; + dd->areq = areq; dd->ctx = ctx; - ctx->dd = dd; + dd->is_async = (areq != new_areq); - err = atmel_aes_write_ctrl(dd); - if (!err) { - if (dd->total > ATMEL_AES_DMA_THRESHOLD) - err = atmel_aes_crypt_dma_start(dd); - else - err = atmel_aes_crypt_cpu_start(dd); - } - if (err) { - /* aes_task will not finish it, so do it here */ - atmel_aes_finish_req(dd, err); - tasklet_schedule(&dd->queue_task); - } - - return ret; + err = ctx->start(dd); + return (dd->is_async) ? ret : err; } -static int atmel_aes_crypt_dma_stop(struct atmel_aes_dev *dd) -{ - int err = -EINVAL; - size_t count; - if (dd->flags & AES_FLAGS_DMA) { - err = 0; - if (dd->flags & AES_FLAGS_FAST) { - dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE); - dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE); - } else { - dma_sync_single_for_cpu(dd->dev, dd->dma_addr_out, - dd->dma_size, DMA_FROM_DEVICE); - - /* copy data */ - count = atmel_aes_sg_copy(&dd->out_sg, &dd->out_offset, - dd->buf_out, dd->buflen, dd->dma_size, 1); - if (count != dd->dma_size) { - err = -EINVAL; - pr_err("not all data converted: %u\n", count); - } - } - } +/* AES async block ciphers */ - return err; +static int atmel_aes_transfer_complete(struct atmel_aes_dev *dd) +{ + return atmel_aes_complete(dd, 0); } - -static int atmel_aes_buff_init(struct atmel_aes_dev *dd) +static int atmel_aes_start(struct atmel_aes_dev *dd) { - int err = -ENOMEM; - - dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, 0); - dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, 0); - dd->buflen = PAGE_SIZE; - dd->buflen &= ~(AES_BLOCK_SIZE - 1); - - if (!dd->buf_in || !dd->buf_out) { - dev_err(dd->dev, "unable to alloc pages.\n"); - goto err_alloc; - } + struct ablkcipher_request *req = ablkcipher_request_cast(dd->areq); + struct atmel_aes_reqctx *rctx = ablkcipher_request_ctx(req); + bool use_dma = (req->nbytes >= ATMEL_AES_DMA_THRESHOLD || + dd->ctx->block_size != AES_BLOCK_SIZE); + int err; - /* MAP here */ - dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in, - dd->buflen, DMA_TO_DEVICE); - if (dma_mapping_error(dd->dev, dd->dma_addr_in)) { - dev_err(dd->dev, "dma %d bytes error\n", dd->buflen); - err = -EINVAL; - goto err_map_in; - } + atmel_aes_set_mode(dd, rctx); - dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out, - dd->buflen, DMA_FROM_DEVICE); - if (dma_mapping_error(dd->dev, dd->dma_addr_out)) { - dev_err(dd->dev, "dma %d bytes error\n", dd->buflen); - err = -EINVAL; - goto err_map_out; - } + err = atmel_aes_hw_init(dd); + if (err) + return atmel_aes_complete(dd, err); - return 0; + atmel_aes_write_ctrl(dd, use_dma, req->info); + if (use_dma) + return atmel_aes_dma_start(dd, req->src, req->dst, req->nbytes, + atmel_aes_transfer_complete); -err_map_out: - dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, - DMA_TO_DEVICE); -err_map_in: -err_alloc: - free_page((unsigned long)dd->buf_out); - free_page((unsigned long)dd->buf_in); - if (err) - pr_err("error: %d\n", err); - return err; + return atmel_aes_cpu_start(dd, req->src, req->dst, req->nbytes, + atmel_aes_transfer_complete); } -static void atmel_aes_buff_cleanup(struct atmel_aes_dev *dd) +static inline struct atmel_aes_ctr_ctx * +atmel_aes_ctr_ctx_cast(struct atmel_aes_base_ctx *ctx) { - dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen, - DMA_FROM_DEVICE); - dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, - DMA_TO_DEVICE); - free_page((unsigned long)dd->buf_out); - free_page((unsigned long)dd->buf_in); + return container_of(ctx, struct atmel_aes_ctr_ctx, base); } -static int atmel_aes_crypt(struct ablkcipher_request *req, unsigned long mode) +static int atmel_aes_ctr_transfer(struct atmel_aes_dev *dd) { - struct atmel_aes_ctx *ctx = crypto_ablkcipher_ctx( - crypto_ablkcipher_reqtfm(req)); - struct atmel_aes_reqctx *rctx = ablkcipher_request_ctx(req); - struct atmel_aes_dev *dd; - - if (mode & AES_FLAGS_CFB8) { - if (!IS_ALIGNED(req->nbytes, CFB8_BLOCK_SIZE)) { - pr_err("request size is not exact amount of CFB8 blocks\n"); - return -EINVAL; - } - ctx->block_size = CFB8_BLOCK_SIZE; - } else if (mode & AES_FLAGS_CFB16) { - if (!IS_ALIGNED(req->nbytes, CFB16_BLOCK_SIZE)) { - pr_err("request size is not exact amount of CFB16 blocks\n"); - return -EINVAL; - } - ctx->block_size = CFB16_BLOCK_SIZE; - } else if (mode & AES_FLAGS_CFB32) { - if (!IS_ALIGNED(req->nbytes, CFB32_BLOCK_SIZE)) { - pr_err("request size is not exact amount of CFB32 blocks\n"); - return -EINVAL; - } - ctx->block_size = CFB32_BLOCK_SIZE; - } else if (mode & AES_FLAGS_CFB64) { - if (!IS_ALIGNED(req->nbytes, CFB64_BLOCK_SIZE)) { - pr_err("request size is not exact amount of CFB64 blocks\n"); - return -EINVAL; + struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx); + struct ablkcipher_request *req = ablkcipher_request_cast(dd->areq); + struct scatterlist *src, *dst; + u32 ctr, blocks; + size_t datalen; + bool use_dma, fragmented = false; + + /* Check for transfer completion. */ + ctx->offset += dd->total; + if (ctx->offset >= req->nbytes) + return atmel_aes_transfer_complete(dd); + + /* Compute data length. */ + datalen = req->nbytes - ctx->offset; + blocks = DIV_ROUND_UP(datalen, AES_BLOCK_SIZE); + ctr = be32_to_cpu(ctx->iv[3]); + if (dd->caps.has_ctr32) { + /* Check 32bit counter overflow. */ + u32 start = ctr; + u32 end = start + blocks - 1; + + if (end < start) { + ctr |= 0xffffffff; + datalen = AES_BLOCK_SIZE * -start; + fragmented = true; } - ctx->block_size = CFB64_BLOCK_SIZE; } else { - if (!IS_ALIGNED(req->nbytes, AES_BLOCK_SIZE)) { - pr_err("request size is not exact amount of AES blocks\n"); - return -EINVAL; + /* Check 16bit counter overflow. */ + u16 start = ctr & 0xffff; + u16 end = start + (u16)blocks - 1; + + if (blocks >> 16 || end < start) { + ctr |= 0xffff; + datalen = AES_BLOCK_SIZE * (0x10000-start); + fragmented = true; } - ctx->block_size = AES_BLOCK_SIZE; + } + use_dma = (datalen >= ATMEL_AES_DMA_THRESHOLD); + + /* Jump to offset. */ + src = scatterwalk_ffwd(ctx->src, req->src, ctx->offset); + dst = ((req->src == req->dst) ? src : + scatterwalk_ffwd(ctx->dst, req->dst, ctx->offset)); + + /* Configure hardware. */ + atmel_aes_write_ctrl(dd, use_dma, ctx->iv); + if (unlikely(fragmented)) { + /* + * Increment the counter manually to cope with the hardware + * counter overflow. + */ + ctx->iv[3] = cpu_to_be32(ctr); + crypto_inc((u8 *)ctx->iv, AES_BLOCK_SIZE); } - dd = atmel_aes_find_dev(ctx); - if (!dd) - return -ENODEV; - - rctx->mode = mode; + if (use_dma) + return atmel_aes_dma_start(dd, src, dst, datalen, + atmel_aes_ctr_transfer); - return atmel_aes_handle_queue(dd, req); + return atmel_aes_cpu_start(dd, src, dst, datalen, + atmel_aes_ctr_transfer); } -static bool atmel_aes_filter(struct dma_chan *chan, void *slave) +static int atmel_aes_ctr_start(struct atmel_aes_dev *dd) { - struct at_dma_slave *sl = slave; + struct atmel_aes_ctr_ctx *ctx = atmel_aes_ctr_ctx_cast(dd->ctx); + struct ablkcipher_request *req = ablkcipher_request_cast(dd->areq); + struct atmel_aes_reqctx *rctx = ablkcipher_request_ctx(req); + int err; - if (sl && sl->dma_dev == chan->device->dev) { - chan->private = sl; - return true; - } else { - return false; - } + atmel_aes_set_mode(dd, rctx); + + err = atmel_aes_hw_init(dd); + if (err) + return atmel_aes_complete(dd, err); + + memcpy(ctx->iv, req->info, AES_BLOCK_SIZE); + ctx->offset = 0; + dd->total = 0; + return atmel_aes_ctr_transfer(dd); } -static int atmel_aes_dma_init(struct atmel_aes_dev *dd, - struct crypto_platform_data *pdata) +static int atmel_aes_crypt(struct ablkcipher_request *req, unsigned long mode) { - int err = -ENOMEM; - dma_cap_mask_t mask; + struct atmel_aes_base_ctx *ctx; + struct atmel_aes_reqctx *rctx; + struct atmel_aes_dev *dd; - dma_cap_zero(mask); - dma_cap_set(DMA_SLAVE, mask); + ctx = crypto_ablkcipher_ctx(crypto_ablkcipher_reqtfm(req)); + switch (mode & AES_FLAGS_OPMODE_MASK) { + case AES_FLAGS_CFB8: + ctx->block_size = CFB8_BLOCK_SIZE; + break; - /* Try to grab 2 DMA channels */ - dd->dma_lch_in.chan = dma_request_slave_channel_compat(mask, - atmel_aes_filter, &pdata->dma_slave->rxdata, dd->dev, "tx"); - if (!dd->dma_lch_in.chan) - goto err_dma_in; + case AES_FLAGS_CFB16: + ctx->block_size = CFB16_BLOCK_SIZE; + break; - dd->dma_lch_in.dma_conf.direction = DMA_MEM_TO_DEV; - dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base + - AES_IDATAR(0); - dd->dma_lch_in.dma_conf.src_maxburst = dd->caps.max_burst_size; - dd->dma_lch_in.dma_conf.src_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_in.dma_conf.dst_maxburst = dd->caps.max_burst_size; - dd->dma_lch_in.dma_conf.dst_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_in.dma_conf.device_fc = false; - - dd->dma_lch_out.chan = dma_request_slave_channel_compat(mask, - atmel_aes_filter, &pdata->dma_slave->txdata, dd->dev, "rx"); - if (!dd->dma_lch_out.chan) - goto err_dma_out; + case AES_FLAGS_CFB32: + ctx->block_size = CFB32_BLOCK_SIZE; + break; - dd->dma_lch_out.dma_conf.direction = DMA_DEV_TO_MEM; - dd->dma_lch_out.dma_conf.src_addr = dd->phys_base + - AES_ODATAR(0); - dd->dma_lch_out.dma_conf.src_maxburst = dd->caps.max_burst_size; - dd->dma_lch_out.dma_conf.src_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_out.dma_conf.dst_maxburst = dd->caps.max_burst_size; - dd->dma_lch_out.dma_conf.dst_addr_width = - DMA_SLAVE_BUSWIDTH_4_BYTES; - dd->dma_lch_out.dma_conf.device_fc = false; + case AES_FLAGS_CFB64: + ctx->block_size = CFB64_BLOCK_SIZE; + break; - return 0; + default: + ctx->block_size = AES_BLOCK_SIZE; + break; + } -err_dma_out: - dma_release_channel(dd->dma_lch_in.chan); -err_dma_in: - dev_warn(dd->dev, "no DMA channel available\n"); - return err; -} + dd = atmel_aes_find_dev(ctx); + if (!dd) + return -ENODEV; -static void atmel_aes_dma_cleanup(struct atmel_aes_dev *dd) -{ - dma_release_channel(dd->dma_lch_in.chan); - dma_release_channel(dd->dma_lch_out.chan); + rctx = ablkcipher_request_ctx(req); + rctx->mode = mode; + + return atmel_aes_handle_queue(dd, &req->base); } static int atmel_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, unsigned int keylen) { - struct atmel_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct atmel_aes_base_ctx *ctx = crypto_ablkcipher_ctx(tfm); - if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && - keylen != AES_KEYSIZE_256) { + if (keylen != AES_KEYSIZE_128 && + keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) { crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } @@ -839,115 +1066,110 @@ static int atmel_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, static int atmel_aes_ecb_encrypt(struct ablkcipher_request *req) { - return atmel_aes_crypt(req, - AES_FLAGS_ENCRYPT); + return atmel_aes_crypt(req, AES_FLAGS_ECB | AES_FLAGS_ENCRYPT); } static int atmel_aes_ecb_decrypt(struct ablkcipher_request *req) { - return atmel_aes_crypt(req, - 0); + return atmel_aes_crypt(req, AES_FLAGS_ECB); } static int atmel_aes_cbc_encrypt(struct ablkcipher_request *req) { - return atmel_aes_crypt(req, - AES_FLAGS_ENCRYPT | AES_FLAGS_CBC); + return atmel_aes_crypt(req, AES_FLAGS_CBC | AES_FLAGS_ENCRYPT); } static int atmel_aes_cbc_decrypt(struct ablkcipher_request *req) { - return atmel_aes_crypt(req, - AES_FLAGS_CBC); + return atmel_aes_crypt(req, AES_FLAGS_CBC); } static int atmel_aes_ofb_encrypt(struct ablkcipher_request *req) { - return atmel_aes_crypt(req, - AES_FLAGS_ENCRYPT | AES_FLAGS_OFB); + return atmel_aes_crypt(req, AES_FLAGS_OFB | AES_FLAGS_ENCRYPT); } static int atmel_aes_ofb_decrypt(struct ablkcipher_request *req) { - return atmel_aes_crypt(req, - AES_FLAGS_OFB); + return atmel_aes_crypt(req, AES_FLAGS_OFB); } static int atmel_aes_cfb_encrypt(struct ablkcipher_request *req) { - return atmel_aes_crypt(req, - AES_FLAGS_ENCRYPT | AES_FLAGS_CFB | AES_FLAGS_CFB128); + return atmel_aes_crypt(req, AES_FLAGS_CFB128 | AES_FLAGS_ENCRYPT); } static int atmel_aes_cfb_decrypt(struct ablkcipher_request *req) { - return atmel_aes_crypt(req, - AES_FLAGS_CFB | AES_FLAGS_CFB128); + return atmel_aes_crypt(req, AES_FLAGS_CFB128); } static int atmel_aes_cfb64_encrypt(struct ablkcipher_request *req) { - return atmel_aes_crypt(req, - AES_FLAGS_ENCRYPT | AES_FLAGS_CFB | AES_FLAGS_CFB64); + return atmel_aes_crypt(req, AES_FLAGS_CFB64 | AES_FLAGS_ENCRYPT); } static int atmel_aes_cfb64_decrypt(struct ablkcipher_request *req) { - return atmel_aes_crypt(req, - AES_FLAGS_CFB | AES_FLAGS_CFB64); + return atmel_aes_crypt(req, AES_FLAGS_CFB64); } static int atmel_aes_cfb32_encrypt(struct ablkcipher_request *req) { - return atmel_aes_crypt(req, - AES_FLAGS_ENCRYPT | AES_FLAGS_CFB | AES_FLAGS_CFB32); + return atmel_aes_crypt(req, AES_FLAGS_CFB32 | AES_FLAGS_ENCRYPT); } static int atmel_aes_cfb32_decrypt(struct ablkcipher_request *req) { - return atmel_aes_crypt(req, - AES_FLAGS_CFB | AES_FLAGS_CFB32); + return atmel_aes_crypt(req, AES_FLAGS_CFB32); } static int atmel_aes_cfb16_encrypt(struct ablkcipher_request *req) { - return atmel_aes_crypt(req, - AES_FLAGS_ENCRYPT | AES_FLAGS_CFB | AES_FLAGS_CFB16); + return atmel_aes_crypt(req, AES_FLAGS_CFB16 | AES_FLAGS_ENCRYPT); } static int atmel_aes_cfb16_decrypt(struct ablkcipher_request *req) { - return atmel_aes_crypt(req, - AES_FLAGS_CFB | AES_FLAGS_CFB16); + return atmel_aes_crypt(req, AES_FLAGS_CFB16); } static int atmel_aes_cfb8_encrypt(struct ablkcipher_request *req) { - return atmel_aes_crypt(req, - AES_FLAGS_ENCRYPT | AES_FLAGS_CFB | AES_FLAGS_CFB8); + return atmel_aes_crypt(req, AES_FLAGS_CFB8 | AES_FLAGS_ENCRYPT); } static int atmel_aes_cfb8_decrypt(struct ablkcipher_request *req) { - return atmel_aes_crypt(req, - AES_FLAGS_CFB | AES_FLAGS_CFB8); + return atmel_aes_crypt(req, AES_FLAGS_CFB8); } static int atmel_aes_ctr_encrypt(struct ablkcipher_request *req) { - return atmel_aes_crypt(req, - AES_FLAGS_ENCRYPT | AES_FLAGS_CTR); + return atmel_aes_crypt(req, AES_FLAGS_CTR | AES_FLAGS_ENCRYPT); } static int atmel_aes_ctr_decrypt(struct ablkcipher_request *req) { - return atmel_aes_crypt(req, - AES_FLAGS_CTR); + return atmel_aes_crypt(req, AES_FLAGS_CTR); } static int atmel_aes_cra_init(struct crypto_tfm *tfm) { + struct atmel_aes_ctx *ctx = crypto_tfm_ctx(tfm); + + tfm->crt_ablkcipher.reqsize = sizeof(struct atmel_aes_reqctx); + ctx->base.start = atmel_aes_start; + + return 0; +} + +static int atmel_aes_ctr_cra_init(struct crypto_tfm *tfm) +{ + struct atmel_aes_ctx *ctx = crypto_tfm_ctx(tfm); + tfm->crt_ablkcipher.reqsize = sizeof(struct atmel_aes_reqctx); + ctx->base.start = atmel_aes_ctr_start; return 0; } @@ -960,7 +1182,7 @@ static struct crypto_alg aes_algs[] = { { .cra_name = "ecb(aes)", .cra_driver_name = "atmel-ecb-aes", - .cra_priority = 100, + .cra_priority = ATMEL_AES_PRIORITY, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_aes_ctx), @@ -980,7 +1202,7 @@ static struct crypto_alg aes_algs[] = { { .cra_name = "cbc(aes)", .cra_driver_name = "atmel-cbc-aes", - .cra_priority = 100, + .cra_priority = ATMEL_AES_PRIORITY, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_aes_ctx), @@ -1001,7 +1223,7 @@ static struct crypto_alg aes_algs[] = { { .cra_name = "ofb(aes)", .cra_driver_name = "atmel-ofb-aes", - .cra_priority = 100, + .cra_priority = ATMEL_AES_PRIORITY, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_aes_ctx), @@ -1022,7 +1244,7 @@ static struct crypto_alg aes_algs[] = { { .cra_name = "cfb(aes)", .cra_driver_name = "atmel-cfb-aes", - .cra_priority = 100, + .cra_priority = ATMEL_AES_PRIORITY, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_aes_ctx), @@ -1043,7 +1265,7 @@ static struct crypto_alg aes_algs[] = { { .cra_name = "cfb32(aes)", .cra_driver_name = "atmel-cfb32-aes", - .cra_priority = 100, + .cra_priority = ATMEL_AES_PRIORITY, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = CFB32_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_aes_ctx), @@ -1064,7 +1286,7 @@ static struct crypto_alg aes_algs[] = { { .cra_name = "cfb16(aes)", .cra_driver_name = "atmel-cfb16-aes", - .cra_priority = 100, + .cra_priority = ATMEL_AES_PRIORITY, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = CFB16_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_aes_ctx), @@ -1085,7 +1307,7 @@ static struct crypto_alg aes_algs[] = { { .cra_name = "cfb8(aes)", .cra_driver_name = "atmel-cfb8-aes", - .cra_priority = 100, + .cra_priority = ATMEL_AES_PRIORITY, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = CFB8_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_aes_ctx), @@ -1106,14 +1328,14 @@ static struct crypto_alg aes_algs[] = { { .cra_name = "ctr(aes)", .cra_driver_name = "atmel-ctr-aes", - .cra_priority = 100, + .cra_priority = ATMEL_AES_PRIORITY, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, - .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct atmel_aes_ctx), + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct atmel_aes_ctr_ctx), .cra_alignmask = 0xf, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, - .cra_init = atmel_aes_cra_init, + .cra_init = atmel_aes_ctr_cra_init, .cra_exit = atmel_aes_cra_exit, .cra_u.ablkcipher = { .min_keysize = AES_MIN_KEY_SIZE, @@ -1129,7 +1351,7 @@ static struct crypto_alg aes_algs[] = { static struct crypto_alg aes_cfb64_alg = { .cra_name = "cfb64(aes)", .cra_driver_name = "atmel-cfb64-aes", - .cra_priority = 100, + .cra_priority = ATMEL_AES_PRIORITY, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = CFB64_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_aes_ctx), @@ -1148,53 +1370,496 @@ static struct crypto_alg aes_cfb64_alg = { } }; -static void atmel_aes_queue_task(unsigned long data) + +/* gcm aead functions */ + +static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd, + const u32 *data, size_t datalen, + const u32 *ghash_in, u32 *ghash_out, + atmel_aes_fn_t resume); +static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd); +static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd); + +static int atmel_aes_gcm_start(struct atmel_aes_dev *dd); +static int atmel_aes_gcm_process(struct atmel_aes_dev *dd); +static int atmel_aes_gcm_length(struct atmel_aes_dev *dd); +static int atmel_aes_gcm_data(struct atmel_aes_dev *dd); +static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd); +static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd); +static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd); + +static inline struct atmel_aes_gcm_ctx * +atmel_aes_gcm_ctx_cast(struct atmel_aes_base_ctx *ctx) { - struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data; + return container_of(ctx, struct atmel_aes_gcm_ctx, base); +} - atmel_aes_handle_queue(dd, NULL); +static int atmel_aes_gcm_ghash(struct atmel_aes_dev *dd, + const u32 *data, size_t datalen, + const u32 *ghash_in, u32 *ghash_out, + atmel_aes_fn_t resume) +{ + struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); + + dd->data = (u32 *)data; + dd->datalen = datalen; + ctx->ghash_in = ghash_in; + ctx->ghash_out = ghash_out; + ctx->ghash_resume = resume; + + atmel_aes_write_ctrl(dd, false, NULL); + return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_ghash_init); } -static void atmel_aes_done_task(unsigned long data) +static int atmel_aes_gcm_ghash_init(struct atmel_aes_dev *dd) { - struct atmel_aes_dev *dd = (struct atmel_aes_dev *) data; + struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); + + /* Set the data length. */ + atmel_aes_write(dd, AES_AADLENR, dd->total); + atmel_aes_write(dd, AES_CLENR, 0); + + /* If needed, overwrite the GCM Intermediate Hash Word Registers */ + if (ctx->ghash_in) + atmel_aes_write_block(dd, AES_GHASHR(0), ctx->ghash_in); + + return atmel_aes_gcm_ghash_finalize(dd); +} + +static int atmel_aes_gcm_ghash_finalize(struct atmel_aes_dev *dd) +{ + struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); + u32 isr; + + /* Write data into the Input Data Registers. */ + while (dd->datalen > 0) { + atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); + dd->data += 4; + dd->datalen -= AES_BLOCK_SIZE; + + isr = atmel_aes_read(dd, AES_ISR); + if (!(isr & AES_INT_DATARDY)) { + dd->resume = atmel_aes_gcm_ghash_finalize; + atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); + return -EINPROGRESS; + } + } + + /* Read the computed hash from GHASHRx. */ + atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash_out); + + return ctx->ghash_resume(dd); +} + + +static int atmel_aes_gcm_start(struct atmel_aes_dev *dd) +{ + struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); + struct aead_request *req = aead_request_cast(dd->areq); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + struct atmel_aes_reqctx *rctx = aead_request_ctx(req); + size_t ivsize = crypto_aead_ivsize(tfm); + size_t datalen, padlen; + const void *iv = req->iv; + u8 *data = dd->buf; int err; - if (!(dd->flags & AES_FLAGS_DMA)) { - atmel_aes_read_n(dd, AES_ODATAR(0), (u32 *) dd->buf_out, - dd->bufcnt >> 2); + atmel_aes_set_mode(dd, rctx); - if (sg_copy_from_buffer(dd->out_sg, dd->nb_out_sg, - dd->buf_out, dd->bufcnt)) - err = 0; - else - err = -EINVAL; + err = atmel_aes_hw_init(dd); + if (err) + return atmel_aes_complete(dd, err); + + if (likely(ivsize == 12)) { + memcpy(ctx->j0, iv, ivsize); + ctx->j0[3] = cpu_to_be32(1); + return atmel_aes_gcm_process(dd); + } + + padlen = atmel_aes_padlen(ivsize, AES_BLOCK_SIZE); + datalen = ivsize + padlen + AES_BLOCK_SIZE; + if (datalen > dd->buflen) + return atmel_aes_complete(dd, -EINVAL); + + memcpy(data, iv, ivsize); + memset(data + ivsize, 0, padlen + sizeof(u64)); + ((u64 *)(data + datalen))[-1] = cpu_to_be64(ivsize * 8); + + return atmel_aes_gcm_ghash(dd, (const u32 *)data, datalen, + NULL, ctx->j0, atmel_aes_gcm_process); +} - goto cpu_end; +static int atmel_aes_gcm_process(struct atmel_aes_dev *dd) +{ + struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); + struct aead_request *req = aead_request_cast(dd->areq); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + bool enc = atmel_aes_is_encrypt(dd); + u32 authsize; + + /* Compute text length. */ + authsize = crypto_aead_authsize(tfm); + ctx->textlen = req->cryptlen - (enc ? 0 : authsize); + + /* + * According to tcrypt test suite, the GCM Automatic Tag Generation + * fails when both the message and its associated data are empty. + */ + if (likely(req->assoclen != 0 || ctx->textlen != 0)) + dd->flags |= AES_FLAGS_GTAGEN; + + atmel_aes_write_ctrl(dd, false, NULL); + return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_length); +} + +static int atmel_aes_gcm_length(struct atmel_aes_dev *dd) +{ + struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); + struct aead_request *req = aead_request_cast(dd->areq); + u32 j0_lsw, *j0 = ctx->j0; + size_t padlen; + + /* Write incr32(J0) into IV. */ + j0_lsw = j0[3]; + j0[3] = cpu_to_be32(be32_to_cpu(j0[3]) + 1); + atmel_aes_write_block(dd, AES_IVR(0), j0); + j0[3] = j0_lsw; + + /* Set aad and text lengths. */ + atmel_aes_write(dd, AES_AADLENR, req->assoclen); + atmel_aes_write(dd, AES_CLENR, ctx->textlen); + + /* Check whether AAD are present. */ + if (unlikely(req->assoclen == 0)) { + dd->datalen = 0; + return atmel_aes_gcm_data(dd); } - err = atmel_aes_crypt_dma_stop(dd); + /* Copy assoc data and add padding. */ + padlen = atmel_aes_padlen(req->assoclen, AES_BLOCK_SIZE); + if (unlikely(req->assoclen + padlen > dd->buflen)) + return atmel_aes_complete(dd, -EINVAL); + sg_copy_to_buffer(req->src, sg_nents(req->src), dd->buf, req->assoclen); - err = dd->err ? : err; + /* Write assoc data into the Input Data register. */ + dd->data = (u32 *)dd->buf; + dd->datalen = req->assoclen + padlen; + return atmel_aes_gcm_data(dd); +} - if (dd->total && !err) { - if (dd->flags & AES_FLAGS_FAST) { - dd->in_sg = sg_next(dd->in_sg); - dd->out_sg = sg_next(dd->out_sg); - if (!dd->in_sg || !dd->out_sg) - err = -EINVAL; +static int atmel_aes_gcm_data(struct atmel_aes_dev *dd) +{ + struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); + struct aead_request *req = aead_request_cast(dd->areq); + bool use_dma = (ctx->textlen >= ATMEL_AES_DMA_THRESHOLD); + struct scatterlist *src, *dst; + u32 isr, mr; + + /* Write AAD first. */ + while (dd->datalen > 0) { + atmel_aes_write_block(dd, AES_IDATAR(0), dd->data); + dd->data += 4; + dd->datalen -= AES_BLOCK_SIZE; + + isr = atmel_aes_read(dd, AES_ISR); + if (!(isr & AES_INT_DATARDY)) { + dd->resume = atmel_aes_gcm_data; + atmel_aes_write(dd, AES_IER, AES_INT_DATARDY); + return -EINPROGRESS; } - if (!err) - err = atmel_aes_crypt_dma_start(dd); - if (!err) - return; /* DMA started. Not fininishing. */ } -cpu_end: - atmel_aes_finish_req(dd, err); + /* GMAC only. */ + if (unlikely(ctx->textlen == 0)) + return atmel_aes_gcm_tag_init(dd); + + /* Prepare src and dst scatter lists to transfer cipher/plain texts */ + src = scatterwalk_ffwd(ctx->src, req->src, req->assoclen); + dst = ((req->src == req->dst) ? src : + scatterwalk_ffwd(ctx->dst, req->dst, req->assoclen)); + + if (use_dma) { + /* Update the Mode Register for DMA transfers. */ + mr = atmel_aes_read(dd, AES_MR); + mr &= ~(AES_MR_SMOD_MASK | AES_MR_DUALBUFF); + mr |= AES_MR_SMOD_IDATAR0; + if (dd->caps.has_dualbuff) + mr |= AES_MR_DUALBUFF; + atmel_aes_write(dd, AES_MR, mr); + + return atmel_aes_dma_start(dd, src, dst, ctx->textlen, + atmel_aes_gcm_tag_init); + } + + return atmel_aes_cpu_start(dd, src, dst, ctx->textlen, + atmel_aes_gcm_tag_init); +} + +static int atmel_aes_gcm_tag_init(struct atmel_aes_dev *dd) +{ + struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); + struct aead_request *req = aead_request_cast(dd->areq); + u64 *data = dd->buf; + + if (likely(dd->flags & AES_FLAGS_GTAGEN)) { + if (!(atmel_aes_read(dd, AES_ISR) & AES_INT_TAGRDY)) { + dd->resume = atmel_aes_gcm_tag_init; + atmel_aes_write(dd, AES_IER, AES_INT_TAGRDY); + return -EINPROGRESS; + } + + return atmel_aes_gcm_finalize(dd); + } + + /* Read the GCM Intermediate Hash Word Registers. */ + atmel_aes_read_block(dd, AES_GHASHR(0), ctx->ghash); + + data[0] = cpu_to_be64(req->assoclen * 8); + data[1] = cpu_to_be64(ctx->textlen * 8); + + return atmel_aes_gcm_ghash(dd, (const u32 *)data, AES_BLOCK_SIZE, + ctx->ghash, ctx->ghash, atmel_aes_gcm_tag); +} + +static int atmel_aes_gcm_tag(struct atmel_aes_dev *dd) +{ + struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); + unsigned long flags; + + /* + * Change mode to CTR to complete the tag generation. + * Use J0 as Initialization Vector. + */ + flags = dd->flags; + dd->flags &= ~(AES_FLAGS_OPMODE_MASK | AES_FLAGS_GTAGEN); + dd->flags |= AES_FLAGS_CTR; + atmel_aes_write_ctrl(dd, false, ctx->j0); + dd->flags = flags; + + atmel_aes_write_block(dd, AES_IDATAR(0), ctx->ghash); + return atmel_aes_wait_for_data_ready(dd, atmel_aes_gcm_finalize); +} + +static int atmel_aes_gcm_finalize(struct atmel_aes_dev *dd) +{ + struct atmel_aes_gcm_ctx *ctx = atmel_aes_gcm_ctx_cast(dd->ctx); + struct aead_request *req = aead_request_cast(dd->areq); + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + bool enc = atmel_aes_is_encrypt(dd); + u32 offset, authsize, itag[4], *otag = ctx->tag; + int err; + + /* Read the computed tag. */ + if (likely(dd->flags & AES_FLAGS_GTAGEN)) + atmel_aes_read_block(dd, AES_TAGR(0), ctx->tag); + else + atmel_aes_read_block(dd, AES_ODATAR(0), ctx->tag); + + offset = req->assoclen + ctx->textlen; + authsize = crypto_aead_authsize(tfm); + if (enc) { + scatterwalk_map_and_copy(otag, req->dst, offset, authsize, 1); + err = 0; + } else { + scatterwalk_map_and_copy(itag, req->src, offset, authsize, 0); + err = crypto_memneq(itag, otag, authsize) ? -EBADMSG : 0; + } + + return atmel_aes_complete(dd, err); +} + +static int atmel_aes_gcm_crypt(struct aead_request *req, + unsigned long mode) +{ + struct atmel_aes_base_ctx *ctx; + struct atmel_aes_reqctx *rctx; + struct atmel_aes_dev *dd; + + ctx = crypto_aead_ctx(crypto_aead_reqtfm(req)); + ctx->block_size = AES_BLOCK_SIZE; + + dd = atmel_aes_find_dev(ctx); + if (!dd) + return -ENODEV; + + rctx = aead_request_ctx(req); + rctx->mode = AES_FLAGS_GCM | mode; + + return atmel_aes_handle_queue(dd, &req->base); +} + +static int atmel_aes_gcm_setkey(struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + struct atmel_aes_base_ctx *ctx = crypto_aead_ctx(tfm); + + if (keylen != AES_KEYSIZE_256 && + keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_128) { + crypto_aead_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + memcpy(ctx->key, key, keylen); + ctx->keylen = keylen; + + return 0; +} + +static int atmel_aes_gcm_setauthsize(struct crypto_aead *tfm, + unsigned int authsize) +{ + /* Same as crypto_gcm_authsize() from crypto/gcm.c */ + switch (authsize) { + case 4: + case 8: + case 12: + case 13: + case 14: + case 15: + case 16: + break; + default: + return -EINVAL; + } + + return 0; +} + +static int atmel_aes_gcm_encrypt(struct aead_request *req) +{ + return atmel_aes_gcm_crypt(req, AES_FLAGS_ENCRYPT); +} + +static int atmel_aes_gcm_decrypt(struct aead_request *req) +{ + return atmel_aes_gcm_crypt(req, 0); +} + +static int atmel_aes_gcm_init(struct crypto_aead *tfm) +{ + struct atmel_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); + + crypto_aead_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx)); + ctx->base.start = atmel_aes_gcm_start; + + return 0; +} + +static void atmel_aes_gcm_exit(struct crypto_aead *tfm) +{ + +} + +static struct aead_alg aes_gcm_alg = { + .setkey = atmel_aes_gcm_setkey, + .setauthsize = atmel_aes_gcm_setauthsize, + .encrypt = atmel_aes_gcm_encrypt, + .decrypt = atmel_aes_gcm_decrypt, + .init = atmel_aes_gcm_init, + .exit = atmel_aes_gcm_exit, + .ivsize = 12, + .maxauthsize = AES_BLOCK_SIZE, + + .base = { + .cra_name = "gcm(aes)", + .cra_driver_name = "atmel-gcm-aes", + .cra_priority = ATMEL_AES_PRIORITY, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct atmel_aes_gcm_ctx), + .cra_alignmask = 0xf, + .cra_module = THIS_MODULE, + }, +}; + + +/* Probe functions */ + +static int atmel_aes_buff_init(struct atmel_aes_dev *dd) +{ + dd->buf = (void *)__get_free_pages(GFP_KERNEL, ATMEL_AES_BUFFER_ORDER); + dd->buflen = ATMEL_AES_BUFFER_SIZE; + dd->buflen &= ~(AES_BLOCK_SIZE - 1); + + if (!dd->buf) { + dev_err(dd->dev, "unable to alloc pages.\n"); + return -ENOMEM; + } + + return 0; +} + +static void atmel_aes_buff_cleanup(struct atmel_aes_dev *dd) +{ + free_page((unsigned long)dd->buf); +} + +static bool atmel_aes_filter(struct dma_chan *chan, void *slave) +{ + struct at_dma_slave *sl = slave; + + if (sl && sl->dma_dev == chan->device->dev) { + chan->private = sl; + return true; + } else { + return false; + } +} + +static int atmel_aes_dma_init(struct atmel_aes_dev *dd, + struct crypto_platform_data *pdata) +{ + struct at_dma_slave *slave; + int err = -ENOMEM; + dma_cap_mask_t mask; + + dma_cap_zero(mask); + dma_cap_set(DMA_SLAVE, mask); + + /* Try to grab 2 DMA channels */ + slave = &pdata->dma_slave->rxdata; + dd->src.chan = dma_request_slave_channel_compat(mask, atmel_aes_filter, + slave, dd->dev, "tx"); + if (!dd->src.chan) + goto err_dma_in; + + slave = &pdata->dma_slave->txdata; + dd->dst.chan = dma_request_slave_channel_compat(mask, atmel_aes_filter, + slave, dd->dev, "rx"); + if (!dd->dst.chan) + goto err_dma_out; + + return 0; + +err_dma_out: + dma_release_channel(dd->src.chan); +err_dma_in: + dev_warn(dd->dev, "no DMA channel available\n"); + return err; +} + +static void atmel_aes_dma_cleanup(struct atmel_aes_dev *dd) +{ + dma_release_channel(dd->dst.chan); + dma_release_channel(dd->src.chan); +} + +static void atmel_aes_queue_task(unsigned long data) +{ + struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data; + atmel_aes_handle_queue(dd, NULL); } +static void atmel_aes_done_task(unsigned long data) +{ + struct atmel_aes_dev *dd = (struct atmel_aes_dev *)data; + + dd->is_async = true; + (void)dd->resume(dd); +} + static irqreturn_t atmel_aes_irq(int irq, void *dev_id) { struct atmel_aes_dev *aes_dd = dev_id; @@ -1217,10 +1882,14 @@ static void atmel_aes_unregister_algs(struct atmel_aes_dev *dd) { int i; - for (i = 0; i < ARRAY_SIZE(aes_algs); i++) - crypto_unregister_alg(&aes_algs[i]); + if (dd->caps.has_gcm) + crypto_unregister_aead(&aes_gcm_alg); + if (dd->caps.has_cfb64) crypto_unregister_alg(&aes_cfb64_alg); + + for (i = 0; i < ARRAY_SIZE(aes_algs); i++) + crypto_unregister_alg(&aes_algs[i]); } static int atmel_aes_register_algs(struct atmel_aes_dev *dd) @@ -1239,8 +1908,16 @@ static int atmel_aes_register_algs(struct atmel_aes_dev *dd) goto err_aes_cfb64_alg; } + if (dd->caps.has_gcm) { + err = crypto_register_aead(&aes_gcm_alg); + if (err) + goto err_aes_gcm_alg; + } + return 0; +err_aes_gcm_alg: + crypto_unregister_alg(&aes_cfb64_alg); err_aes_cfb64_alg: i = ARRAY_SIZE(aes_algs); err_aes_algs: @@ -1254,13 +1931,24 @@ static void atmel_aes_get_cap(struct atmel_aes_dev *dd) { dd->caps.has_dualbuff = 0; dd->caps.has_cfb64 = 0; + dd->caps.has_ctr32 = 0; + dd->caps.has_gcm = 0; dd->caps.max_burst_size = 1; /* keep only major version number */ switch (dd->hw_version & 0xff0) { + case 0x500: + dd->caps.has_dualbuff = 1; + dd->caps.has_cfb64 = 1; + dd->caps.has_ctr32 = 1; + dd->caps.has_gcm = 1; + dd->caps.max_burst_size = 4; + break; case 0x200: dd->caps.has_dualbuff = 1; dd->caps.has_cfb64 = 1; + dd->caps.has_ctr32 = 1; + dd->caps.has_gcm = 1; dd->caps.max_burst_size = 4; break; case 0x130: @@ -1402,7 +2090,9 @@ static int atmel_aes_probe(struct platform_device *pdev) goto res_err; } - atmel_aes_hw_version_init(aes_dd); + err = atmel_aes_hw_version_init(aes_dd); + if (err) + goto res_err; atmel_aes_get_cap(aes_dd); @@ -1423,8 +2113,8 @@ static int atmel_aes_probe(struct platform_device *pdev) goto err_algs; dev_info(dev, "Atmel AES - Using %s, %s for DMA transfers\n", - dma_chan_name(aes_dd->dma_lch_in.chan), - dma_chan_name(aes_dd->dma_lch_out.chan)); + dma_chan_name(aes_dd->src.chan), + dma_chan_name(aes_dd->dst.chan)); return 0; @@ -1462,6 +2152,7 @@ static int atmel_aes_remove(struct platform_device *pdev) tasklet_kill(&aes_dd->queue_task); atmel_aes_dma_cleanup(aes_dd); + atmel_aes_buff_cleanup(aes_dd); return 0; } diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index 660d8c06540b..20de861aa0ea 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -755,7 +755,6 @@ static int atmel_sha_finish(struct ahash_request *req) { struct atmel_sha_reqctx *ctx = ahash_request_ctx(req); struct atmel_sha_dev *dd = ctx->dd; - int err = 0; if (ctx->digcnt[0] || ctx->digcnt[1]) atmel_sha_copy_ready_hash(req); @@ -763,7 +762,7 @@ static int atmel_sha_finish(struct ahash_request *req) dev_dbg(dd->dev, "digcnt: 0x%llx 0x%llx, bufcnt: %d\n", ctx->digcnt[1], ctx->digcnt[0], ctx->bufcnt); - return err; + return 0; } static void atmel_sha_finish_req(struct ahash_request *req, int err) diff --git a/drivers/crypto/caam/caamhash.c b/drivers/crypto/caam/caamhash.c index 49106ea42887..5845d4a08797 100644 --- a/drivers/crypto/caam/caamhash.c +++ b/drivers/crypto/caam/caamhash.c @@ -803,6 +803,10 @@ static int ahash_update_ctx(struct ahash_request *req) if (to_hash) { src_nents = sg_nents_for_len(req->src, req->nbytes - (*next_buflen)); + if (src_nents < 0) { + dev_err(jrdev, "Invalid number of src SG.\n"); + return src_nents; + } sec4_sg_src_index = 1 + (*buflen ? 1 : 0); sec4_sg_bytes = (sec4_sg_src_index + src_nents) * sizeof(struct sec4_sg_entry); @@ -1002,6 +1006,10 @@ static int ahash_finup_ctx(struct ahash_request *req) int sh_len; src_nents = sg_nents_for_len(req->src, req->nbytes); + if (src_nents < 0) { + dev_err(jrdev, "Invalid number of src SG.\n"); + return src_nents; + } sec4_sg_src_index = 1 + (buflen ? 1 : 0); sec4_sg_bytes = (sec4_sg_src_index + src_nents) * sizeof(struct sec4_sg_entry); @@ -1086,6 +1094,10 @@ static int ahash_digest(struct ahash_request *req) int sh_len; src_nents = sg_count(req->src, req->nbytes); + if (src_nents < 0) { + dev_err(jrdev, "Invalid number of src SG.\n"); + return src_nents; + } dma_map_sg(jrdev, req->src, src_nents ? : 1, DMA_TO_DEVICE); sec4_sg_bytes = src_nents * sizeof(struct sec4_sg_entry); @@ -1234,6 +1246,10 @@ static int ahash_update_no_ctx(struct ahash_request *req) if (to_hash) { src_nents = sg_nents_for_len(req->src, req->nbytes - (*next_buflen)); + if (src_nents < 0) { + dev_err(jrdev, "Invalid number of src SG.\n"); + return src_nents; + } sec4_sg_bytes = (1 + src_nents) * sizeof(struct sec4_sg_entry); @@ -1342,6 +1358,10 @@ static int ahash_finup_no_ctx(struct ahash_request *req) int ret = 0; src_nents = sg_nents_for_len(req->src, req->nbytes); + if (src_nents < 0) { + dev_err(jrdev, "Invalid number of src SG.\n"); + return src_nents; + } sec4_sg_src_index = 2; sec4_sg_bytes = (sec4_sg_src_index + src_nents) * sizeof(struct sec4_sg_entry); @@ -1430,6 +1450,10 @@ static int ahash_update_first(struct ahash_request *req) if (to_hash) { src_nents = sg_count(req->src, req->nbytes - (*next_buflen)); + if (src_nents < 0) { + dev_err(jrdev, "Invalid number of src SG.\n"); + return src_nents; + } dma_map_sg(jrdev, req->src, src_nents ? : 1, DMA_TO_DEVICE); sec4_sg_bytes = src_nents * sizeof(struct sec4_sg_entry); @@ -1572,7 +1596,7 @@ static int ahash_export(struct ahash_request *req, void *out) len = state->buflen_1; } else { buf = state->buf_0; - len = state->buflen_1; + len = state->buflen_0; } memcpy(export->buf, buf, len); diff --git a/drivers/crypto/ccp/Kconfig b/drivers/crypto/ccp/Kconfig index 3cd8481065f8..6e37845abf8f 100644 --- a/drivers/crypto/ccp/Kconfig +++ b/drivers/crypto/ccp/Kconfig @@ -3,6 +3,8 @@ config CRYPTO_DEV_CCP_DD depends on CRYPTO_DEV_CCP default m select HW_RANDOM + select CRYPTO_SHA1 + select CRYPTO_SHA256 help Provides the interface to use the AMD Cryptographic Coprocessor which can be used to offload encryption operations such as SHA, diff --git a/drivers/crypto/ccp/ccp-ops.c b/drivers/crypto/ccp/ccp-ops.c index c6e883b296a9..6613aee79b87 100644 --- a/drivers/crypto/ccp/ccp-ops.c +++ b/drivers/crypto/ccp/ccp-ops.c @@ -152,32 +152,6 @@ static const __be32 ccp_sha256_init[CCP_SHA_CTXSIZE / sizeof(__be32)] = { cpu_to_be32(SHA256_H6), cpu_to_be32(SHA256_H7), }; -/* The CCP cannot perform zero-length sha operations so the caller - * is required to buffer data for the final operation. However, a - * sha operation for a message with a total length of zero is valid - * so known values are required to supply the result. - */ -static const u8 ccp_sha1_zero[CCP_SHA_CTXSIZE] = { - 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, - 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, - 0xaf, 0xd8, 0x07, 0x09, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, -}; - -static const u8 ccp_sha224_zero[CCP_SHA_CTXSIZE] = { - 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, - 0x47, 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, - 0x15, 0xa2, 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, - 0xc5, 0xb3, 0xe4, 0x2f, 0x00, 0x00, 0x00, 0x00, -}; - -static const u8 ccp_sha256_zero[CCP_SHA_CTXSIZE] = { - 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, - 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, - 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, - 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55, -}; - static u32 ccp_addr_lo(struct ccp_dma_info *info) { return lower_32_bits(info->address + info->offset); @@ -1391,18 +1365,21 @@ static int ccp_run_sha_cmd(struct ccp_cmd_queue *cmd_q, struct ccp_cmd *cmd) if (sha->msg_bits) return -EINVAL; - /* A sha operation for a message with a total length of zero, - * return known result. + /* The CCP cannot perform zero-length sha operations so the + * caller is required to buffer data for the final operation. + * However, a sha operation for a message with a total length + * of zero is valid so known values are required to supply + * the result. */ switch (sha->type) { case CCP_SHA_TYPE_1: - sha_zero = ccp_sha1_zero; + sha_zero = sha1_zero_message_hash; break; case CCP_SHA_TYPE_224: - sha_zero = ccp_sha224_zero; + sha_zero = sha224_zero_message_hash; break; case CCP_SHA_TYPE_256: - sha_zero = ccp_sha256_zero; + sha_zero = sha256_zero_message_hash; break; default: return -EINVAL; diff --git a/drivers/crypto/ccp/ccp-pci.c b/drivers/crypto/ccp/ccp-pci.c index 6ade02f04f91..7690467c42f8 100644 --- a/drivers/crypto/ccp/ccp-pci.c +++ b/drivers/crypto/ccp/ccp-pci.c @@ -44,7 +44,7 @@ static int ccp_get_msix_irqs(struct ccp_device *ccp) { struct ccp_pci *ccp_pci = ccp->dev_specific; struct device *dev = ccp->dev; - struct pci_dev *pdev = container_of(dev, struct pci_dev, dev); + struct pci_dev *pdev = to_pci_dev(dev); struct msix_entry msix_entry[MSIX_VECTORS]; unsigned int name_len = sizeof(ccp_pci->msix[0].name) - 1; int v, ret; @@ -86,7 +86,7 @@ e_irq: static int ccp_get_msi_irq(struct ccp_device *ccp) { struct device *dev = ccp->dev; - struct pci_dev *pdev = container_of(dev, struct pci_dev, dev); + struct pci_dev *pdev = to_pci_dev(dev); int ret; ret = pci_enable_msi(pdev); @@ -133,7 +133,7 @@ static void ccp_free_irqs(struct ccp_device *ccp) { struct ccp_pci *ccp_pci = ccp->dev_specific; struct device *dev = ccp->dev; - struct pci_dev *pdev = container_of(dev, struct pci_dev, dev); + struct pci_dev *pdev = to_pci_dev(dev); if (ccp_pci->msix_count) { while (ccp_pci->msix_count--) @@ -149,7 +149,7 @@ static void ccp_free_irqs(struct ccp_device *ccp) static int ccp_find_mmio_area(struct ccp_device *ccp) { struct device *dev = ccp->dev; - struct pci_dev *pdev = container_of(dev, struct pci_dev, dev); + struct pci_dev *pdev = to_pci_dev(dev); resource_size_t io_len; unsigned long io_flags; diff --git a/drivers/crypto/ccp/ccp-platform.c b/drivers/crypto/ccp/ccp-platform.c index 01b50cb4c982..66dd7c9d08c3 100644 --- a/drivers/crypto/ccp/ccp-platform.c +++ b/drivers/crypto/ccp/ccp-platform.c @@ -35,8 +35,7 @@ struct ccp_platform { static int ccp_get_irq(struct ccp_device *ccp) { struct device *dev = ccp->dev; - struct platform_device *pdev = container_of(dev, - struct platform_device, dev); + struct platform_device *pdev = to_platform_device(dev); int ret; ret = platform_get_irq(pdev, 0); @@ -78,8 +77,7 @@ static void ccp_free_irqs(struct ccp_device *ccp) static struct resource *ccp_find_mmio_area(struct ccp_device *ccp) { struct device *dev = ccp->dev; - struct platform_device *pdev = container_of(dev, - struct platform_device, dev); + struct platform_device *pdev = to_platform_device(dev); struct resource *ior; ior = platform_get_resource(pdev, IORESOURCE_MEM, 0); diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c index ca5c71ab4b4d..eee2c7e6c299 100644 --- a/drivers/crypto/hifn_795x.c +++ b/drivers/crypto/hifn_795x.c @@ -11,10 +11,6 @@ * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ #include <linux/kernel.h> @@ -36,14 +32,6 @@ #include <crypto/algapi.h> #include <crypto/des.h> -//#define HIFN_DEBUG - -#ifdef HIFN_DEBUG -#define dprintk(f, a...) printk(f, ##a) -#else -#define dprintk(f, a...) do {} while (0) -#endif - static char hifn_pll_ref[sizeof("extNNN")] = "ext"; module_param_string(hifn_pll_ref, hifn_pll_ref, sizeof(hifn_pll_ref), 0444); MODULE_PARM_DESC(hifn_pll_ref, @@ -79,12 +67,12 @@ static atomic_t hifn_dev_number; /* DMA registres */ -#define HIFN_DMA_CRA 0x0C /* DMA Command Ring Address */ -#define HIFN_DMA_SDRA 0x1C /* DMA Source Data Ring Address */ +#define HIFN_DMA_CRA 0x0C /* DMA Command Ring Address */ +#define HIFN_DMA_SDRA 0x1C /* DMA Source Data Ring Address */ #define HIFN_DMA_RRA 0x2C /* DMA Result Ring Address */ #define HIFN_DMA_DDRA 0x3C /* DMA Destination Data Ring Address */ #define HIFN_DMA_STCTL 0x40 /* DMA Status and Control */ -#define HIFN_DMA_INTREN 0x44 /* DMA Interrupt Enable */ +#define HIFN_DMA_INTREN 0x44 /* DMA Interrupt Enable */ #define HIFN_DMA_CFG1 0x48 /* DMA Configuration #1 */ #define HIFN_DMA_CFG2 0x6C /* DMA Configuration #2 */ #define HIFN_CHIP_ID 0x98 /* Chip ID */ @@ -358,10 +346,10 @@ static atomic_t hifn_dev_number; #define HIFN_NAMESIZE 32 #define HIFN_MAX_RESULT_ORDER 5 -#define HIFN_D_CMD_RSIZE 24*1 -#define HIFN_D_SRC_RSIZE 80*1 -#define HIFN_D_DST_RSIZE 80*1 -#define HIFN_D_RES_RSIZE 24*1 +#define HIFN_D_CMD_RSIZE (24 * 1) +#define HIFN_D_SRC_RSIZE (80 * 1) +#define HIFN_D_DST_RSIZE (80 * 1) +#define HIFN_D_RES_RSIZE (24 * 1) #define HIFN_D_DST_DALIGN 4 @@ -386,17 +374,16 @@ static atomic_t hifn_dev_number; #define HIFN_MAX_RESULT (8 + 4 + 4 + 20 + 4) #define HIFN_USED_RESULT 12 -struct hifn_desc -{ +struct hifn_desc { volatile __le32 l; volatile __le32 p; }; struct hifn_dma { - struct hifn_desc cmdr[HIFN_D_CMD_RSIZE+1]; - struct hifn_desc srcr[HIFN_D_SRC_RSIZE+1]; - struct hifn_desc dstr[HIFN_D_DST_RSIZE+1]; - struct hifn_desc resr[HIFN_D_RES_RSIZE+1]; + struct hifn_desc cmdr[HIFN_D_CMD_RSIZE + 1]; + struct hifn_desc srcr[HIFN_D_SRC_RSIZE + 1]; + struct hifn_desc dstr[HIFN_D_DST_RSIZE + 1]; + struct hifn_desc resr[HIFN_D_RES_RSIZE + 1]; u8 command_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_COMMAND]; u8 result_bufs[HIFN_D_CMD_RSIZE][HIFN_MAX_RESULT]; @@ -410,16 +397,15 @@ struct hifn_dma { int cmdk, srck, dstk, resk; }; -#define HIFN_FLAG_CMD_BUSY (1<<0) -#define HIFN_FLAG_SRC_BUSY (1<<1) -#define HIFN_FLAG_DST_BUSY (1<<2) -#define HIFN_FLAG_RES_BUSY (1<<3) -#define HIFN_FLAG_OLD_KEY (1<<4) +#define HIFN_FLAG_CMD_BUSY (1 << 0) +#define HIFN_FLAG_SRC_BUSY (1 << 1) +#define HIFN_FLAG_DST_BUSY (1 << 2) +#define HIFN_FLAG_RES_BUSY (1 << 3) +#define HIFN_FLAG_OLD_KEY (1 << 4) #define HIFN_DEFAULT_ACTIVE_NUM 5 -struct hifn_device -{ +struct hifn_device { char name[HIFN_NAMESIZE]; int irq; @@ -432,7 +418,7 @@ struct hifn_device u32 dmareg; - void *sa[HIFN_D_RES_RSIZE]; + void *sa[HIFN_D_RES_RSIZE]; spinlock_t lock; @@ -447,7 +433,7 @@ struct hifn_device struct tasklet_struct tasklet; - struct crypto_queue queue; + struct crypto_queue queue; struct list_head alg_list; unsigned int pk_clk_freq; @@ -468,8 +454,7 @@ struct hifn_device #define HIFN_D_JUMP 0x40000000 #define HIFN_D_VALID 0x80000000 -struct hifn_base_command -{ +struct hifn_base_command { volatile __le16 masks; volatile __le16 session_num; volatile __le16 total_source_count; @@ -491,12 +476,11 @@ struct hifn_base_command /* * Structure to help build up the command data structure. */ -struct hifn_crypt_command -{ - volatile __le16 masks; - volatile __le16 header_skip; - volatile __le16 source_count; - volatile __le16 reserved; +struct hifn_crypt_command { + volatile __le16 masks; + volatile __le16 header_skip; + volatile __le16 source_count; + volatile __le16 reserved; }; #define HIFN_CRYPT_CMD_ALG_MASK 0x0003 /* algorithm: */ @@ -522,12 +506,11 @@ struct hifn_crypt_command /* * Structure to help build up the command data structure. */ -struct hifn_mac_command -{ - volatile __le16 masks; - volatile __le16 header_skip; - volatile __le16 source_count; - volatile __le16 reserved; +struct hifn_mac_command { + volatile __le16 masks; + volatile __le16 header_skip; + volatile __le16 source_count; + volatile __le16 reserved; }; #define HIFN_MAC_CMD_ALG_MASK 0x0001 @@ -551,12 +534,11 @@ struct hifn_mac_command #define HIFN_MAC_CMD_POS_IPSEC 0x0200 #define HIFN_MAC_CMD_NEW_KEY 0x0800 -struct hifn_comp_command -{ - volatile __le16 masks; - volatile __le16 header_skip; - volatile __le16 source_count; - volatile __le16 reserved; +struct hifn_comp_command { + volatile __le16 masks; + volatile __le16 header_skip; + volatile __le16 source_count; + volatile __le16 reserved; }; #define HIFN_COMP_CMD_SRCLEN_M 0xc000 @@ -570,12 +552,11 @@ struct hifn_comp_command #define HIFN_COMP_CMD_ALG_MPPC 0x0001 /* MPPC */ #define HIFN_COMP_CMD_ALG_LZS 0x0000 /* LZS */ -struct hifn_base_result -{ - volatile __le16 flags; - volatile __le16 session; - volatile __le16 src_cnt; /* 15:0 of source count */ - volatile __le16 dst_cnt; /* 15:0 of dest count */ +struct hifn_base_result { + volatile __le16 flags; + volatile __le16 session; + volatile __le16 src_cnt; /* 15:0 of source count */ + volatile __le16 dst_cnt; /* 15:0 of dest count */ }; #define HIFN_BASE_RES_DSTOVERRUN 0x0200 /* destination overrun */ @@ -584,8 +565,7 @@ struct hifn_base_result #define HIFN_BASE_RES_DSTLEN_M 0x3000 /* 17:16 of dest count */ #define HIFN_BASE_RES_DSTLEN_S 12 -struct hifn_comp_result -{ +struct hifn_comp_result { volatile __le16 flags; volatile __le16 crc; }; @@ -596,18 +576,16 @@ struct hifn_comp_result #define HIFN_COMP_RES_ENDMARKER 0x0002 /* LZS: end marker seen */ #define HIFN_COMP_RES_SRC_NOTZERO 0x0001 /* source expired */ -struct hifn_mac_result -{ - volatile __le16 flags; - volatile __le16 reserved; +struct hifn_mac_result { + volatile __le16 flags; + volatile __le16 reserved; /* followed by 0, 6, 8, or 10 u16's of the MAC, then crypt */ }; #define HIFN_MAC_RES_MISCOMPARE 0x0002 /* compare failed */ #define HIFN_MAC_RES_SRC_NOTZERO 0x0001 /* source expired */ -struct hifn_crypt_result -{ +struct hifn_crypt_result { volatile __le16 flags; volatile __le16 reserved; }; @@ -622,11 +600,10 @@ struct hifn_crypt_result #define HIFN_POLL_SCALAR 0x0 #endif -#define HIFN_MAX_SEGLEN 0xffff /* maximum dma segment len */ +#define HIFN_MAX_SEGLEN 0xffff /* maximum dma segment len */ #define HIFN_MAX_DMALEN 0x3ffff /* maximum dma length */ -struct hifn_crypto_alg -{ +struct hifn_crypto_alg { struct list_head entry; struct crypto_alg alg; struct hifn_device *dev; @@ -634,24 +611,21 @@ struct hifn_crypto_alg #define ASYNC_SCATTERLIST_CACHE 16 -#define ASYNC_FLAGS_MISALIGNED (1<<0) +#define ASYNC_FLAGS_MISALIGNED (1 << 0) -struct hifn_cipher_walk -{ +struct hifn_cipher_walk { struct scatterlist cache[ASYNC_SCATTERLIST_CACHE]; u32 flags; int num; }; -struct hifn_context -{ +struct hifn_context { u8 key[HIFN_MAX_CRYPT_KEY_LENGTH]; struct hifn_device *dev; unsigned int keysize; }; -struct hifn_request_context -{ +struct hifn_request_context { u8 *iv; unsigned int ivsize; u8 op, type, mode, unused; @@ -693,7 +667,7 @@ static void hifn_wait_puc(struct hifn_device *dev) int i; u32 ret; - for (i=10000; i > 0; --i) { + for (i = 10000; i > 0; --i) { ret = hifn_read_0(dev, HIFN_0_PUCTRL); if (!(ret & HIFN_PUCTRL_RESET)) break; @@ -702,7 +676,7 @@ static void hifn_wait_puc(struct hifn_device *dev) } if (!i) - dprintk("%s: Failed to reset PUC unit.\n", dev->name); + dev_err(&dev->pdev->dev, "Failed to reset PUC unit.\n"); } static void hifn_reset_puc(struct hifn_device *dev) @@ -749,13 +723,12 @@ static void hifn_reset_dma(struct hifn_device *dev, int full) hifn_reset_puc(dev); } -static u32 hifn_next_signature(u_int32_t a, u_int cnt) +static u32 hifn_next_signature(u32 a, u_int cnt) { int i; u32 v; for (i = 0; i < cnt; i++) { - /* get the parity */ v = a & 0x80080125; v ^= v >> 16; @@ -846,33 +819,28 @@ static int hifn_init_pubrng(struct hifn_device *dev) hifn_write_1(dev, HIFN_1_PUB_RESET, hifn_read_1(dev, HIFN_1_PUB_RESET) | HIFN_PUBRST_RESET); - for (i=100; i > 0; --i) { + for (i = 100; i > 0; --i) { mdelay(1); if ((hifn_read_1(dev, HIFN_1_PUB_RESET) & HIFN_PUBRST_RESET) == 0) break; } - if (!i) - dprintk("Chip %s: Failed to initialise public key engine.\n", - dev->name); - else { + if (!i) { + dev_err(&dev->pdev->dev, "Failed to initialise public key engine.\n"); + } else { hifn_write_1(dev, HIFN_1_PUB_IEN, HIFN_PUBIEN_DONE); dev->dmareg |= HIFN_DMAIER_PUBDONE; hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); - dprintk("Chip %s: Public key engine has been successfully " - "initialised.\n", dev->name); + dev_dbg(&dev->pdev->dev, "Public key engine has been successfully initialised.\n"); } - /* - * Enable RNG engine. - */ + /* Enable RNG engine. */ hifn_write_1(dev, HIFN_1_RNG_CONFIG, hifn_read_1(dev, HIFN_1_RNG_CONFIG) | HIFN_RNGCFG_ENA); - dprintk("Chip %s: RNG engine has been successfully initialised.\n", - dev->name); + dev_dbg(&dev->pdev->dev, "RNG engine has been successfully initialised.\n"); #ifdef CONFIG_CRYPTO_DEV_HIFN_795X_RNG /* First value must be discarded */ @@ -896,8 +864,8 @@ static int hifn_enable_crypto(struct hifn_device *dev) } } - if (offtbl == NULL) { - dprintk("Chip %s: Unknown card!\n", dev->name); + if (!offtbl) { + dev_err(&dev->pdev->dev, "Unknown card!\n"); return -ENODEV; } @@ -912,7 +880,7 @@ static int hifn_enable_crypto(struct hifn_device *dev) hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, 0); mdelay(1); - for (i=0; i<12; ++i) { + for (i = 0; i < 12; ++i) { addr = hifn_next_signature(addr, offtbl[i] + 0x101); hifn_write_1(dev, HIFN_1_UNLOCK_SECRET2, addr); @@ -920,7 +888,7 @@ static int hifn_enable_crypto(struct hifn_device *dev) } hifn_write_1(dev, HIFN_1_DMA_CNFG, dmacfg); - dprintk("Chip %s: %s.\n", dev->name, pci_name(dev->pdev)); + dev_dbg(&dev->pdev->dev, "%s %s.\n", dev->name, pci_name(dev->pdev)); return 0; } @@ -931,16 +899,14 @@ static void hifn_init_dma(struct hifn_device *dev) u32 dptr = dev->desc_dma; int i; - for (i=0; i<HIFN_D_CMD_RSIZE; ++i) + for (i = 0; i < HIFN_D_CMD_RSIZE; ++i) dma->cmdr[i].p = __cpu_to_le32(dptr + offsetof(struct hifn_dma, command_bufs[i][0])); - for (i=0; i<HIFN_D_RES_RSIZE; ++i) + for (i = 0; i < HIFN_D_RES_RSIZE; ++i) dma->resr[i].p = __cpu_to_le32(dptr + offsetof(struct hifn_dma, result_bufs[i][0])); - /* - * Setup LAST descriptors. - */ + /* Setup LAST descriptors. */ dma->cmdr[HIFN_D_CMD_RSIZE].p = __cpu_to_le32(dptr + offsetof(struct hifn_dma, cmdr[0])); dma->srcr[HIFN_D_SRC_RSIZE].p = __cpu_to_le32(dptr + @@ -960,7 +926,7 @@ static void hifn_init_dma(struct hifn_device *dev) * to calculate the optimal multiplier. For PCI we assume 66MHz, since that * allows us to operate without the risk of overclocking the chip. If it * actually uses 33MHz, the chip will operate at half the speed, this can be - * overriden by specifying the frequency as module parameter (pci33). + * overridden by specifying the frequency as module parameter (pci33). * * Unfortunately the PCI clock is not very suitable since the HIFN needs a * stable clock and the PCI clock frequency may vary, so the default is the @@ -984,9 +950,8 @@ static void hifn_init_pll(struct hifn_device *dev) freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10); else { freq = 66; - printk(KERN_INFO "hifn795x: assuming %uMHz clock speed, " - "override with hifn_pll_ref=%.3s<frequency>\n", - freq, hifn_pll_ref); + dev_info(&dev->pdev->dev, "assuming %uMHz clock speed, override with hifn_pll_ref=%.3s<frequency>\n", + freq, hifn_pll_ref); } m = HIFN_PLL_FCK_MAX / freq; @@ -1174,17 +1139,17 @@ static int hifn_setup_cmd_desc(struct hifn_device *dev, mask = 0; switch (rctx->op) { - case ACRYPTO_OP_DECRYPT: - mask = HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE; - break; - case ACRYPTO_OP_ENCRYPT: - mask = HIFN_BASE_CMD_CRYPT; - break; - case ACRYPTO_OP_HMAC: - mask = HIFN_BASE_CMD_MAC; - break; - default: - goto err_out; + case ACRYPTO_OP_DECRYPT: + mask = HIFN_BASE_CMD_CRYPT | HIFN_BASE_CMD_DECODE; + break; + case ACRYPTO_OP_ENCRYPT: + mask = HIFN_BASE_CMD_CRYPT; + break; + case ACRYPTO_OP_HMAC: + mask = HIFN_BASE_CMD_MAC; + break; + default: + goto err_out; } buf_pos += hifn_setup_base_command(dev, buf_pos, nbytes, @@ -1199,53 +1164,53 @@ static int hifn_setup_cmd_desc(struct hifn_device *dev, md |= HIFN_CRYPT_CMD_NEW_IV; switch (rctx->mode) { - case ACRYPTO_MODE_ECB: - md |= HIFN_CRYPT_CMD_MODE_ECB; - break; - case ACRYPTO_MODE_CBC: - md |= HIFN_CRYPT_CMD_MODE_CBC; - break; - case ACRYPTO_MODE_CFB: - md |= HIFN_CRYPT_CMD_MODE_CFB; - break; - case ACRYPTO_MODE_OFB: - md |= HIFN_CRYPT_CMD_MODE_OFB; - break; - default: - goto err_out; + case ACRYPTO_MODE_ECB: + md |= HIFN_CRYPT_CMD_MODE_ECB; + break; + case ACRYPTO_MODE_CBC: + md |= HIFN_CRYPT_CMD_MODE_CBC; + break; + case ACRYPTO_MODE_CFB: + md |= HIFN_CRYPT_CMD_MODE_CFB; + break; + case ACRYPTO_MODE_OFB: + md |= HIFN_CRYPT_CMD_MODE_OFB; + break; + default: + goto err_out; } switch (rctx->type) { - case ACRYPTO_TYPE_AES_128: - if (ctx->keysize != 16) - goto err_out; - md |= HIFN_CRYPT_CMD_KSZ_128 | - HIFN_CRYPT_CMD_ALG_AES; - break; - case ACRYPTO_TYPE_AES_192: - if (ctx->keysize != 24) - goto err_out; - md |= HIFN_CRYPT_CMD_KSZ_192 | - HIFN_CRYPT_CMD_ALG_AES; - break; - case ACRYPTO_TYPE_AES_256: - if (ctx->keysize != 32) - goto err_out; - md |= HIFN_CRYPT_CMD_KSZ_256 | - HIFN_CRYPT_CMD_ALG_AES; - break; - case ACRYPTO_TYPE_3DES: - if (ctx->keysize != 24) - goto err_out; - md |= HIFN_CRYPT_CMD_ALG_3DES; - break; - case ACRYPTO_TYPE_DES: - if (ctx->keysize != 8) - goto err_out; - md |= HIFN_CRYPT_CMD_ALG_DES; - break; - default: + case ACRYPTO_TYPE_AES_128: + if (ctx->keysize != 16) + goto err_out; + md |= HIFN_CRYPT_CMD_KSZ_128 | + HIFN_CRYPT_CMD_ALG_AES; + break; + case ACRYPTO_TYPE_AES_192: + if (ctx->keysize != 24) goto err_out; + md |= HIFN_CRYPT_CMD_KSZ_192 | + HIFN_CRYPT_CMD_ALG_AES; + break; + case ACRYPTO_TYPE_AES_256: + if (ctx->keysize != 32) + goto err_out; + md |= HIFN_CRYPT_CMD_KSZ_256 | + HIFN_CRYPT_CMD_ALG_AES; + break; + case ACRYPTO_TYPE_3DES: + if (ctx->keysize != 24) + goto err_out; + md |= HIFN_CRYPT_CMD_ALG_3DES; + break; + case ACRYPTO_TYPE_DES: + if (ctx->keysize != 8) + goto err_out; + md |= HIFN_CRYPT_CMD_ALG_DES; + break; + default: + goto err_out; } buf_pos += hifn_setup_crypto_command(dev, buf_pos, @@ -1265,8 +1230,9 @@ static int hifn_setup_cmd_desc(struct hifn_device *dev, HIFN_D_VALID | HIFN_D_LAST | HIFN_D_MASKDONEIRQ | HIFN_D_JUMP); dma->cmdi = 0; - } else - dma->cmdr[dma->cmdi-1].l |= __cpu_to_le32(HIFN_D_VALID); + } else { + dma->cmdr[dma->cmdi - 1].l |= __cpu_to_le32(HIFN_D_VALID); + } if (!(dev->flags & HIFN_FLAG_CMD_BUSY)) { hifn_write_1(dev, HIFN_1_DMA_CSR, HIFN_DMACSR_C_CTRL_ENA); @@ -1424,7 +1390,7 @@ static int hifn_cipher_walk_init(struct hifn_cipher_walk *w, sg_init_table(w->cache, num); w->num = 0; - for (i=0; i<num; ++i) { + for (i = 0; i < num; ++i) { struct page *page = alloc_page(gfp_flags); struct scatterlist *s; @@ -1444,7 +1410,7 @@ static void hifn_cipher_walk_exit(struct hifn_cipher_walk *w) { int i; - for (i=0; i<w->num; ++i) { + for (i = 0; i < w->num; ++i) { struct scatterlist *s = &w->cache[i]; __free_page(sg_page(s)); @@ -1471,8 +1437,8 @@ static int ablkcipher_add(unsigned int *drestp, struct scatterlist *dst, drest -= copy; nbytes -= copy; - dprintk("%s: copy: %u, size: %u, drest: %u, nbytes: %u.\n", - __func__, copy, size, drest, nbytes); + pr_debug("%s: copy: %u, size: %u, drest: %u, nbytes: %u.\n", + __func__, copy, size, drest, nbytes); dst++; idx++; @@ -1499,8 +1465,8 @@ static int hifn_cipher_walk(struct ablkcipher_request *req, dst = &req->dst[idx]; - dprintk("\n%s: dlen: %u, doff: %u, offset: %u, nbytes: %u.\n", - __func__, dst->length, dst->offset, offset, nbytes); + pr_debug("\n%s: dlen: %u, doff: %u, offset: %u, nbytes: %u.\n", + __func__, dst->length, dst->offset, offset, nbytes); if (!IS_ALIGNED(dst->offset, HIFN_D_DST_DALIGN) || !IS_ALIGNED(dst->length, HIFN_D_DST_DALIGN) || @@ -1525,10 +1491,10 @@ static int hifn_cipher_walk(struct ablkcipher_request *req, * to put there additional blocksized chunk, * so we mark that page as containing only * blocksize aligned chunks: - * t->length = (slen & ~(HIFN_D_DST_DALIGN - 1)); + * t->length = (slen & ~(HIFN_D_DST_DALIGN - 1)); * and increase number of bytes to be processed * in next chunk: - * nbytes += diff; + * nbytes += diff; */ nbytes += diff; @@ -1536,14 +1502,13 @@ static int hifn_cipher_walk(struct ablkcipher_request *req, * Temporary of course... * Kick author if you will catch this one. */ - printk(KERN_ERR "%s: dlen: %u, nbytes: %u," - "slen: %u, offset: %u.\n", - __func__, dlen, nbytes, slen, offset); - printk(KERN_ERR "%s: please contact author to fix this " - "issue, generally you should not catch " - "this path under any condition but who " - "knows how did you use crypto code.\n" - "Thank you.\n", __func__); + pr_err("%s: dlen: %u, nbytes: %u, slen: %u, offset: %u.\n", + __func__, dlen, nbytes, slen, offset); + pr_err("%s: please contact author to fix this " + "issue, generally you should not catch " + "this path under any condition but who " + "knows how did you use crypto code.\n" + "Thank you.\n", __func__); BUG(); } else { copy += diff + nbytes; @@ -1630,70 +1595,16 @@ err_out: spin_unlock_irqrestore(&dev->lock, flags); err_out_exit: if (err) { - printk("%s: iv: %p [%d], key: %p [%d], mode: %u, op: %u, " - "type: %u, err: %d.\n", - dev->name, rctx->iv, rctx->ivsize, - ctx->key, ctx->keysize, - rctx->mode, rctx->op, rctx->type, err); + dev_info(&dev->pdev->dev, "iv: %p [%d], key: %p [%d], mode: %u, op: %u, " + "type: %u, err: %d.\n", + rctx->iv, rctx->ivsize, + ctx->key, ctx->keysize, + rctx->mode, rctx->op, rctx->type, err); } return err; } -static int hifn_test(struct hifn_device *dev, int encdec, u8 snum) -{ - int n, err; - u8 src[16]; - struct hifn_context ctx; - struct hifn_request_context rctx; - u8 fips_aes_ecb_from_zero[16] = { - 0x66, 0xE9, 0x4B, 0xD4, - 0xEF, 0x8A, 0x2C, 0x3B, - 0x88, 0x4C, 0xFA, 0x59, - 0xCA, 0x34, 0x2B, 0x2E}; - struct scatterlist sg; - - memset(src, 0, sizeof(src)); - memset(ctx.key, 0, sizeof(ctx.key)); - - ctx.dev = dev; - ctx.keysize = 16; - rctx.ivsize = 0; - rctx.iv = NULL; - rctx.op = (encdec)?ACRYPTO_OP_ENCRYPT:ACRYPTO_OP_DECRYPT; - rctx.mode = ACRYPTO_MODE_ECB; - rctx.type = ACRYPTO_TYPE_AES_128; - rctx.walk.cache[0].length = 0; - - sg_init_one(&sg, &src, sizeof(src)); - - err = hifn_setup_dma(dev, &ctx, &rctx, &sg, &sg, sizeof(src), NULL); - if (err) - goto err_out; - - dev->started = 0; - msleep(200); - - dprintk("%s: decoded: ", dev->name); - for (n=0; n<sizeof(src); ++n) - dprintk("%02x ", src[n]); - dprintk("\n"); - dprintk("%s: FIPS : ", dev->name); - for (n=0; n<sizeof(fips_aes_ecb_from_zero); ++n) - dprintk("%02x ", fips_aes_ecb_from_zero[n]); - dprintk("\n"); - - if (!memcmp(src, fips_aes_ecb_from_zero, sizeof(fips_aes_ecb_from_zero))) { - printk(KERN_INFO "%s: AES 128 ECB test has been successfully " - "passed.\n", dev->name); - return 0; - } - -err_out: - printk(KERN_INFO "%s: AES 128 ECB test has been failed.\n", dev->name); - return -1; -} - static int hifn_start_device(struct hifn_device *dev) { int err; @@ -1739,8 +1650,8 @@ static int ablkcipher_get(void *saddr, unsigned int *srestp, unsigned int offset saddr += copy; offset = 0; - dprintk("%s: copy: %u, size: %u, srest: %u, nbytes: %u.\n", - __func__, copy, size, srest, nbytes); + pr_debug("%s: copy: %u, size: %u, srest: %u, nbytes: %u.\n", + __func__, copy, size, srest, nbytes); dst++; idx++; @@ -1760,7 +1671,8 @@ static inline void hifn_complete_sa(struct hifn_device *dev, int i) dev->sa[i] = NULL; dev->started--; if (dev->started < 0) - printk("%s: started: %d.\n", __func__, dev->started); + dev_info(&dev->pdev->dev, "%s: started: %d.\n", __func__, + dev->started); spin_unlock_irqrestore(&dev->lock, flags); BUG_ON(dev->started < 0); } @@ -1779,7 +1691,7 @@ static void hifn_process_ready(struct ablkcipher_request *req, int error) t = &rctx->walk.cache[idx]; dst = &req->dst[idx]; - dprintk("\n%s: sg_page(t): %p, t->length: %u, " + pr_debug("\n%s: sg_page(t): %p, t->length: %u, " "sg_page(dst): %p, dst->length: %u, " "nbytes: %u.\n", __func__, sg_page(t), t->length, @@ -1815,9 +1727,8 @@ static void hifn_clear_rings(struct hifn_device *dev, int error) struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; int i, u; - dprintk("%s: ring cleanup 1: i: %d.%d.%d.%d, u: %d.%d.%d.%d, " + dev_dbg(&dev->pdev->dev, "ring cleanup 1: i: %d.%d.%d.%d, u: %d.%d.%d.%d, " "k: %d.%d.%d.%d.\n", - dev->name, dma->cmdi, dma->srci, dma->dsti, dma->resi, dma->cmdu, dma->srcu, dma->dstu, dma->resu, dma->cmdk, dma->srck, dma->dstk, dma->resk); @@ -1870,9 +1781,8 @@ static void hifn_clear_rings(struct hifn_device *dev, int error) } dma->dstk = i; dma->dstu = u; - dprintk("%s: ring cleanup 2: i: %d.%d.%d.%d, u: %d.%d.%d.%d, " + dev_dbg(&dev->pdev->dev, "ring cleanup 2: i: %d.%d.%d.%d, u: %d.%d.%d.%d, " "k: %d.%d.%d.%d.\n", - dev->name, dma->cmdi, dma->srci, dma->dsti, dma->resi, dma->cmdu, dma->srcu, dma->dstu, dma->resu, dma->cmdk, dma->srck, dma->dstk, dma->resk); @@ -1921,21 +1831,22 @@ static void hifn_work(struct work_struct *work) int i; struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; - printk("%s: r: %08x, active: %d, started: %d, " - "success: %lu: qlen: %u/%u, reset: %d.\n", - dev->name, r, dev->active, dev->started, - dev->success, dev->queue.qlen, dev->queue.max_qlen, - reset); + dev_info(&dev->pdev->dev, + "r: %08x, active: %d, started: %d, " + "success: %lu: qlen: %u/%u, reset: %d.\n", + r, dev->active, dev->started, + dev->success, dev->queue.qlen, dev->queue.max_qlen, + reset); - printk("%s: res: ", __func__); - for (i=0; i<HIFN_D_RES_RSIZE; ++i) { - printk("%x.%p ", dma->resr[i].l, dev->sa[i]); + dev_info(&dev->pdev->dev, "%s: res: ", __func__); + for (i = 0; i < HIFN_D_RES_RSIZE; ++i) { + pr_info("%x.%p ", dma->resr[i].l, dev->sa[i]); if (dev->sa[i]) { hifn_process_ready(dev->sa[i], -ENODEV); hifn_complete_sa(dev, i); } } - printk("\n"); + pr_info("\n"); hifn_reset_dma(dev, 1); hifn_stop_device(dev); @@ -1957,9 +1868,9 @@ static irqreturn_t hifn_interrupt(int irq, void *data) dmacsr = hifn_read_1(dev, HIFN_1_DMA_CSR); - dprintk("%s: 1 dmacsr: %08x, dmareg: %08x, res: %08x [%d], " + dev_dbg(&dev->pdev->dev, "1 dmacsr: %08x, dmareg: %08x, res: %08x [%d], " "i: %d.%d.%d.%d, u: %d.%d.%d.%d.\n", - dev->name, dmacsr, dev->dmareg, dmacsr & dev->dmareg, dma->cmdi, + dmacsr, dev->dmareg, dmacsr & dev->dmareg, dma->cmdi, dma->cmdi, dma->srci, dma->dsti, dma->resi, dma->cmdu, dma->srcu, dma->dstu, dma->resu); @@ -1978,9 +1889,9 @@ static irqreturn_t hifn_interrupt(int irq, void *data) if (restart) { u32 puisr = hifn_read_0(dev, HIFN_0_PUISR); - printk(KERN_WARNING "%s: overflow: r: %d, d: %d, puisr: %08x, d: %u.\n", - dev->name, !!(dmacsr & HIFN_DMACSR_R_OVER), - !!(dmacsr & HIFN_DMACSR_D_OVER), + dev_warn(&dev->pdev->dev, "overflow: r: %d, d: %d, puisr: %08x, d: %u.\n", + !!(dmacsr & HIFN_DMACSR_R_OVER), + !!(dmacsr & HIFN_DMACSR_D_OVER), puisr, !!(puisr & HIFN_PUISR_DSTOVER)); if (!!(puisr & HIFN_PUISR_DSTOVER)) hifn_write_0(dev, HIFN_0_PUISR, HIFN_PUISR_DSTOVER); @@ -1991,18 +1902,18 @@ static irqreturn_t hifn_interrupt(int irq, void *data) restart = dmacsr & (HIFN_DMACSR_C_ABORT | HIFN_DMACSR_S_ABORT | HIFN_DMACSR_D_ABORT | HIFN_DMACSR_R_ABORT); if (restart) { - printk(KERN_WARNING "%s: abort: c: %d, s: %d, d: %d, r: %d.\n", - dev->name, !!(dmacsr & HIFN_DMACSR_C_ABORT), - !!(dmacsr & HIFN_DMACSR_S_ABORT), - !!(dmacsr & HIFN_DMACSR_D_ABORT), - !!(dmacsr & HIFN_DMACSR_R_ABORT)); + dev_warn(&dev->pdev->dev, "abort: c: %d, s: %d, d: %d, r: %d.\n", + !!(dmacsr & HIFN_DMACSR_C_ABORT), + !!(dmacsr & HIFN_DMACSR_S_ABORT), + !!(dmacsr & HIFN_DMACSR_D_ABORT), + !!(dmacsr & HIFN_DMACSR_R_ABORT)); hifn_reset_dma(dev, 1); hifn_init_dma(dev); hifn_init_registers(dev); } if ((dmacsr & HIFN_DMACSR_C_WAIT) && (dma->cmdu == 0)) { - dprintk("%s: wait on command.\n", dev->name); + dev_dbg(&dev->pdev->dev, "wait on command.\n"); dev->dmareg &= ~(HIFN_DMAIER_C_WAIT); hifn_write_1(dev, HIFN_1_DMA_IER, dev->dmareg); } @@ -2020,19 +1931,19 @@ static void hifn_flush(struct hifn_device *dev) struct hifn_dma *dma = (struct hifn_dma *)dev->desc_virt; int i; - for (i=0; i<HIFN_D_RES_RSIZE; ++i) { + for (i = 0; i < HIFN_D_RES_RSIZE; ++i) { struct hifn_desc *d = &dma->resr[i]; if (dev->sa[i]) { hifn_process_ready(dev->sa[i], - (d->l & __cpu_to_le32(HIFN_D_VALID))?-ENODEV:0); + (d->l & __cpu_to_le32(HIFN_D_VALID)) ? -ENODEV : 0); hifn_complete_sa(dev, i); } } spin_lock_irqsave(&dev->lock, flags); while ((async_req = crypto_dequeue_request(&dev->queue))) { - req = container_of(async_req, struct ablkcipher_request, base); + req = ablkcipher_request_cast(async_req); spin_unlock_irqrestore(&dev->lock, flags); hifn_process_ready(req, -ENODEV); @@ -2057,7 +1968,7 @@ static int hifn_setkey(struct crypto_ablkcipher *cipher, const u8 *key, if (len == HIFN_DES_KEY_LENGTH) { u32 tmp[DES_EXPKEY_WORDS]; int ret = des_ekey(tmp, key); - + if (unlikely(ret == 0) && (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) { tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; @@ -2151,7 +2062,7 @@ static int hifn_process_queue(struct hifn_device *dev) if (backlog) backlog->complete(backlog, -EINPROGRESS); - req = container_of(async_req, struct ablkcipher_request, base); + req = ablkcipher_request_cast(async_req); err = hifn_handle_req(req); if (err) @@ -2298,9 +2209,7 @@ static inline int hifn_encrypt_3des_ofb(struct ablkcipher_request *req) ACRYPTO_TYPE_3DES, ACRYPTO_MODE_OFB); } -/* - * 3DES decryption functions. - */ +/* 3DES decryption functions. */ static inline int hifn_decrypt_3des_ecb(struct ablkcipher_request *req) { return hifn_setup_crypto(req, ACRYPTO_OP_DECRYPT, @@ -2322,8 +2231,7 @@ static inline int hifn_decrypt_3des_ofb(struct ablkcipher_request *req) ACRYPTO_TYPE_3DES, ACRYPTO_MODE_OFB); } -struct hifn_alg_template -{ +struct hifn_alg_template { char name[CRYPTO_MAX_ALG_NAME]; char drv_name[CRYPTO_MAX_ALG_NAME]; unsigned int bsize; @@ -2483,7 +2391,7 @@ static int hifn_alg_alloc(struct hifn_device *dev, struct hifn_alg_template *t) struct hifn_crypto_alg *alg; int err; - alg = kzalloc(sizeof(struct hifn_crypto_alg), GFP_KERNEL); + alg = kzalloc(sizeof(*alg), GFP_KERNEL); if (!alg) return -ENOMEM; @@ -2530,7 +2438,7 @@ static int hifn_register_alg(struct hifn_device *dev) { int i, err; - for (i=0; i<ARRAY_SIZE(hifn_alg_templates); ++i) { + for (i = 0; i < ARRAY_SIZE(hifn_alg_templates); ++i) { err = hifn_alg_alloc(dev, &hifn_alg_templates[i]); if (err) goto err_out_exit; @@ -2575,7 +2483,7 @@ static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_out_disable_pci_device; snprintf(name, sizeof(name), "hifn%d", - atomic_inc_return(&hifn_dev_number)-1); + atomic_inc_return(&hifn_dev_number) - 1); err = pci_request_regions(pdev, name); if (err) @@ -2584,8 +2492,7 @@ static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (pci_resource_len(pdev, 0) < HIFN_BAR0_SIZE || pci_resource_len(pdev, 1) < HIFN_BAR1_SIZE || pci_resource_len(pdev, 2) < HIFN_BAR2_SIZE) { - dprintk("%s: Broken hardware - I/O regions are too small.\n", - pci_name(pdev)); + dev_err(&pdev->dev, "Broken hardware - I/O regions are too small.\n"); err = -ENODEV; goto err_out_free_regions; } @@ -2602,7 +2509,7 @@ static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) snprintf(dev->name, sizeof(dev->name), "%s", name); spin_lock_init(&dev->lock); - for (i=0; i<3; ++i) { + for (i = 0; i < 3; ++i) { unsigned long addr, size; addr = pci_resource_start(pdev, i); @@ -2618,7 +2525,7 @@ static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) dev->desc_virt = pci_zalloc_consistent(pdev, sizeof(struct hifn_dma), &dev->desc_dma); if (!dev->desc_virt) { - dprintk("Failed to allocate descriptor rings.\n"); + dev_err(&pdev->dev, "Failed to allocate descriptor rings.\n"); err = -ENOMEM; goto err_out_unmap_bars; } @@ -2626,7 +2533,7 @@ static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) dev->pdev = pdev; dev->irq = pdev->irq; - for (i=0; i<HIFN_D_RES_RSIZE; ++i) + for (i = 0; i < HIFN_D_RES_RSIZE; ++i) dev->sa[i] = NULL; pci_set_drvdata(pdev, dev); @@ -2637,7 +2544,8 @@ static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) err = request_irq(dev->irq, hifn_interrupt, IRQF_SHARED, dev->name, dev); if (err) { - dprintk("Failed to request IRQ%d: err: %d.\n", dev->irq, err); + dev_err(&pdev->dev, "Failed to request IRQ%d: err: %d.\n", + dev->irq, err); dev->irq = 0; goto err_out_free_desc; } @@ -2646,10 +2554,6 @@ static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (err) goto err_out_free_irq; - err = hifn_test(dev, 1, 0); - if (err) - goto err_out_stop_device; - err = hifn_register_rng(dev); if (err) goto err_out_stop_device; @@ -2661,9 +2565,9 @@ static int hifn_probe(struct pci_dev *pdev, const struct pci_device_id *id) INIT_DELAYED_WORK(&dev->work, hifn_work); schedule_delayed_work(&dev->work, HZ); - dprintk("HIFN crypto accelerator card at %s has been " - "successfully registered as %s.\n", - pci_name(pdev), dev->name); + dev_dbg(&pdev->dev, "HIFN crypto accelerator card at %s has been " + "successfully registered as %s.\n", + pci_name(pdev), dev->name); return 0; @@ -2680,7 +2584,7 @@ err_out_free_desc: dev->desc_virt, dev->desc_dma); err_out_unmap_bars: - for (i=0; i<3; ++i) + for (i = 0; i < 3; ++i) if (dev->bar[i]) iounmap(dev->bar[i]); @@ -2715,7 +2619,7 @@ static void hifn_remove(struct pci_dev *pdev) pci_free_consistent(pdev, sizeof(struct hifn_dma), dev->desc_virt, dev->desc_dma); - for (i=0; i<3; ++i) + for (i = 0; i < 3; ++i) if (dev->bar[i]) iounmap(dev->bar[i]); @@ -2750,8 +2654,7 @@ static int __init hifn_init(void) if (strncmp(hifn_pll_ref, "ext", 3) && strncmp(hifn_pll_ref, "pci", 3)) { - printk(KERN_ERR "hifn795x: invalid hifn_pll_ref clock, " - "must be pci or ext"); + pr_err("hifn795x: invalid hifn_pll_ref clock, must be pci or ext"); return -EINVAL; } @@ -2763,22 +2666,21 @@ static int __init hifn_init(void) if (hifn_pll_ref[3] != '\0') { freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10); if (freq < 20 || freq > 100) { - printk(KERN_ERR "hifn795x: invalid hifn_pll_ref " - "frequency, must be in the range " - "of 20-100"); + pr_err("hifn795x: invalid hifn_pll_ref frequency, must" + "be in the range of 20-100"); return -EINVAL; } } err = pci_register_driver(&hifn_pci_driver); if (err < 0) { - dprintk("Failed to register PCI driver for %s device.\n", - hifn_pci_driver.name); + pr_err("Failed to register PCI driver for %s device.\n", + hifn_pci_driver.name); return -ENODEV; } - printk(KERN_INFO "Driver for HIFN 795x crypto accelerator chip " - "has been successfully registered.\n"); + pr_info("Driver for HIFN 795x crypto accelerator chip " + "has been successfully registered.\n"); return 0; } @@ -2787,8 +2689,8 @@ static void __exit hifn_fini(void) { pci_unregister_driver(&hifn_pci_driver); - printk(KERN_INFO "Driver for HIFN 795x crypto accelerator chip " - "has been successfully unregistered.\n"); + pr_info("Driver for HIFN 795x crypto accelerator chip " + "has been successfully unregistered.\n"); } module_init(hifn_init); diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c index 8f2790353281..e52496a172d0 100644 --- a/drivers/crypto/ixp4xx_crypto.c +++ b/drivers/crypto/ixp4xx_crypto.c @@ -510,10 +510,8 @@ npe_error: printk(KERN_ERR "%s not responding\n", npe_name(npe_c)); ret = -EIO; err: - if (ctx_pool) - dma_pool_destroy(ctx_pool); - if (buffer_pool) - dma_pool_destroy(buffer_pool); + dma_pool_destroy(ctx_pool); + dma_pool_destroy(buffer_pool); npe_release(npe_c); return ret; } diff --git a/drivers/crypto/marvell/cipher.c b/drivers/crypto/marvell/cipher.c index 6edae64bb387..dcf1fceb9336 100644 --- a/drivers/crypto/marvell/cipher.c +++ b/drivers/crypto/marvell/cipher.c @@ -401,7 +401,15 @@ static int mv_cesa_ablkcipher_req_init(struct ablkcipher_request *req, return -EINVAL; creq->src_nents = sg_nents_for_len(req->src, req->nbytes); + if (creq->src_nents < 0) { + dev_err(cesa_dev->dev, "Invalid number of src SG"); + return creq->src_nents; + } creq->dst_nents = sg_nents_for_len(req->dst, req->nbytes); + if (creq->dst_nents < 0) { + dev_err(cesa_dev->dev, "Invalid number of dst SG"); + return creq->dst_nents; + } mv_cesa_update_op_cfg(tmpl, CESA_SA_DESC_CFG_OP_CRYPT_ONLY, CESA_SA_DESC_CFG_OP_MSK); diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index 6ec55b4a087b..683cca9ac3c4 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -712,6 +712,10 @@ static int mv_cesa_ahash_req_init(struct ahash_request *req, bool *cached) creq->req.base.type = CESA_STD_REQ; creq->src_nents = sg_nents_for_len(req->src, req->nbytes); + if (creq->src_nents < 0) { + dev_err(cesa_dev->dev, "Invalid number of src SG"); + return creq->src_nents; + } ret = mv_cesa_ahash_cache_req(req, cached); if (ret) diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c index 5450880abb7b..b85a7a7dbf63 100644 --- a/drivers/crypto/n2_core.c +++ b/drivers/crypto/n2_core.c @@ -241,7 +241,7 @@ static inline bool n2_should_run_async(struct spu_queue *qp, int this_len) struct n2_ahash_alg { struct list_head entry; - const char *hash_zero; + const u8 *hash_zero; const u32 *hash_init; u8 hw_op_hashsz; u8 digest_size; @@ -1267,7 +1267,7 @@ static LIST_HEAD(cipher_algs); struct n2_hash_tmpl { const char *name; - const char *hash_zero; + const u8 *hash_zero; const u32 *hash_init; u8 hw_op_hashsz; u8 digest_size; @@ -1276,40 +1276,19 @@ struct n2_hash_tmpl { u8 hmac_type; }; -static const char md5_zero[MD5_DIGEST_SIZE] = { - 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, - 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e, -}; static const u32 md5_init[MD5_HASH_WORDS] = { cpu_to_le32(MD5_H0), cpu_to_le32(MD5_H1), cpu_to_le32(MD5_H2), cpu_to_le32(MD5_H3), }; -static const char sha1_zero[SHA1_DIGEST_SIZE] = { - 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, 0x32, - 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, 0xaf, 0xd8, - 0x07, 0x09 -}; static const u32 sha1_init[SHA1_DIGEST_SIZE / 4] = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4, }; -static const char sha256_zero[SHA256_DIGEST_SIZE] = { - 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, 0x9a, - 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, 0x27, 0xae, - 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, 0xa4, 0x95, 0x99, - 0x1b, 0x78, 0x52, 0xb8, 0x55 -}; static const u32 sha256_init[SHA256_DIGEST_SIZE / 4] = { SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7, }; -static const char sha224_zero[SHA224_DIGEST_SIZE] = { - 0xd1, 0x4a, 0x02, 0x8c, 0x2a, 0x3a, 0x2b, 0xc9, 0x47, - 0x61, 0x02, 0xbb, 0x28, 0x82, 0x34, 0xc4, 0x15, 0xa2, - 0xb0, 0x1f, 0x82, 0x8e, 0xa6, 0x2a, 0xc5, 0xb3, 0xe4, - 0x2f -}; static const u32 sha224_init[SHA256_DIGEST_SIZE / 4] = { SHA224_H0, SHA224_H1, SHA224_H2, SHA224_H3, SHA224_H4, SHA224_H5, SHA224_H6, SHA224_H7, @@ -1317,7 +1296,7 @@ static const u32 sha224_init[SHA256_DIGEST_SIZE / 4] = { static const struct n2_hash_tmpl hash_tmpls[] = { { .name = "md5", - .hash_zero = md5_zero, + .hash_zero = md5_zero_message_hash, .hash_init = md5_init, .auth_type = AUTH_TYPE_MD5, .hmac_type = AUTH_TYPE_HMAC_MD5, @@ -1325,7 +1304,7 @@ static const struct n2_hash_tmpl hash_tmpls[] = { .digest_size = MD5_DIGEST_SIZE, .block_size = MD5_HMAC_BLOCK_SIZE }, { .name = "sha1", - .hash_zero = sha1_zero, + .hash_zero = sha1_zero_message_hash, .hash_init = sha1_init, .auth_type = AUTH_TYPE_SHA1, .hmac_type = AUTH_TYPE_HMAC_SHA1, @@ -1333,7 +1312,7 @@ static const struct n2_hash_tmpl hash_tmpls[] = { .digest_size = SHA1_DIGEST_SIZE, .block_size = SHA1_BLOCK_SIZE }, { .name = "sha256", - .hash_zero = sha256_zero, + .hash_zero = sha256_zero_message_hash, .hash_init = sha256_init, .auth_type = AUTH_TYPE_SHA256, .hmac_type = AUTH_TYPE_HMAC_SHA256, @@ -1341,7 +1320,7 @@ static const struct n2_hash_tmpl hash_tmpls[] = { .digest_size = SHA256_DIGEST_SIZE, .block_size = SHA256_BLOCK_SIZE }, { .name = "sha224", - .hash_zero = sha224_zero, + .hash_zero = sha224_zero_message_hash, .hash_init = sha224_init, .auth_type = AUTH_TYPE_SHA256, .hmac_type = AUTH_TYPE_RESERVED, @@ -2243,22 +2222,19 @@ static struct platform_driver n2_mau_driver = { .remove = n2_mau_remove, }; +static struct platform_driver * const drivers[] = { + &n2_crypto_driver, + &n2_mau_driver, +}; + static int __init n2_init(void) { - int err = platform_driver_register(&n2_crypto_driver); - - if (!err) { - err = platform_driver_register(&n2_mau_driver); - if (err) - platform_driver_unregister(&n2_crypto_driver); - } - return err; + return platform_register_drivers(drivers, ARRAY_SIZE(drivers)); } static void __exit n2_exit(void) { - platform_driver_unregister(&n2_mau_driver); - platform_driver_unregister(&n2_crypto_driver); + platform_unregister_drivers(drivers, ARRAY_SIZE(drivers)); } module_init(n2_init); diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c index 9ef51fafdbff..1710f80a09ec 100644 --- a/drivers/crypto/nx/nx-842-powernv.c +++ b/drivers/crypto/nx/nx-842-powernv.c @@ -442,6 +442,14 @@ static int nx842_powernv_function(const unsigned char *in, unsigned int inlen, (unsigned int)ccw, (unsigned int)be32_to_cpu(crb->ccw)); + /* + * NX842 coprocessor sets 3rd bit in CR register with XER[S0]. + * XER[S0] is the integer summary overflow bit which is nothing + * to do NX. Since this bit can be set with other return values, + * mask this bit. + */ + ret &= ~ICSWX_XERS0; + switch (ret) { case ICSWX_INITIATED: ret = wait_for_csb(wmem, csb); @@ -454,10 +462,6 @@ static int nx842_powernv_function(const unsigned char *in, unsigned int inlen, pr_err_ratelimited("ICSWX rejected\n"); ret = -EPROTO; break; - default: - pr_err_ratelimited("Invalid ICSWX return code %x\n", ret); - ret = -EPROTO; - break; } if (!ret) @@ -525,7 +529,6 @@ static int nx842_powernv_decompress(const unsigned char *in, unsigned int inlen, static int __init nx842_powernv_probe(struct device_node *dn) { struct nx842_coproc *coproc; - struct property *ct_prop, *ci_prop; unsigned int ct, ci; int chip_id; @@ -534,18 +537,16 @@ static int __init nx842_powernv_probe(struct device_node *dn) pr_err("ibm,chip-id missing\n"); return -EINVAL; } - ct_prop = of_find_property(dn, "ibm,842-coprocessor-type", NULL); - if (!ct_prop) { + + if (of_property_read_u32(dn, "ibm,842-coprocessor-type", &ct)) { pr_err("ibm,842-coprocessor-type missing\n"); return -EINVAL; } - ct = be32_to_cpu(*(unsigned int *)ct_prop->value); - ci_prop = of_find_property(dn, "ibm,842-coprocessor-instance", NULL); - if (!ci_prop) { + + if (of_property_read_u32(dn, "ibm,842-coprocessor-instance", &ci)) { pr_err("ibm,842-coprocessor-instance missing\n"); return -EINVAL; } - ci = be32_to_cpu(*(unsigned int *)ci_prop->value); coproc = kmalloc(sizeof(*coproc), GFP_KERNEL); if (!coproc) diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index eba23147c0ee..dd355bd19474 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -539,8 +539,6 @@ static void omap_aes_finish_req(struct omap_aes_dev *dd, int err) static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd) { - int err = 0; - pr_debug("total: %d\n", dd->total); omap_aes_dma_stop(dd); @@ -548,7 +546,7 @@ static int omap_aes_crypt_dma_stop(struct omap_aes_dev *dd) dmaengine_terminate_all(dd->dma_lch_in); dmaengine_terminate_all(dd->dma_lch_out); - return err; + return 0; } static int omap_aes_check_aligned(struct scatterlist *sg, int total) diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c index 0a70e46d5416..dd7b93f2f94c 100644 --- a/drivers/crypto/omap-des.c +++ b/drivers/crypto/omap-des.c @@ -527,8 +527,6 @@ static void omap_des_finish_req(struct omap_des_dev *dd, int err) static int omap_des_crypt_dma_stop(struct omap_des_dev *dd) { - int err = 0; - pr_debug("total: %d\n", dd->total); omap_des_dma_stop(dd); @@ -536,7 +534,7 @@ static int omap_des_crypt_dma_stop(struct omap_des_dev *dd) dmaengine_terminate_all(dd->dma_lch_in); dmaengine_terminate_all(dd->dma_lch_out); - return err; + return 0; } static int omap_des_copy_needed(struct scatterlist *sg) @@ -1086,6 +1084,7 @@ static int omap_des_probe(struct platform_device *pdev) dd->phys_base = res->start; pm_runtime_enable(dev); + pm_runtime_irq_safe(dev); err = pm_runtime_get_sync(dev); if (err < 0) { pm_runtime_put_noidle(dev); diff --git a/drivers/crypto/padlock-aes.c b/drivers/crypto/padlock-aes.c index 97a364694bfc..441e86b23571 100644 --- a/drivers/crypto/padlock-aes.c +++ b/drivers/crypto/padlock-aes.c @@ -238,7 +238,7 @@ static inline void ecb_crypt(const u8 *in, u8 *out, u32 *key, /* Padlock in ECB mode fetches at least ecb_fetch_bytes of data. * We could avoid some copying here but it's probably not worth it. */ - if (unlikely(((unsigned long)in & ~PAGE_MASK) + ecb_fetch_bytes > PAGE_SIZE)) { + if (unlikely(offset_in_page(in) + ecb_fetch_bytes > PAGE_SIZE)) { ecb_crypt_copy(in, out, key, cword, count); return; } @@ -250,7 +250,7 @@ static inline u8 *cbc_crypt(const u8 *in, u8 *out, u32 *key, u8 *iv, struct cword *cword, int count) { /* Padlock in CBC mode fetches at least cbc_fetch_bytes of data. */ - if (unlikely(((unsigned long)in & ~PAGE_MASK) + cbc_fetch_bytes > PAGE_SIZE)) + if (unlikely(offset_in_page(in) + cbc_fetch_bytes > PAGE_SIZE)) return cbc_crypt_copy(in, out, key, iv, cword, count); return rep_xcrypt_cbc(in, out, key, iv, cword, count); diff --git a/drivers/crypto/picoxcell_crypto.c b/drivers/crypto/picoxcell_crypto.c index 615da961c4d8..3b1c7ecf078f 100644 --- a/drivers/crypto/picoxcell_crypto.c +++ b/drivers/crypto/picoxcell_crypto.c @@ -272,12 +272,6 @@ static unsigned spacc_load_ctx(struct spacc_generic_ctx *ctx, return indx; } -/* Count the number of scatterlist entries in a scatterlist. */ -static inline int sg_count(struct scatterlist *sg_list, int nbytes) -{ - return sg_nents_for_len(sg_list, nbytes); -} - static inline void ddt_set(struct spacc_ddt *ddt, dma_addr_t phys, size_t len) { ddt->p = phys; @@ -295,12 +289,17 @@ static struct spacc_ddt *spacc_sg_to_ddt(struct spacc_engine *engine, enum dma_data_direction dir, dma_addr_t *ddt_phys) { - unsigned nents, mapped_ents; + unsigned mapped_ents; struct scatterlist *cur; struct spacc_ddt *ddt; int i; + int nents; - nents = sg_count(payload, nbytes); + nents = sg_nents_for_len(payload, nbytes); + if (nents < 0) { + dev_err(engine->dev, "Invalid numbers of SG.\n"); + return NULL; + } mapped_ents = dma_map_sg(engine->dev, payload, nents, dir); if (mapped_ents + 1 > MAX_DDT_LEN) @@ -328,7 +327,7 @@ static int spacc_aead_make_ddts(struct aead_request *areq) struct spacc_engine *engine = req->engine; struct spacc_ddt *src_ddt, *dst_ddt; unsigned total; - unsigned int src_nents, dst_nents; + int src_nents, dst_nents; struct scatterlist *cur; int i, dst_ents, src_ents; @@ -336,13 +335,21 @@ static int spacc_aead_make_ddts(struct aead_request *areq) if (req->is_encrypt) total += crypto_aead_authsize(aead); - src_nents = sg_count(areq->src, total); + src_nents = sg_nents_for_len(areq->src, total); + if (src_nents < 0) { + dev_err(engine->dev, "Invalid numbers of src SG.\n"); + return src_nents; + } if (src_nents + 1 > MAX_DDT_LEN) return -E2BIG; dst_nents = 0; if (areq->src != areq->dst) { - dst_nents = sg_count(areq->dst, total); + dst_nents = sg_nents_for_len(areq->dst, total); + if (dst_nents < 0) { + dev_err(engine->dev, "Invalid numbers of dst SG.\n"); + return dst_nents; + } if (src_nents + 1 > MAX_DDT_LEN) return -E2BIG; } @@ -422,13 +429,22 @@ static void spacc_aead_free_ddts(struct spacc_req *req) (req->is_encrypt ? crypto_aead_authsize(aead) : 0); struct spacc_aead_ctx *aead_ctx = crypto_aead_ctx(aead); struct spacc_engine *engine = aead_ctx->generic.engine; - unsigned nents = sg_count(areq->src, total); + int nents = sg_nents_for_len(areq->src, total); + + /* sg_nents_for_len should not fail since it works when mapping sg */ + if (unlikely(nents < 0)) { + dev_err(engine->dev, "Invalid numbers of src SG.\n"); + return; + } if (areq->src != areq->dst) { dma_unmap_sg(engine->dev, areq->src, nents, DMA_TO_DEVICE); - dma_unmap_sg(engine->dev, areq->dst, - sg_count(areq->dst, total), - DMA_FROM_DEVICE); + nents = sg_nents_for_len(areq->dst, total); + if (unlikely(nents < 0)) { + dev_err(engine->dev, "Invalid numbers of dst SG.\n"); + return; + } + dma_unmap_sg(engine->dev, areq->dst, nents, DMA_FROM_DEVICE); } else dma_unmap_sg(engine->dev, areq->src, nents, DMA_BIDIRECTIONAL); @@ -440,7 +456,12 @@ static void spacc_free_ddt(struct spacc_req *req, struct spacc_ddt *ddt, dma_addr_t ddt_addr, struct scatterlist *payload, unsigned nbytes, enum dma_data_direction dir) { - unsigned nents = sg_count(payload, nbytes); + int nents = sg_nents_for_len(payload, nbytes); + + if (nents < 0) { + dev_err(req->engine->dev, "Invalid numbers of SG.\n"); + return; + } dma_unmap_sg(req->engine->dev, payload, nents, dir); dma_pool_free(req->engine->req_pool, ddt, ddt_addr); @@ -835,8 +856,7 @@ static int spacc_ablk_need_fallback(struct spacc_req *req) static void spacc_ablk_complete(struct spacc_req *req) { - struct ablkcipher_request *ablk_req = - container_of(req->req, struct ablkcipher_request, base); + struct ablkcipher_request *ablk_req = ablkcipher_request_cast(req->req); if (ablk_req->src != ablk_req->dst) { spacc_free_ddt(req, req->src_ddt, req->src_addr, ablk_req->src, diff --git a/drivers/crypto/qat/Kconfig b/drivers/crypto/qat/Kconfig index eefccf7b8be7..85b44e577684 100644 --- a/drivers/crypto/qat/Kconfig +++ b/drivers/crypto/qat/Kconfig @@ -22,6 +22,28 @@ config CRYPTO_DEV_QAT_DH895xCC To compile this as a module, choose M here: the module will be called qat_dh895xcc. +config CRYPTO_DEV_QAT_C3XXX + tristate "Support for Intel(R) C3XXX" + depends on X86 && PCI + select CRYPTO_DEV_QAT + help + Support for Intel(R) C3xxx with Intel(R) QuickAssist Technology + for accelerating crypto and compression workloads. + + To compile this as a module, choose M here: the module + will be called qat_c3xxx. + +config CRYPTO_DEV_QAT_C62X + tristate "Support for Intel(R) C62X" + depends on X86 && PCI + select CRYPTO_DEV_QAT + help + Support for Intel(R) C62x with Intel(R) QuickAssist Technology + for accelerating crypto and compression workloads. + + To compile this as a module, choose M here: the module + will be called qat_c62x. + config CRYPTO_DEV_QAT_DH895xCCVF tristate "Support for Intel(R) DH895xCC Virtual Function" depends on X86 && PCI @@ -34,3 +56,27 @@ config CRYPTO_DEV_QAT_DH895xCCVF To compile this as a module, choose M here: the module will be called qat_dh895xccvf. + +config CRYPTO_DEV_QAT_C3XXXVF + tristate "Support for Intel(R) C3XXX Virtual Function" + depends on X86 && PCI + select PCI_IOV + select CRYPTO_DEV_QAT + help + Support for Intel(R) C3xxx with Intel(R) QuickAssist Technology + Virtual Function for accelerating crypto and compression workloads. + + To compile this as a module, choose M here: the module + will be called qat_c3xxxvf. + +config CRYPTO_DEV_QAT_C62XVF + tristate "Support for Intel(R) C62X Virtual Function" + depends on X86 && PCI + select PCI_IOV + select CRYPTO_DEV_QAT + help + Support for Intel(R) C62x with Intel(R) QuickAssist Technology + Virtual Function for accelerating crypto and compression workloads. + + To compile this as a module, choose M here: the module + will be called qat_c62xvf. diff --git a/drivers/crypto/qat/Makefile b/drivers/crypto/qat/Makefile index a3ce0b70e32f..8265106f1c8e 100644 --- a/drivers/crypto/qat/Makefile +++ b/drivers/crypto/qat/Makefile @@ -1,3 +1,7 @@ obj-$(CONFIG_CRYPTO_DEV_QAT) += qat_common/ obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc/ +obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXX) += qat_c3xxx/ +obj-$(CONFIG_CRYPTO_DEV_QAT_C62X) += qat_c62x/ obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCCVF) += qat_dh895xccvf/ +obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXXVF) += qat_c3xxxvf/ +obj-$(CONFIG_CRYPTO_DEV_QAT_C62XVF) += qat_c62xvf/ diff --git a/drivers/crypto/qat/qat_c3xxx/Makefile b/drivers/crypto/qat/qat_c3xxx/Makefile new file mode 100644 index 000000000000..8f5fd4838a96 --- /dev/null +++ b/drivers/crypto/qat/qat_c3xxx/Makefile @@ -0,0 +1,3 @@ +ccflags-y := -I$(src)/../qat_common +obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXX) += qat_c3xxx.o +qat_c3xxx-objs := adf_drv.o adf_c3xxx_hw_data.o diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c new file mode 100644 index 000000000000..c5bd5a9abc4d --- /dev/null +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c @@ -0,0 +1,238 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <adf_accel_devices.h> +#include <adf_common_drv.h> +#include <adf_pf2vf_msg.h> +#include "adf_c3xxx_hw_data.h" + +/* Worker thread to service arbiter mappings based on dev SKUs */ +static const u32 thrd_to_arb_map_6_me_sku[] = { + 0x12222AAA, 0x11222AAA, 0x12222AAA, + 0x11222AAA, 0x12222AAA, 0x11222AAA +}; + +static struct adf_hw_device_class c3xxx_class = { + .name = ADF_C3XXX_DEVICE_NAME, + .type = DEV_C3XXX, + .instances = 0 +}; + +static u32 get_accel_mask(u32 fuse) +{ + return (~fuse) >> ADF_C3XXX_ACCELERATORS_REG_OFFSET & + ADF_C3XXX_ACCELERATORS_MASK; +} + +static u32 get_ae_mask(u32 fuse) +{ + return (~fuse) & ADF_C3XXX_ACCELENGINES_MASK; +} + +static u32 get_num_accels(struct adf_hw_device_data *self) +{ + u32 i, ctr = 0; + + if (!self || !self->accel_mask) + return 0; + + for (i = 0; i < ADF_C3XXX_MAX_ACCELERATORS; i++) { + if (self->accel_mask & (1 << i)) + ctr++; + } + return ctr; +} + +static u32 get_num_aes(struct adf_hw_device_data *self) +{ + u32 i, ctr = 0; + + if (!self || !self->ae_mask) + return 0; + + for (i = 0; i < ADF_C3XXX_MAX_ACCELENGINES; i++) { + if (self->ae_mask & (1 << i)) + ctr++; + } + return ctr; +} + +static u32 get_misc_bar_id(struct adf_hw_device_data *self) +{ + return ADF_C3XXX_PMISC_BAR; +} + +static u32 get_etr_bar_id(struct adf_hw_device_data *self) +{ + return ADF_C3XXX_ETR_BAR; +} + +static u32 get_sram_bar_id(struct adf_hw_device_data *self) +{ + return 0; +} + +static enum dev_sku_info get_sku(struct adf_hw_device_data *self) +{ + int aes = get_num_aes(self); + + if (aes == 6) + return DEV_SKU_4; + + return DEV_SKU_UNKNOWN; +} + +static void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, + u32 const **arb_map_config) +{ + switch (accel_dev->accel_pci_dev.sku) { + case DEV_SKU_4: + *arb_map_config = thrd_to_arb_map_6_me_sku; + break; + default: + dev_err(&GET_DEV(accel_dev), + "The configuration doesn't match any SKU"); + *arb_map_config = NULL; + } +} + +static u32 get_pf2vf_offset(u32 i) +{ + return ADF_C3XXX_PF2VF_OFFSET(i); +} + +static u32 get_vintmsk_offset(u32 i) +{ + return ADF_C3XXX_VINTMSK_OFFSET(i); +} + +static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_C3XXX_PMISC_BAR]; + void __iomem *csr = misc_bar->virt_addr; + unsigned int val, i; + + /* Enable Accel Engine error detection & correction */ + for (i = 0; i < hw_device->get_num_aes(hw_device); i++) { + val = ADF_CSR_RD(csr, ADF_C3XXX_AE_CTX_ENABLES(i)); + val |= ADF_C3XXX_ENABLE_AE_ECC_ERR; + ADF_CSR_WR(csr, ADF_C3XXX_AE_CTX_ENABLES(i), val); + val = ADF_CSR_RD(csr, ADF_C3XXX_AE_MISC_CONTROL(i)); + val |= ADF_C3XXX_ENABLE_AE_ECC_PARITY_CORR; + ADF_CSR_WR(csr, ADF_C3XXX_AE_MISC_CONTROL(i), val); + } + + /* Enable shared memory error detection & correction */ + for (i = 0; i < hw_device->get_num_accels(hw_device); i++) { + val = ADF_CSR_RD(csr, ADF_C3XXX_UERRSSMSH(i)); + val |= ADF_C3XXX_ERRSSMSH_EN; + ADF_CSR_WR(csr, ADF_C3XXX_UERRSSMSH(i), val); + val = ADF_CSR_RD(csr, ADF_C3XXX_CERRSSMSH(i)); + val |= ADF_C3XXX_ERRSSMSH_EN; + ADF_CSR_WR(csr, ADF_C3XXX_CERRSSMSH(i), val); + } +} + +static void adf_enable_ints(struct adf_accel_dev *accel_dev) +{ + void __iomem *addr; + + addr = (&GET_BARS(accel_dev)[ADF_C3XXX_PMISC_BAR])->virt_addr; + + /* Enable bundle and misc interrupts */ + ADF_CSR_WR(addr, ADF_C3XXX_SMIAPF0_MASK_OFFSET, + ADF_C3XXX_SMIA0_MASK); + ADF_CSR_WR(addr, ADF_C3XXX_SMIAPF1_MASK_OFFSET, + ADF_C3XXX_SMIA1_MASK); +} + +static int adf_pf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class = &c3xxx_class; + hw_data->instance_id = c3xxx_class.instances++; + hw_data->num_banks = ADF_C3XXX_ETR_MAX_BANKS; + hw_data->num_accel = ADF_C3XXX_MAX_ACCELERATORS; + hw_data->num_logical_accel = 1; + hw_data->num_engines = ADF_C3XXX_MAX_ACCELENGINES; + hw_data->tx_rx_gap = ADF_C3XXX_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_C3XXX_TX_RINGS_MASK; + hw_data->alloc_irq = adf_isr_resource_alloc; + hw_data->free_irq = adf_isr_resource_free; + hw_data->enable_error_correction = adf_enable_error_correction; + hw_data->get_accel_mask = get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_num_accels = get_num_accels; + hw_data->get_num_aes = get_num_aes; + hw_data->get_sram_bar_id = get_sram_bar_id; + hw_data->get_etr_bar_id = get_etr_bar_id; + hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_pf2vf_offset = get_pf2vf_offset; + hw_data->get_vintmsk_offset = get_vintmsk_offset; + hw_data->get_sku = get_sku; + hw_data->fw_name = ADF_C3XXX_FW; + hw_data->fw_mmp_name = ADF_C3XXX_MMP; + hw_data->init_admin_comms = adf_init_admin_comms; + hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->disable_iov = adf_disable_sriov; + hw_data->send_admin_init = adf_send_admin_init; + hw_data->init_arb = adf_init_arb; + hw_data->exit_arb = adf_exit_arb; + hw_data->get_arb_mapping = adf_get_arbiter_mapping; + hw_data->enable_ints = adf_enable_ints; + hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; + hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; +} + +void adf_clean_hw_data_c3xxx(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class->instances--; +} diff --git a/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h new file mode 100644 index 000000000000..2f2681d3458a --- /dev/null +++ b/drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.h @@ -0,0 +1,83 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef ADF_C3XXX_HW_DATA_H_ +#define ADF_C3XXX_HW_DATA_H_ + +/* PCIe configuration space */ +#define ADF_C3XXX_PMISC_BAR 0 +#define ADF_C3XXX_ETR_BAR 1 +#define ADF_C3XXX_RX_RINGS_OFFSET 8 +#define ADF_C3XXX_TX_RINGS_MASK 0xFF +#define ADF_C3XXX_MAX_ACCELERATORS 3 +#define ADF_C3XXX_MAX_ACCELENGINES 6 +#define ADF_C3XXX_ACCELERATORS_REG_OFFSET 16 +#define ADF_C3XXX_ACCELERATORS_MASK 0x3 +#define ADF_C3XXX_ACCELENGINES_MASK 0x3F +#define ADF_C3XXX_ETR_MAX_BANKS 16 +#define ADF_C3XXX_SMIAPF0_MASK_OFFSET (0x3A000 + 0x28) +#define ADF_C3XXX_SMIAPF1_MASK_OFFSET (0x3A000 + 0x30) +#define ADF_C3XXX_SMIA0_MASK 0xFFFF +#define ADF_C3XXX_SMIA1_MASK 0x1 +/* Error detection and correction */ +#define ADF_C3XXX_AE_CTX_ENABLES(i) (i * 0x1000 + 0x20818) +#define ADF_C3XXX_AE_MISC_CONTROL(i) (i * 0x1000 + 0x20960) +#define ADF_C3XXX_ENABLE_AE_ECC_ERR BIT(28) +#define ADF_C3XXX_ENABLE_AE_ECC_PARITY_CORR (BIT(24) | BIT(12)) +#define ADF_C3XXX_UERRSSMSH(i) (i * 0x4000 + 0x18) +#define ADF_C3XXX_CERRSSMSH(i) (i * 0x4000 + 0x10) +#define ADF_C3XXX_ERRSSMSH_EN BIT(3) + +#define ADF_C3XXX_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) +#define ADF_C3XXX_VINTMSK_OFFSET(i) (0x3A000 + 0x200 + ((i) * 0x04)) + +/* Firmware Binary */ +#define ADF_C3XXX_FW "qat_c3xxx.bin" +#define ADF_C3XXX_MMP "qat_c3xxx_mmp.bin" + +void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data); +void adf_clean_hw_data_c3xxx(struct adf_hw_device_data *hw_data); +#endif diff --git a/drivers/crypto/qat/qat_c3xxx/adf_drv.c b/drivers/crypto/qat/qat_c3xxx/adf_drv.c new file mode 100644 index 000000000000..e13bd08ddd1e --- /dev/null +++ b/drivers/crypto/qat/qat_c3xxx/adf_drv.c @@ -0,0 +1,335 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/init.h> +#include <linux/types.h> +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/errno.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/platform_device.h> +#include <linux/workqueue.h> +#include <linux/io.h> +#include <adf_accel_devices.h> +#include <adf_common_drv.h> +#include <adf_cfg.h> +#include "adf_c3xxx_hw_data.h" + +#define ADF_SYSTEM_DEVICE(device_id) \ + {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} + +static const struct pci_device_id adf_pci_tbl[] = { + ADF_SYSTEM_DEVICE(ADF_C3XXX_PCI_DEVICE_ID), + {0,} +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); +static void adf_remove(struct pci_dev *dev); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_C3XXX_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, + .sriov_configure = adf_sriov_configure, +}; + +static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) +{ + pci_release_regions(accel_dev->accel_pci_dev.pci_dev); + pci_disable_device(accel_dev->accel_pci_dev.pci_dev); +} + +static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; + int i; + + for (i = 0; i < ADF_PCI_MAX_BARS; i++) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; + + if (bar->virt_addr) + pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr); + } + + if (accel_dev->hw_device) { + switch (accel_pci_dev->pci_dev->device) { + case ADF_C3XXX_PCI_DEVICE_ID: + adf_clean_hw_data_c3xxx(accel_dev->hw_device); + break; + default: + break; + } + kfree(accel_dev->hw_device); + accel_dev->hw_device = NULL; + } + adf_cfg_dev_remove(accel_dev); + debugfs_remove(accel_dev->debugfs_dir); + adf_devmgr_rm_dev(accel_dev, NULL); +} + +static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct adf_accel_dev *accel_dev; + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + char name[ADF_DEVICE_NAME_LENGTH]; + unsigned int i, bar_nr; + int ret, bar_mask; + + switch (ent->device) { + case ADF_C3XXX_PCI_DEVICE_ID: + break; + default: + dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); + return -ENODEV; + } + + if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) { + /* If the accelerator is connected to a node with no memory + * there is no point in using the accelerator since the remote + * memory transaction will be very slow. */ + dev_err(&pdev->dev, "Invalid NUMA configuration.\n"); + return -EINVAL; + } + + accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!accel_dev) + return -ENOMEM; + + INIT_LIST_HEAD(&accel_dev->crypto_list); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; + + /* Add accel device to accel table. + * This should be called before adf_cleanup_accel is called */ + if (adf_devmgr_add_dev(accel_dev, NULL)) { + dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); + kfree(accel_dev); + return -EFAULT; + } + + accel_dev->owner = THIS_MODULE; + /* Allocate and configure device configuration structure */ + hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!hw_data) { + ret = -ENOMEM; + goto out_err; + } + + accel_dev->hw_device = hw_data; + adf_init_hw_data_c3xxx(accel_dev->hw_device); + pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); + pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, + &hw_data->fuses); + + /* Get Accelerators and Accelerators Engines masks */ + hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + accel_pci_dev->sku = hw_data->get_sku(hw_data); + /* If the device has no acceleration engines then ignore it. */ + if (!hw_data->accel_mask || !hw_data->ae_mask || + ((~hw_data->ae_mask) & 0x01)) { + dev_err(&pdev->dev, "No acceleration units found"); + ret = -EFAULT; + goto out_err; + } + + /* Create dev top level debugfs entry */ + snprintf(name, sizeof(name), "%s%s_%02x:%02d.%02d", + ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, + pdev->bus->number, PCI_SLOT(pdev->devfn), + PCI_FUNC(pdev->devfn)); + + accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); + if (!accel_dev->debugfs_dir) { + dev_err(&pdev->dev, "Could not create debugfs dir %s\n", name); + ret = -EINVAL; + goto out_err; + } + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + goto out_err; + + /* enable PCI device */ + if (pci_enable_device(pdev)) { + ret = -EFAULT; + goto out_err; + } + + /* set dma identifier */ + if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { + if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) { + dev_err(&pdev->dev, "No usable DMA configuration\n"); + ret = -EFAULT; + goto out_err_disable; + } else { + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + } + + } else { + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); + } + + if (pci_request_regions(pdev, ADF_C3XXX_DEVICE_NAME)) { + ret = -EFAULT; + goto out_err_disable; + } + + /* Read accelerator capabilities mask */ + pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, + &hw_data->accel_capabilities_mask); + + /* Find and map all the device's BARS */ + i = 0; + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + for_each_set_bit(bar_nr, (const unsigned long *)&bar_mask, + ADF_PCI_MAX_BARS * 2) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; + + bar->base_addr = pci_resource_start(pdev, bar_nr); + if (!bar->base_addr) + break; + bar->size = pci_resource_len(pdev, bar_nr); + bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); + if (!bar->virt_addr) { + dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); + ret = -EFAULT; + goto out_err_free_reg; + } + } + pci_set_master(pdev); + + if (adf_enable_aer(accel_dev, &adf_driver)) { + dev_err(&pdev->dev, "Failed to enable aer\n"); + ret = -EFAULT; + goto out_err_free_reg; + } + + if (pci_save_state(pdev)) { + dev_err(&pdev->dev, "Failed to save pci state\n"); + ret = -ENOMEM; + goto out_err_free_reg; + } + + ret = qat_crypto_dev_config(accel_dev); + if (ret) + goto out_err_free_reg; + + ret = adf_dev_init(accel_dev); + if (ret) + goto out_err_dev_shutdown; + + ret = adf_dev_start(accel_dev); + if (ret) + goto out_err_dev_stop; + + return ret; + +out_err_dev_stop: + adf_dev_stop(accel_dev); +out_err_dev_shutdown: + adf_dev_shutdown(accel_dev); +out_err_free_reg: + pci_release_regions(accel_pci_dev->pci_dev); +out_err_disable: + pci_disable_device(accel_pci_dev->pci_dev); +out_err: + adf_cleanup_accel(accel_dev); + kfree(accel_dev); + return ret; +} + +static void adf_remove(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + pr_err("QAT: Driver removal failed\n"); + return; + } + if (adf_dev_stop(accel_dev)) + dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n"); + + adf_dev_shutdown(accel_dev); + adf_disable_aer(accel_dev); + adf_cleanup_accel(accel_dev); + adf_cleanup_pci_dev(accel_dev); + kfree(accel_dev); +} + +static int __init adfdrv_init(void) +{ + request_module("intel_qat"); + + if (pci_register_driver(&adf_driver)) { + pr_err("QAT: Driver initialization failed\n"); + return -EFAULT; + } + return 0; +} + +static void __exit adfdrv_release(void) +{ + pci_unregister_driver(&adf_driver); +} + +module_init(adfdrv_init); +module_exit(adfdrv_release); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel"); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); +MODULE_VERSION(ADF_DRV_VERSION); diff --git a/drivers/crypto/qat/qat_c3xxxvf/Makefile b/drivers/crypto/qat/qat_c3xxxvf/Makefile new file mode 100644 index 000000000000..16d178e2eaa2 --- /dev/null +++ b/drivers/crypto/qat/qat_c3xxxvf/Makefile @@ -0,0 +1,3 @@ +ccflags-y := -I$(src)/../qat_common +obj-$(CONFIG_CRYPTO_DEV_QAT_C3XXXVF) += qat_c3xxxvf.o +qat_c3xxxvf-objs := adf_drv.o adf_c3xxxvf_hw_data.o diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c new file mode 100644 index 000000000000..1af321c2ce1a --- /dev/null +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c @@ -0,0 +1,173 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2015 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2015 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <adf_accel_devices.h> +#include <adf_pf2vf_msg.h> +#include <adf_common_drv.h> +#include "adf_c3xxxvf_hw_data.h" + +static struct adf_hw_device_class c3xxxiov_class = { + .name = ADF_C3XXXVF_DEVICE_NAME, + .type = DEV_C3XXXVF, + .instances = 0 +}; + +static u32 get_accel_mask(u32 fuse) +{ + return ADF_C3XXXIOV_ACCELERATORS_MASK; +} + +static u32 get_ae_mask(u32 fuse) +{ + return ADF_C3XXXIOV_ACCELENGINES_MASK; +} + +static u32 get_num_accels(struct adf_hw_device_data *self) +{ + return ADF_C3XXXIOV_MAX_ACCELERATORS; +} + +static u32 get_num_aes(struct adf_hw_device_data *self) +{ + return ADF_C3XXXIOV_MAX_ACCELENGINES; +} + +static u32 get_misc_bar_id(struct adf_hw_device_data *self) +{ + return ADF_C3XXXIOV_PMISC_BAR; +} + +static u32 get_etr_bar_id(struct adf_hw_device_data *self) +{ + return ADF_C3XXXIOV_ETR_BAR; +} + +static enum dev_sku_info get_sku(struct adf_hw_device_data *self) +{ + return DEV_SKU_VF; +} + +static u32 get_pf2vf_offset(u32 i) +{ + return ADF_C3XXXIOV_PF2VF_OFFSET; +} + +static u32 get_vintmsk_offset(u32 i) +{ + return ADF_C3XXXIOV_VINTMSK_OFFSET; +} + +static int adf_vf_int_noop(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +static void adf_vf_void_noop(struct adf_accel_dev *accel_dev) +{ +} + +static int adf_vf2pf_init(struct adf_accel_dev *accel_dev) +{ + u32 msg = (ADF_VF2PF_MSGORIGIN_SYSTEM | + (ADF_VF2PF_MSGTYPE_INIT << ADF_VF2PF_MSGTYPE_SHIFT)); + + if (adf_iov_putmsg(accel_dev, msg, 0)) { + dev_err(&GET_DEV(accel_dev), + "Failed to send Init event to PF\n"); + return -EFAULT; + } + return 0; +} + +static void adf_vf2pf_shutdown(struct adf_accel_dev *accel_dev) +{ + u32 msg = (ADF_VF2PF_MSGORIGIN_SYSTEM | + (ADF_VF2PF_MSGTYPE_SHUTDOWN << ADF_VF2PF_MSGTYPE_SHIFT)); + + if (adf_iov_putmsg(accel_dev, msg, 0)) + dev_err(&GET_DEV(accel_dev), + "Failed to send Shutdown event to PF\n"); +} + +void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class = &c3xxxiov_class; + hw_data->num_banks = ADF_C3XXXIOV_ETR_MAX_BANKS; + hw_data->num_accel = ADF_C3XXXIOV_MAX_ACCELERATORS; + hw_data->num_logical_accel = 1; + hw_data->num_engines = ADF_C3XXXIOV_MAX_ACCELENGINES; + hw_data->tx_rx_gap = ADF_C3XXXIOV_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_C3XXXIOV_TX_RINGS_MASK; + hw_data->alloc_irq = adf_vf_isr_resource_alloc; + hw_data->free_irq = adf_vf_isr_resource_free; + hw_data->enable_error_correction = adf_vf_void_noop; + hw_data->init_admin_comms = adf_vf_int_noop; + hw_data->exit_admin_comms = adf_vf_void_noop; + hw_data->send_admin_init = adf_vf2pf_init; + hw_data->init_arb = adf_vf_int_noop; + hw_data->exit_arb = adf_vf_void_noop; + hw_data->disable_iov = adf_vf2pf_shutdown; + hw_data->get_accel_mask = get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_num_accels = get_num_accels; + hw_data->get_num_aes = get_num_aes; + hw_data->get_etr_bar_id = get_etr_bar_id; + hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_pf2vf_offset = get_pf2vf_offset; + hw_data->get_vintmsk_offset = get_vintmsk_offset; + hw_data->get_sku = get_sku; + hw_data->enable_ints = adf_vf_void_noop; + hw_data->enable_vf2pf_comms = adf_enable_vf2pf_comms; + hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; + hw_data->dev_class->instances++; + adf_devmgr_update_class_index(hw_data); +} + +void adf_clean_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class->instances--; + adf_devmgr_update_class_index(hw_data); +} diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.h b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h index e270e4a63d14..934f216acf39 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.h +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h @@ -3,7 +3,7 @@ redistributing this file, you may do so under either license. GPL LICENSE SUMMARY - Copyright(c) 2014 Intel Corporation. + Copyright(c) 2015 Intel Corporation. This program is free software; you can redistribute it and/or modify it under the terms of version 2 of the GNU General Public License as published by the Free Software Foundation. @@ -17,7 +17,7 @@ qat-linux@intel.com BSD LICENSE - Copyright(c) 2014 Intel Corporation. + Copyright(c) 2015 Intel Corporation. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -44,14 +44,21 @@ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef ADF_DH895xVF_DRV_H_ -#define ADF_DH895xVF_DRV_H_ -#include <adf_accel_devices.h> -#include <adf_transport.h> - -void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data); -void adf_clean_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data); -int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev); -void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev); -void adf_update_ring_arb_enable(struct adf_etr_ring_data *ring); +#ifndef ADF_C3XXXVF_HW_DATA_H_ +#define ADF_C3XXXVF_HW_DATA_H_ + +#define ADF_C3XXXIOV_PMISC_BAR 1 +#define ADF_C3XXXIOV_ACCELERATORS_MASK 0x1 +#define ADF_C3XXXIOV_ACCELENGINES_MASK 0x1 +#define ADF_C3XXXIOV_MAX_ACCELERATORS 1 +#define ADF_C3XXXIOV_MAX_ACCELENGINES 1 +#define ADF_C3XXXIOV_RX_RINGS_OFFSET 8 +#define ADF_C3XXXIOV_TX_RINGS_MASK 0xFF +#define ADF_C3XXXIOV_ETR_BAR 0 +#define ADF_C3XXXIOV_ETR_MAX_BANKS 1 +#define ADF_C3XXXIOV_PF2VF_OFFSET 0x200 +#define ADF_C3XXXIOV_VINTMSK_OFFSET 0x208 + +void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data); +void adf_clean_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data); #endif diff --git a/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c new file mode 100644 index 000000000000..1ac4ae90e072 --- /dev/null +++ b/drivers/crypto/qat/qat_c3xxxvf/adf_drv.c @@ -0,0 +1,305 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/init.h> +#include <linux/types.h> +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/errno.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/platform_device.h> +#include <linux/workqueue.h> +#include <linux/io.h> +#include <adf_accel_devices.h> +#include <adf_common_drv.h> +#include <adf_cfg.h> +#include "adf_c3xxxvf_hw_data.h" + +#define ADF_SYSTEM_DEVICE(device_id) \ + {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} + +static const struct pci_device_id adf_pci_tbl[] = { + ADF_SYSTEM_DEVICE(ADF_C3XXXIOV_PCI_DEVICE_ID), + {0,} +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); +static void adf_remove(struct pci_dev *dev); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_C3XXXVF_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, +}; + +static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) +{ + pci_release_regions(accel_dev->accel_pci_dev.pci_dev); + pci_disable_device(accel_dev->accel_pci_dev.pci_dev); +} + +static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; + struct adf_accel_dev *pf; + int i; + + for (i = 0; i < ADF_PCI_MAX_BARS; i++) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; + + if (bar->virt_addr) + pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr); + } + + if (accel_dev->hw_device) { + switch (accel_pci_dev->pci_dev->device) { + case ADF_C3XXXIOV_PCI_DEVICE_ID: + adf_clean_hw_data_c3xxxiov(accel_dev->hw_device); + break; + default: + break; + } + kfree(accel_dev->hw_device); + accel_dev->hw_device = NULL; + } + adf_cfg_dev_remove(accel_dev); + debugfs_remove(accel_dev->debugfs_dir); + pf = adf_devmgr_pci_to_accel_dev(accel_pci_dev->pci_dev->physfn); + adf_devmgr_rm_dev(accel_dev, pf); +} + +static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct adf_accel_dev *accel_dev; + struct adf_accel_dev *pf; + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + char name[ADF_DEVICE_NAME_LENGTH]; + unsigned int i, bar_nr; + int ret, bar_mask; + + switch (ent->device) { + case ADF_C3XXXIOV_PCI_DEVICE_ID: + break; + default: + dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); + return -ENODEV; + } + + accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!accel_dev) + return -ENOMEM; + + accel_dev->is_vf = true; + pf = adf_devmgr_pci_to_accel_dev(pdev->physfn); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; + + /* Add accel device to accel table */ + if (adf_devmgr_add_dev(accel_dev, pf)) { + dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); + kfree(accel_dev); + return -EFAULT; + } + INIT_LIST_HEAD(&accel_dev->crypto_list); + + accel_dev->owner = THIS_MODULE; + /* Allocate and configure device configuration structure */ + hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!hw_data) { + ret = -ENOMEM; + goto out_err; + } + accel_dev->hw_device = hw_data; + adf_init_hw_data_c3xxxiov(accel_dev->hw_device); + + /* Get Accelerators and Accelerators Engines masks */ + hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + accel_pci_dev->sku = hw_data->get_sku(hw_data); + + /* Create dev top level debugfs entry */ + snprintf(name, sizeof(name), "%s%s_%02x:%02d.%02d", + ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, + pdev->bus->number, PCI_SLOT(pdev->devfn), + PCI_FUNC(pdev->devfn)); + + accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); + if (!accel_dev->debugfs_dir) { + dev_err(&pdev->dev, "Could not create debugfs dir %s\n", name); + ret = -EINVAL; + goto out_err; + } + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + goto out_err; + + /* enable PCI device */ + if (pci_enable_device(pdev)) { + ret = -EFAULT; + goto out_err; + } + + /* set dma identifier */ + if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { + if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) { + dev_err(&pdev->dev, "No usable DMA configuration\n"); + ret = -EFAULT; + goto out_err_disable; + } else { + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + } + + } else { + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); + } + + if (pci_request_regions(pdev, ADF_C3XXXVF_DEVICE_NAME)) { + ret = -EFAULT; + goto out_err_disable; + } + + /* Find and map all the device's BARS */ + i = 0; + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + for_each_set_bit(bar_nr, (const unsigned long *)&bar_mask, + ADF_PCI_MAX_BARS * 2) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; + + bar->base_addr = pci_resource_start(pdev, bar_nr); + if (!bar->base_addr) + break; + bar->size = pci_resource_len(pdev, bar_nr); + bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); + if (!bar->virt_addr) { + dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); + ret = -EFAULT; + goto out_err_free_reg; + } + } + pci_set_master(pdev); + /* Completion for VF2PF request/response message exchange */ + init_completion(&accel_dev->vf.iov_msg_completion); + + ret = qat_crypto_dev_config(accel_dev); + if (ret) + goto out_err_free_reg; + + ret = adf_dev_init(accel_dev); + if (ret) + goto out_err_dev_shutdown; + + ret = adf_dev_start(accel_dev); + if (ret) + goto out_err_dev_stop; + + return ret; + +out_err_dev_stop: + adf_dev_stop(accel_dev); +out_err_dev_shutdown: + adf_dev_shutdown(accel_dev); +out_err_free_reg: + pci_release_regions(accel_pci_dev->pci_dev); +out_err_disable: + pci_disable_device(accel_pci_dev->pci_dev); +out_err: + adf_cleanup_accel(accel_dev); + kfree(accel_dev); + return ret; +} + +static void adf_remove(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + pr_err("QAT: Driver removal failed\n"); + return; + } + if (adf_dev_stop(accel_dev)) + dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n"); + + adf_dev_shutdown(accel_dev); + adf_cleanup_accel(accel_dev); + adf_cleanup_pci_dev(accel_dev); + kfree(accel_dev); +} + +static int __init adfdrv_init(void) +{ + request_module("intel_qat"); + + if (pci_register_driver(&adf_driver)) { + pr_err("QAT: Driver initialization failed\n"); + return -EFAULT; + } + return 0; +} + +static void __exit adfdrv_release(void) +{ + pci_unregister_driver(&adf_driver); + adf_clean_vf_map(true); +} + +module_init(adfdrv_init); +module_exit(adfdrv_release); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel"); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); +MODULE_VERSION(ADF_DRV_VERSION); diff --git a/drivers/crypto/qat/qat_c62x/Makefile b/drivers/crypto/qat/qat_c62x/Makefile new file mode 100644 index 000000000000..bd75ace59b76 --- /dev/null +++ b/drivers/crypto/qat/qat_c62x/Makefile @@ -0,0 +1,3 @@ +ccflags-y := -I$(src)/../qat_common +obj-$(CONFIG_CRYPTO_DEV_QAT_C62X) += qat_c62x.o +qat_c62x-objs := adf_drv.o adf_c62x_hw_data.o diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c new file mode 100644 index 000000000000..879e04cae714 --- /dev/null +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c @@ -0,0 +1,248 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <adf_accel_devices.h> +#include <adf_common_drv.h> +#include <adf_pf2vf_msg.h> +#include "adf_c62x_hw_data.h" + +/* Worker thread to service arbiter mappings based on dev SKUs */ +static const u32 thrd_to_arb_map_8_me_sku[] = { + 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, + 0x11222AAA, 0x12222AAA, 0x11222AAA, 0, 0 +}; + +static const u32 thrd_to_arb_map_10_me_sku[] = { + 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, + 0x11222AAA, 0x12222AAA, 0x11222AAA, 0x12222AAA, 0x11222AAA +}; + +static struct adf_hw_device_class c62x_class = { + .name = ADF_C62X_DEVICE_NAME, + .type = DEV_C62X, + .instances = 0 +}; + +static u32 get_accel_mask(u32 fuse) +{ + return (~fuse) >> ADF_C62X_ACCELERATORS_REG_OFFSET & + ADF_C62X_ACCELERATORS_MASK; +} + +static u32 get_ae_mask(u32 fuse) +{ + return (~fuse) & ADF_C62X_ACCELENGINES_MASK; +} + +static u32 get_num_accels(struct adf_hw_device_data *self) +{ + u32 i, ctr = 0; + + if (!self || !self->accel_mask) + return 0; + + for (i = 0; i < ADF_C62X_MAX_ACCELERATORS; i++) { + if (self->accel_mask & (1 << i)) + ctr++; + } + return ctr; +} + +static u32 get_num_aes(struct adf_hw_device_data *self) +{ + u32 i, ctr = 0; + + if (!self || !self->ae_mask) + return 0; + + for (i = 0; i < ADF_C62X_MAX_ACCELENGINES; i++) { + if (self->ae_mask & (1 << i)) + ctr++; + } + return ctr; +} + +static u32 get_misc_bar_id(struct adf_hw_device_data *self) +{ + return ADF_C62X_PMISC_BAR; +} + +static u32 get_etr_bar_id(struct adf_hw_device_data *self) +{ + return ADF_C62X_ETR_BAR; +} + +static u32 get_sram_bar_id(struct adf_hw_device_data *self) +{ + return ADF_C62X_SRAM_BAR; +} + +static enum dev_sku_info get_sku(struct adf_hw_device_data *self) +{ + int aes = get_num_aes(self); + + if (aes == 8) + return DEV_SKU_2; + else if (aes == 10) + return DEV_SKU_4; + + return DEV_SKU_UNKNOWN; +} + +static void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, + u32 const **arb_map_config) +{ + switch (accel_dev->accel_pci_dev.sku) { + case DEV_SKU_2: + *arb_map_config = thrd_to_arb_map_8_me_sku; + break; + case DEV_SKU_4: + *arb_map_config = thrd_to_arb_map_10_me_sku; + break; + default: + dev_err(&GET_DEV(accel_dev), + "The configuration doesn't match any SKU"); + *arb_map_config = NULL; + } +} + +static u32 get_pf2vf_offset(u32 i) +{ + return ADF_C62X_PF2VF_OFFSET(i); +} + +static u32 get_vintmsk_offset(u32 i) +{ + return ADF_C62X_VINTMSK_OFFSET(i); +} + +static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_device = accel_dev->hw_device; + struct adf_bar *misc_bar = &GET_BARS(accel_dev)[ADF_C62X_PMISC_BAR]; + void __iomem *csr = misc_bar->virt_addr; + unsigned int val, i; + + /* Enable Accel Engine error detection & correction */ + for (i = 0; i < hw_device->get_num_aes(hw_device); i++) { + val = ADF_CSR_RD(csr, ADF_C62X_AE_CTX_ENABLES(i)); + val |= ADF_C62X_ENABLE_AE_ECC_ERR; + ADF_CSR_WR(csr, ADF_C62X_AE_CTX_ENABLES(i), val); + val = ADF_CSR_RD(csr, ADF_C62X_AE_MISC_CONTROL(i)); + val |= ADF_C62X_ENABLE_AE_ECC_PARITY_CORR; + ADF_CSR_WR(csr, ADF_C62X_AE_MISC_CONTROL(i), val); + } + + /* Enable shared memory error detection & correction */ + for (i = 0; i < hw_device->get_num_accels(hw_device); i++) { + val = ADF_CSR_RD(csr, ADF_C62X_UERRSSMSH(i)); + val |= ADF_C62X_ERRSSMSH_EN; + ADF_CSR_WR(csr, ADF_C62X_UERRSSMSH(i), val); + val = ADF_CSR_RD(csr, ADF_C62X_CERRSSMSH(i)); + val |= ADF_C62X_ERRSSMSH_EN; + ADF_CSR_WR(csr, ADF_C62X_CERRSSMSH(i), val); + } +} + +static void adf_enable_ints(struct adf_accel_dev *accel_dev) +{ + void __iomem *addr; + + addr = (&GET_BARS(accel_dev)[ADF_C62X_PMISC_BAR])->virt_addr; + + /* Enable bundle and misc interrupts */ + ADF_CSR_WR(addr, ADF_C62X_SMIAPF0_MASK_OFFSET, + ADF_C62X_SMIA0_MASK); + ADF_CSR_WR(addr, ADF_C62X_SMIAPF1_MASK_OFFSET, + ADF_C62X_SMIA1_MASK); +} + +static int adf_pf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class = &c62x_class; + hw_data->instance_id = c62x_class.instances++; + hw_data->num_banks = ADF_C62X_ETR_MAX_BANKS; + hw_data->num_accel = ADF_C62X_MAX_ACCELERATORS; + hw_data->num_logical_accel = 1; + hw_data->num_engines = ADF_C62X_MAX_ACCELENGINES; + hw_data->tx_rx_gap = ADF_C62X_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_C62X_TX_RINGS_MASK; + hw_data->alloc_irq = adf_isr_resource_alloc; + hw_data->free_irq = adf_isr_resource_free; + hw_data->enable_error_correction = adf_enable_error_correction; + hw_data->get_accel_mask = get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_num_accels = get_num_accels; + hw_data->get_num_aes = get_num_aes; + hw_data->get_sram_bar_id = get_sram_bar_id; + hw_data->get_etr_bar_id = get_etr_bar_id; + hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_pf2vf_offset = get_pf2vf_offset; + hw_data->get_vintmsk_offset = get_vintmsk_offset; + hw_data->get_sku = get_sku; + hw_data->fw_name = ADF_C62X_FW; + hw_data->fw_mmp_name = ADF_C62X_MMP; + hw_data->init_admin_comms = adf_init_admin_comms; + hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->disable_iov = adf_disable_sriov; + hw_data->send_admin_init = adf_send_admin_init; + hw_data->init_arb = adf_init_arb; + hw_data->exit_arb = adf_exit_arb; + hw_data->get_arb_mapping = adf_get_arbiter_mapping; + hw_data->enable_ints = adf_enable_ints; + hw_data->enable_vf2pf_comms = adf_pf_enable_vf2pf_comms; + hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; +} + +void adf_clean_hw_data_c62x(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class->instances--; +} diff --git a/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h new file mode 100644 index 000000000000..17a8a32d5c63 --- /dev/null +++ b/drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.h @@ -0,0 +1,84 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef ADF_C62X_HW_DATA_H_ +#define ADF_C62X_HW_DATA_H_ + +/* PCIe configuration space */ +#define ADF_C62X_SRAM_BAR 0 +#define ADF_C62X_PMISC_BAR 1 +#define ADF_C62X_ETR_BAR 2 +#define ADF_C62X_RX_RINGS_OFFSET 8 +#define ADF_C62X_TX_RINGS_MASK 0xFF +#define ADF_C62X_MAX_ACCELERATORS 5 +#define ADF_C62X_MAX_ACCELENGINES 10 +#define ADF_C62X_ACCELERATORS_REG_OFFSET 16 +#define ADF_C62X_ACCELERATORS_MASK 0x1F +#define ADF_C62X_ACCELENGINES_MASK 0x3FF +#define ADF_C62X_ETR_MAX_BANKS 16 +#define ADF_C62X_SMIAPF0_MASK_OFFSET (0x3A000 + 0x28) +#define ADF_C62X_SMIAPF1_MASK_OFFSET (0x3A000 + 0x30) +#define ADF_C62X_SMIA0_MASK 0xFFFF +#define ADF_C62X_SMIA1_MASK 0x1 +/* Error detection and correction */ +#define ADF_C62X_AE_CTX_ENABLES(i) (i * 0x1000 + 0x20818) +#define ADF_C62X_AE_MISC_CONTROL(i) (i * 0x1000 + 0x20960) +#define ADF_C62X_ENABLE_AE_ECC_ERR BIT(28) +#define ADF_C62X_ENABLE_AE_ECC_PARITY_CORR (BIT(24) | BIT(12)) +#define ADF_C62X_UERRSSMSH(i) (i * 0x4000 + 0x18) +#define ADF_C62X_CERRSSMSH(i) (i * 0x4000 + 0x10) +#define ADF_C62X_ERRSSMSH_EN BIT(3) + +#define ADF_C62X_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) +#define ADF_C62X_VINTMSK_OFFSET(i) (0x3A000 + 0x200 + ((i) * 0x04)) + +/* Firmware Binary */ +#define ADF_C62X_FW "qat_c62x.bin" +#define ADF_C62X_MMP "qat_c62x_mmp.bin" + +void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data); +void adf_clean_hw_data_c62x(struct adf_hw_device_data *hw_data); +#endif diff --git a/drivers/crypto/qat/qat_c62x/adf_drv.c b/drivers/crypto/qat/qat_c62x/adf_drv.c new file mode 100644 index 000000000000..512c56509718 --- /dev/null +++ b/drivers/crypto/qat/qat_c62x/adf_drv.c @@ -0,0 +1,335 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/init.h> +#include <linux/types.h> +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/errno.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/platform_device.h> +#include <linux/workqueue.h> +#include <linux/io.h> +#include <adf_accel_devices.h> +#include <adf_common_drv.h> +#include <adf_cfg.h> +#include "adf_c62x_hw_data.h" + +#define ADF_SYSTEM_DEVICE(device_id) \ + {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} + +static const struct pci_device_id adf_pci_tbl[] = { + ADF_SYSTEM_DEVICE(ADF_C62X_PCI_DEVICE_ID), + {0,} +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); +static void adf_remove(struct pci_dev *dev); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_C62X_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, + .sriov_configure = adf_sriov_configure, +}; + +static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) +{ + pci_release_regions(accel_dev->accel_pci_dev.pci_dev); + pci_disable_device(accel_dev->accel_pci_dev.pci_dev); +} + +static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; + int i; + + for (i = 0; i < ADF_PCI_MAX_BARS; i++) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; + + if (bar->virt_addr) + pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr); + } + + if (accel_dev->hw_device) { + switch (accel_pci_dev->pci_dev->device) { + case ADF_C62X_PCI_DEVICE_ID: + adf_clean_hw_data_c62x(accel_dev->hw_device); + break; + default: + break; + } + kfree(accel_dev->hw_device); + accel_dev->hw_device = NULL; + } + adf_cfg_dev_remove(accel_dev); + debugfs_remove(accel_dev->debugfs_dir); + adf_devmgr_rm_dev(accel_dev, NULL); +} + +static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct adf_accel_dev *accel_dev; + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + char name[ADF_DEVICE_NAME_LENGTH]; + unsigned int i, bar_nr; + int ret, bar_mask; + + switch (ent->device) { + case ADF_C62X_PCI_DEVICE_ID: + break; + default: + dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); + return -ENODEV; + } + + if (num_possible_nodes() > 1 && dev_to_node(&pdev->dev) < 0) { + /* If the accelerator is connected to a node with no memory + * there is no point in using the accelerator since the remote + * memory transaction will be very slow. */ + dev_err(&pdev->dev, "Invalid NUMA configuration.\n"); + return -EINVAL; + } + + accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!accel_dev) + return -ENOMEM; + + INIT_LIST_HEAD(&accel_dev->crypto_list); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; + + /* Add accel device to accel table. + * This should be called before adf_cleanup_accel is called */ + if (adf_devmgr_add_dev(accel_dev, NULL)) { + dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); + kfree(accel_dev); + return -EFAULT; + } + + accel_dev->owner = THIS_MODULE; + /* Allocate and configure device configuration structure */ + hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!hw_data) { + ret = -ENOMEM; + goto out_err; + } + + accel_dev->hw_device = hw_data; + adf_init_hw_data_c62x(accel_dev->hw_device); + pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); + pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, + &hw_data->fuses); + + /* Get Accelerators and Accelerators Engines masks */ + hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + accel_pci_dev->sku = hw_data->get_sku(hw_data); + /* If the device has no acceleration engines then ignore it. */ + if (!hw_data->accel_mask || !hw_data->ae_mask || + ((~hw_data->ae_mask) & 0x01)) { + dev_err(&pdev->dev, "No acceleration units found"); + ret = -EFAULT; + goto out_err; + } + + /* Create dev top level debugfs entry */ + snprintf(name, sizeof(name), "%s%s_%02x:%02d.%02d", + ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, + pdev->bus->number, PCI_SLOT(pdev->devfn), + PCI_FUNC(pdev->devfn)); + + accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); + if (!accel_dev->debugfs_dir) { + dev_err(&pdev->dev, "Could not create debugfs dir %s\n", name); + ret = -EINVAL; + goto out_err; + } + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + goto out_err; + + /* enable PCI device */ + if (pci_enable_device(pdev)) { + ret = -EFAULT; + goto out_err; + } + + /* set dma identifier */ + if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { + if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) { + dev_err(&pdev->dev, "No usable DMA configuration\n"); + ret = -EFAULT; + goto out_err_disable; + } else { + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + } + + } else { + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); + } + + if (pci_request_regions(pdev, ADF_C62X_DEVICE_NAME)) { + ret = -EFAULT; + goto out_err_disable; + } + + /* Read accelerator capabilities mask */ + pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, + &hw_data->accel_capabilities_mask); + + /* Find and map all the device's BARS */ + i = 0; + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + for_each_set_bit(bar_nr, (const unsigned long *)&bar_mask, + ADF_PCI_MAX_BARS * 2) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; + + bar->base_addr = pci_resource_start(pdev, bar_nr); + if (!bar->base_addr) + break; + bar->size = pci_resource_len(pdev, bar_nr); + bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); + if (!bar->virt_addr) { + dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); + ret = -EFAULT; + goto out_err_free_reg; + } + } + pci_set_master(pdev); + + if (adf_enable_aer(accel_dev, &adf_driver)) { + dev_err(&pdev->dev, "Failed to enable aer\n"); + ret = -EFAULT; + goto out_err_free_reg; + } + + if (pci_save_state(pdev)) { + dev_err(&pdev->dev, "Failed to save pci state\n"); + ret = -ENOMEM; + goto out_err_free_reg; + } + + ret = qat_crypto_dev_config(accel_dev); + if (ret) + goto out_err_free_reg; + + ret = adf_dev_init(accel_dev); + if (ret) + goto out_err_dev_shutdown; + + ret = adf_dev_start(accel_dev); + if (ret) + goto out_err_dev_stop; + + return ret; + +out_err_dev_stop: + adf_dev_stop(accel_dev); +out_err_dev_shutdown: + adf_dev_shutdown(accel_dev); +out_err_free_reg: + pci_release_regions(accel_pci_dev->pci_dev); +out_err_disable: + pci_disable_device(accel_pci_dev->pci_dev); +out_err: + adf_cleanup_accel(accel_dev); + kfree(accel_dev); + return ret; +} + +static void adf_remove(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + pr_err("QAT: Driver removal failed\n"); + return; + } + if (adf_dev_stop(accel_dev)) + dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n"); + + adf_dev_shutdown(accel_dev); + adf_disable_aer(accel_dev); + adf_cleanup_accel(accel_dev); + adf_cleanup_pci_dev(accel_dev); + kfree(accel_dev); +} + +static int __init adfdrv_init(void) +{ + request_module("intel_qat"); + + if (pci_register_driver(&adf_driver)) { + pr_err("QAT: Driver initialization failed\n"); + return -EFAULT; + } + return 0; +} + +static void __exit adfdrv_release(void) +{ + pci_unregister_driver(&adf_driver); +} + +module_init(adfdrv_init); +module_exit(adfdrv_release); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel"); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); +MODULE_VERSION(ADF_DRV_VERSION); diff --git a/drivers/crypto/qat/qat_c62xvf/Makefile b/drivers/crypto/qat/qat_c62xvf/Makefile new file mode 100644 index 000000000000..ecd708c213b2 --- /dev/null +++ b/drivers/crypto/qat/qat_c62xvf/Makefile @@ -0,0 +1,3 @@ +ccflags-y := -I$(src)/../qat_common +obj-$(CONFIG_CRYPTO_DEV_QAT_C62XVF) += qat_c62xvf.o +qat_c62xvf-objs := adf_drv.o adf_c62xvf_hw_data.o diff --git a/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c new file mode 100644 index 000000000000..baf4b509c892 --- /dev/null +++ b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c @@ -0,0 +1,173 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2015 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2015 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <adf_accel_devices.h> +#include <adf_pf2vf_msg.h> +#include <adf_common_drv.h> +#include "adf_c62xvf_hw_data.h" + +static struct adf_hw_device_class c62xiov_class = { + .name = ADF_C62XVF_DEVICE_NAME, + .type = DEV_C62XVF, + .instances = 0 +}; + +static u32 get_accel_mask(u32 fuse) +{ + return ADF_C62XIOV_ACCELERATORS_MASK; +} + +static u32 get_ae_mask(u32 fuse) +{ + return ADF_C62XIOV_ACCELENGINES_MASK; +} + +static u32 get_num_accels(struct adf_hw_device_data *self) +{ + return ADF_C62XIOV_MAX_ACCELERATORS; +} + +static u32 get_num_aes(struct adf_hw_device_data *self) +{ + return ADF_C62XIOV_MAX_ACCELENGINES; +} + +static u32 get_misc_bar_id(struct adf_hw_device_data *self) +{ + return ADF_C62XIOV_PMISC_BAR; +} + +static u32 get_etr_bar_id(struct adf_hw_device_data *self) +{ + return ADF_C62XIOV_ETR_BAR; +} + +static enum dev_sku_info get_sku(struct adf_hw_device_data *self) +{ + return DEV_SKU_VF; +} + +static u32 get_pf2vf_offset(u32 i) +{ + return ADF_C62XIOV_PF2VF_OFFSET; +} + +static u32 get_vintmsk_offset(u32 i) +{ + return ADF_C62XIOV_VINTMSK_OFFSET; +} + +static int adf_vf_int_noop(struct adf_accel_dev *accel_dev) +{ + return 0; +} + +static void adf_vf_void_noop(struct adf_accel_dev *accel_dev) +{ +} + +static int adf_vf2pf_init(struct adf_accel_dev *accel_dev) +{ + u32 msg = (ADF_VF2PF_MSGORIGIN_SYSTEM | + (ADF_VF2PF_MSGTYPE_INIT << ADF_VF2PF_MSGTYPE_SHIFT)); + + if (adf_iov_putmsg(accel_dev, msg, 0)) { + dev_err(&GET_DEV(accel_dev), + "Failed to send Init event to PF\n"); + return -EFAULT; + } + return 0; +} + +static void adf_vf2pf_shutdown(struct adf_accel_dev *accel_dev) +{ + u32 msg = (ADF_VF2PF_MSGORIGIN_SYSTEM | + (ADF_VF2PF_MSGTYPE_SHUTDOWN << ADF_VF2PF_MSGTYPE_SHIFT)); + + if (adf_iov_putmsg(accel_dev, msg, 0)) + dev_err(&GET_DEV(accel_dev), + "Failed to send Shutdown event to PF\n"); +} + +void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class = &c62xiov_class; + hw_data->num_banks = ADF_C62XIOV_ETR_MAX_BANKS; + hw_data->num_accel = ADF_C62XIOV_MAX_ACCELERATORS; + hw_data->num_logical_accel = 1; + hw_data->num_engines = ADF_C62XIOV_MAX_ACCELENGINES; + hw_data->tx_rx_gap = ADF_C62XIOV_RX_RINGS_OFFSET; + hw_data->tx_rings_mask = ADF_C62XIOV_TX_RINGS_MASK; + hw_data->alloc_irq = adf_vf_isr_resource_alloc; + hw_data->free_irq = adf_vf_isr_resource_free; + hw_data->enable_error_correction = adf_vf_void_noop; + hw_data->init_admin_comms = adf_vf_int_noop; + hw_data->exit_admin_comms = adf_vf_void_noop; + hw_data->send_admin_init = adf_vf2pf_init; + hw_data->init_arb = adf_vf_int_noop; + hw_data->exit_arb = adf_vf_void_noop; + hw_data->disable_iov = adf_vf2pf_shutdown; + hw_data->get_accel_mask = get_accel_mask; + hw_data->get_ae_mask = get_ae_mask; + hw_data->get_num_accels = get_num_accels; + hw_data->get_num_aes = get_num_aes; + hw_data->get_etr_bar_id = get_etr_bar_id; + hw_data->get_misc_bar_id = get_misc_bar_id; + hw_data->get_pf2vf_offset = get_pf2vf_offset; + hw_data->get_vintmsk_offset = get_vintmsk_offset; + hw_data->get_sku = get_sku; + hw_data->enable_ints = adf_vf_void_noop; + hw_data->enable_vf2pf_comms = adf_enable_vf2pf_comms; + hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; + hw_data->dev_class->instances++; + adf_devmgr_update_class_index(hw_data); +} + +void adf_clean_hw_data_c62xiov(struct adf_hw_device_data *hw_data) +{ + hw_data->dev_class->instances--; + adf_devmgr_update_class_index(hw_data); +} diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_drv.h b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.h index 85ff245bd1d8..a28d83e77422 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_drv.h +++ b/drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.h @@ -3,7 +3,7 @@ redistributing this file, you may do so under either license. GPL LICENSE SUMMARY - Copyright(c) 2014 Intel Corporation. + Copyright(c) 2015 Intel Corporation. This program is free software; you can redistribute it and/or modify it under the terms of version 2 of the GNU General Public License as published by the Free Software Foundation. @@ -17,7 +17,7 @@ qat-linux@intel.com BSD LICENSE - Copyright(c) 2014 Intel Corporation. + Copyright(c) 2015 Intel Corporation. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -44,15 +44,21 @@ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef ADF_DH895x_DRV_H_ -#define ADF_DH895x_DRV_H_ -#include <adf_accel_devices.h> -#include <adf_transport.h> - -void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data); -void adf_clean_hw_data_dh895xcc(struct adf_hw_device_data *hw_data); -int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev); -void adf_isr_resource_free(struct adf_accel_dev *accel_dev); -void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, - uint32_t const **arb_map_config); +#ifndef ADF_C62XVF_HW_DATA_H_ +#define ADF_C62XVF_HW_DATA_H_ + +#define ADF_C62XIOV_PMISC_BAR 1 +#define ADF_C62XIOV_ACCELERATORS_MASK 0x1 +#define ADF_C62XIOV_ACCELENGINES_MASK 0x1 +#define ADF_C62XIOV_MAX_ACCELERATORS 1 +#define ADF_C62XIOV_MAX_ACCELENGINES 1 +#define ADF_C62XIOV_RX_RINGS_OFFSET 8 +#define ADF_C62XIOV_TX_RINGS_MASK 0xFF +#define ADF_C62XIOV_ETR_BAR 0 +#define ADF_C62XIOV_ETR_MAX_BANKS 1 +#define ADF_C62XIOV_PF2VF_OFFSET 0x200 +#define ADF_C62XIOV_VINTMSK_OFFSET 0x208 + +void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data); +void adf_clean_hw_data_c62xiov(struct adf_hw_device_data *hw_data); #endif diff --git a/drivers/crypto/qat/qat_c62xvf/adf_drv.c b/drivers/crypto/qat/qat_c62xvf/adf_drv.c new file mode 100644 index 000000000000..d2e4b928f3be --- /dev/null +++ b/drivers/crypto/qat/qat_c62xvf/adf_drv.c @@ -0,0 +1,305 @@ +/* + This file is provided under a dual BSD/GPLv2 license. When using or + redistributing this file, you may do so under either license. + + GPL LICENSE SUMMARY + Copyright(c) 2014 Intel Corporation. + This program is free software; you can redistribute it and/or modify + it under the terms of version 2 of the GNU General Public License as + published by the Free Software Foundation. + + This program is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + Contact Information: + qat-linux@intel.com + + BSD LICENSE + Copyright(c) 2014 Intel Corporation. + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + * Neither the name of Intel Corporation nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/init.h> +#include <linux/types.h> +#include <linux/fs.h> +#include <linux/slab.h> +#include <linux/errno.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/platform_device.h> +#include <linux/workqueue.h> +#include <linux/io.h> +#include <adf_accel_devices.h> +#include <adf_common_drv.h> +#include <adf_cfg.h> +#include "adf_c62xvf_hw_data.h" + +#define ADF_SYSTEM_DEVICE(device_id) \ + {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} + +static const struct pci_device_id adf_pci_tbl[] = { + ADF_SYSTEM_DEVICE(ADF_C62XIOV_PCI_DEVICE_ID), + {0,} +}; +MODULE_DEVICE_TABLE(pci, adf_pci_tbl); + +static int adf_probe(struct pci_dev *dev, const struct pci_device_id *ent); +static void adf_remove(struct pci_dev *dev); + +static struct pci_driver adf_driver = { + .id_table = adf_pci_tbl, + .name = ADF_C62XVF_DEVICE_NAME, + .probe = adf_probe, + .remove = adf_remove, +}; + +static void adf_cleanup_pci_dev(struct adf_accel_dev *accel_dev) +{ + pci_release_regions(accel_dev->accel_pci_dev.pci_dev); + pci_disable_device(accel_dev->accel_pci_dev.pci_dev); +} + +static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) +{ + struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; + struct adf_accel_dev *pf; + int i; + + for (i = 0; i < ADF_PCI_MAX_BARS; i++) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; + + if (bar->virt_addr) + pci_iounmap(accel_pci_dev->pci_dev, bar->virt_addr); + } + + if (accel_dev->hw_device) { + switch (accel_pci_dev->pci_dev->device) { + case ADF_C62XIOV_PCI_DEVICE_ID: + adf_clean_hw_data_c62xiov(accel_dev->hw_device); + break; + default: + break; + } + kfree(accel_dev->hw_device); + accel_dev->hw_device = NULL; + } + adf_cfg_dev_remove(accel_dev); + debugfs_remove(accel_dev->debugfs_dir); + pf = adf_devmgr_pci_to_accel_dev(accel_pci_dev->pci_dev->physfn); + adf_devmgr_rm_dev(accel_dev, pf); +} + +static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct adf_accel_dev *accel_dev; + struct adf_accel_dev *pf; + struct adf_accel_pci *accel_pci_dev; + struct adf_hw_device_data *hw_data; + char name[ADF_DEVICE_NAME_LENGTH]; + unsigned int i, bar_nr; + int ret, bar_mask; + + switch (ent->device) { + case ADF_C62XIOV_PCI_DEVICE_ID: + break; + default: + dev_err(&pdev->dev, "Invalid device 0x%x.\n", ent->device); + return -ENODEV; + } + + accel_dev = kzalloc_node(sizeof(*accel_dev), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!accel_dev) + return -ENOMEM; + + accel_dev->is_vf = true; + pf = adf_devmgr_pci_to_accel_dev(pdev->physfn); + accel_pci_dev = &accel_dev->accel_pci_dev; + accel_pci_dev->pci_dev = pdev; + + /* Add accel device to accel table */ + if (adf_devmgr_add_dev(accel_dev, pf)) { + dev_err(&pdev->dev, "Failed to add new accelerator device.\n"); + kfree(accel_dev); + return -EFAULT; + } + INIT_LIST_HEAD(&accel_dev->crypto_list); + + accel_dev->owner = THIS_MODULE; + /* Allocate and configure device configuration structure */ + hw_data = kzalloc_node(sizeof(*hw_data), GFP_KERNEL, + dev_to_node(&pdev->dev)); + if (!hw_data) { + ret = -ENOMEM; + goto out_err; + } + accel_dev->hw_device = hw_data; + adf_init_hw_data_c62xiov(accel_dev->hw_device); + + /* Get Accelerators and Accelerators Engines masks */ + hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); + hw_data->ae_mask = hw_data->get_ae_mask(hw_data->fuses); + accel_pci_dev->sku = hw_data->get_sku(hw_data); + + /* Create dev top level debugfs entry */ + snprintf(name, sizeof(name), "%s%s_%02x:%02d.%02d", + ADF_DEVICE_NAME_PREFIX, hw_data->dev_class->name, + pdev->bus->number, PCI_SLOT(pdev->devfn), + PCI_FUNC(pdev->devfn)); + + accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); + if (!accel_dev->debugfs_dir) { + dev_err(&pdev->dev, "Could not create debugfs dir %s\n", name); + ret = -EINVAL; + goto out_err; + } + + /* Create device configuration table */ + ret = adf_cfg_dev_add(accel_dev); + if (ret) + goto out_err; + + /* enable PCI device */ + if (pci_enable_device(pdev)) { + ret = -EFAULT; + goto out_err; + } + + /* set dma identifier */ + if (pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { + if ((pci_set_dma_mask(pdev, DMA_BIT_MASK(32)))) { + dev_err(&pdev->dev, "No usable DMA configuration\n"); + ret = -EFAULT; + goto out_err_disable; + } else { + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32)); + } + + } else { + pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); + } + + if (pci_request_regions(pdev, ADF_C62XVF_DEVICE_NAME)) { + ret = -EFAULT; + goto out_err_disable; + } + + /* Find and map all the device's BARS */ + i = 0; + bar_mask = pci_select_bars(pdev, IORESOURCE_MEM); + for_each_set_bit(bar_nr, (const unsigned long *)&bar_mask, + ADF_PCI_MAX_BARS * 2) { + struct adf_bar *bar = &accel_pci_dev->pci_bars[i++]; + + bar->base_addr = pci_resource_start(pdev, bar_nr); + if (!bar->base_addr) + break; + bar->size = pci_resource_len(pdev, bar_nr); + bar->virt_addr = pci_iomap(accel_pci_dev->pci_dev, bar_nr, 0); + if (!bar->virt_addr) { + dev_err(&pdev->dev, "Failed to map BAR %d\n", bar_nr); + ret = -EFAULT; + goto out_err_free_reg; + } + } + pci_set_master(pdev); + /* Completion for VF2PF request/response message exchange */ + init_completion(&accel_dev->vf.iov_msg_completion); + + ret = qat_crypto_dev_config(accel_dev); + if (ret) + goto out_err_free_reg; + + ret = adf_dev_init(accel_dev); + if (ret) + goto out_err_dev_shutdown; + + ret = adf_dev_start(accel_dev); + if (ret) + goto out_err_dev_stop; + + return ret; + +out_err_dev_stop: + adf_dev_stop(accel_dev); +out_err_dev_shutdown: + adf_dev_shutdown(accel_dev); +out_err_free_reg: + pci_release_regions(accel_pci_dev->pci_dev); +out_err_disable: + pci_disable_device(accel_pci_dev->pci_dev); +out_err: + adf_cleanup_accel(accel_dev); + kfree(accel_dev); + return ret; +} + +static void adf_remove(struct pci_dev *pdev) +{ + struct adf_accel_dev *accel_dev = adf_devmgr_pci_to_accel_dev(pdev); + + if (!accel_dev) { + pr_err("QAT: Driver removal failed\n"); + return; + } + if (adf_dev_stop(accel_dev)) + dev_err(&GET_DEV(accel_dev), "Failed to stop QAT accel dev\n"); + + adf_dev_shutdown(accel_dev); + adf_cleanup_accel(accel_dev); + adf_cleanup_pci_dev(accel_dev); + kfree(accel_dev); +} + +static int __init adfdrv_init(void) +{ + request_module("intel_qat"); + + if (pci_register_driver(&adf_driver)) { + pr_err("QAT: Driver initialization failed\n"); + return -EFAULT; + } + return 0; +} + +static void __exit adfdrv_release(void) +{ + pci_unregister_driver(&adf_driver); + adf_clean_vf_map(true); +} + +module_init(adfdrv_init); +module_exit(adfdrv_release); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_AUTHOR("Intel"); +MODULE_DESCRIPTION("Intel(R) QuickAssist Technology"); +MODULE_VERSION(ADF_DRV_VERSION); diff --git a/drivers/crypto/qat/qat_common/Makefile b/drivers/crypto/qat/qat_common/Makefile index 9e9e196c6d51..29c7c53d2845 100644 --- a/drivers/crypto/qat/qat_common/Makefile +++ b/drivers/crypto/qat/qat_common/Makefile @@ -4,10 +4,12 @@ $(obj)/qat_rsaprivkey-asn1.o: $(obj)/qat_rsaprivkey-asn1.c \ $(obj)/qat_rsaprivkey-asn1.h clean-files += qat_rsapubkey-asn1.c qat_rsapubkey-asn1.h -clean-files += qat_rsaprivkey-asn1.c qat_rsapvivkey-asn1.h +clean-files += qat_rsaprivkey-asn1.c qat_rsaprivkey-asn1.h obj-$(CONFIG_CRYPTO_DEV_QAT) += intel_qat.o intel_qat-objs := adf_cfg.o \ + adf_isr.o \ + adf_vf_isr.o \ adf_ctl_drv.o \ adf_dev_mgr.o \ adf_init.o \ diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index ca853d50b4b7..f96d427e502c 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -55,8 +55,20 @@ #define ADF_DH895XCC_DEVICE_NAME "dh895xcc" #define ADF_DH895XCCVF_DEVICE_NAME "dh895xccvf" +#define ADF_C62X_DEVICE_NAME "c62x" +#define ADF_C62XVF_DEVICE_NAME "c62xvf" +#define ADF_C3XXX_DEVICE_NAME "c3xxx" +#define ADF_C3XXXVF_DEVICE_NAME "c3xxxvf" #define ADF_DH895XCC_PCI_DEVICE_ID 0x435 #define ADF_DH895XCCIOV_PCI_DEVICE_ID 0x443 +#define ADF_C62X_PCI_DEVICE_ID 0x37c8 +#define ADF_C62XIOV_PCI_DEVICE_ID 0x37c9 +#define ADF_C3XXX_PCI_DEVICE_ID 0x19e2 +#define ADF_C3XXXIOV_PCI_DEVICE_ID 0x19e3 +#define ADF_ERRSOU3 (0x3A000 + 0x0C) +#define ADF_ERRSOU5 (0x3A000 + 0xD8) +#define ADF_DEVICE_FUSECTL_OFFSET 0x40 +#define ADF_DEVICE_LEGFUSE_OFFSET 0x4C #define ADF_PCI_MAX_BARS 3 #define ADF_DEVICE_NAME_LENGTH 32 #define ADF_ETR_MAX_RINGS_PER_BANK 16 @@ -168,11 +180,11 @@ struct adf_hw_device_data { const char *fw_mmp_name; uint32_t fuses; uint32_t accel_capabilities_mask; + uint32_t instance_id; uint16_t accel_mask; uint16_t ae_mask; uint16_t tx_rings_mask; uint8_t tx_rx_gap; - uint8_t instance_id; uint8_t num_banks; uint8_t num_accel; uint8_t num_logical_accel; @@ -239,6 +251,6 @@ struct adf_accel_dev { } vf; }; bool is_vf; - uint8_t accel_id; + u32 accel_id; } __packed; #endif diff --git a/drivers/crypto/qat/qat_common/adf_accel_engine.c b/drivers/crypto/qat/qat_common/adf_accel_engine.c index 20b08bdcb146..a42fc42704be 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_engine.c +++ b/drivers/crypto/qat/qat_common/adf_accel_engine.c @@ -78,9 +78,12 @@ int adf_ae_fw_load(struct adf_accel_dev *accel_dev) uof_addr = (void *)loader_data->uof_fw->data; mmp_size = loader_data->mmp_fw->size; mmp_addr = (void *)loader_data->mmp_fw->data; - qat_uclo_wr_mimage(loader_data->fw_loader, mmp_addr, mmp_size); - if (qat_uclo_map_uof_obj(loader_data->fw_loader, uof_addr, uof_size)) { - dev_err(&GET_DEV(accel_dev), "Failed to map UOF\n"); + if (qat_uclo_wr_mimage(loader_data->fw_loader, mmp_addr, mmp_size)) { + dev_err(&GET_DEV(accel_dev), "Failed to load MMP\n"); + goto out_err; + } + if (qat_uclo_map_obj(loader_data->fw_loader, uof_addr, uof_size)) { + dev_err(&GET_DEV(accel_dev), "Failed to map FW\n"); goto out_err; } if (qat_uclo_wr_all_uimage(loader_data->fw_loader)) { diff --git a/drivers/crypto/qat/qat_common/adf_admin.c b/drivers/crypto/qat/qat_common/adf_admin.c index 147d755fed97..eb557f69e367 100644 --- a/drivers/crypto/qat/qat_common/adf_admin.c +++ b/drivers/crypto/qat/qat_common/adf_admin.c @@ -51,6 +51,7 @@ #include <linux/pci.h> #include <linux/dma-mapping.h> #include "adf_accel_devices.h" +#include "adf_common_drv.h" #include "icp_qat_fw_init_admin.h" /* Admin Messages Registers */ @@ -234,7 +235,8 @@ int adf_init_admin_comms(struct adf_accel_dev *accel_dev) struct adf_bar *pmisc = &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; void __iomem *csr = pmisc->virt_addr; - void __iomem *mailbox = csr + ADF_DH895XCC_MAILBOX_BASE_OFFSET; + void __iomem *mailbox = (void __iomem *)((uintptr_t)csr + + ADF_DH895XCC_MAILBOX_BASE_OFFSET); u64 reg_val; admin = kzalloc_node(sizeof(*accel_dev->admin), GFP_KERNEL, diff --git a/drivers/crypto/qat/qat_common/adf_aer.c b/drivers/crypto/qat/qat_common/adf_aer.c index 0a5ca0ba5d64..e78a1d7d88fc 100644 --- a/drivers/crypto/qat/qat_common/adf_aer.c +++ b/drivers/crypto/qat/qat_common/adf_aer.c @@ -82,7 +82,7 @@ struct adf_reset_dev_data { struct work_struct reset_work; }; -static void adf_dev_restore(struct adf_accel_dev *accel_dev) +void adf_dev_restore(struct adf_accel_dev *accel_dev) { struct pci_dev *pdev = accel_to_pci_dev(accel_dev); struct pci_dev *parent = pdev->bus->self; @@ -197,7 +197,7 @@ static void adf_resume(struct pci_dev *pdev) dev_info(&pdev->dev, "Device is up and runnig\n"); } -static struct pci_error_handlers adf_err_handler = { +static const struct pci_error_handlers adf_err_handler = { .error_detected = adf_error_detected, .slot_reset = adf_slot_reset, .resume = adf_resume, diff --git a/drivers/crypto/qat/qat_common/adf_cfg_common.h b/drivers/crypto/qat/qat_common/adf_cfg_common.h index c697fb1cdfb5..8c4f6573ce59 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg_common.h +++ b/drivers/crypto/qat/qat_common/adf_cfg_common.h @@ -72,12 +72,16 @@ enum adf_device_type { DEV_UNKNOWN = 0, DEV_DH895XCC, DEV_DH895XCCVF, + DEV_C62X, + DEV_C62XVF, + DEV_C3XXX, + DEV_C3XXXVF }; struct adf_dev_status_info { enum adf_device_type type; - uint8_t accel_id; - uint8_t instance_id; + u32 accel_id; + u32 instance_id; uint8_t num_ae; uint8_t num_accel; uint8_t num_logical_accel; diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index 3f76bd495bcb..0e82ce3c383e 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -54,7 +54,7 @@ #include "icp_qat_hal.h" #define ADF_MAJOR_VERSION 0 -#define ADF_MINOR_VERSION 2 +#define ADF_MINOR_VERSION 6 #define ADF_BUILD_VERSION 0 #define ADF_DRV_VERSION __stringify(ADF_MAJOR_VERSION) "." \ __stringify(ADF_MINOR_VERSION) "." \ @@ -106,8 +106,6 @@ int adf_dev_start(struct adf_accel_dev *accel_dev); int adf_dev_stop(struct adf_accel_dev *accel_dev); void adf_dev_shutdown(struct adf_accel_dev *accel_dev); -void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); -void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); int adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr); void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev); int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev); @@ -143,6 +141,7 @@ int adf_ae_stop(struct adf_accel_dev *accel_dev); int adf_enable_aer(struct adf_accel_dev *accel_dev, struct pci_driver *adf); void adf_disable_aer(struct adf_accel_dev *accel_dev); +void adf_dev_restore(struct adf_accel_dev *accel_dev); int adf_init_aer(void); void adf_exit_aer(void); int adf_init_admin_comms(struct adf_accel_dev *accel_dev); @@ -159,6 +158,7 @@ int adf_init_etr_data(struct adf_accel_dev *accel_dev); void adf_cleanup_etr_data(struct adf_accel_dev *accel_dev); int qat_crypto_register(void); int qat_crypto_unregister(void); +int qat_crypto_dev_config(struct adf_accel_dev *accel_dev); struct qat_crypto_instance *qat_crypto_get_instance_node(int node); void qat_crypto_put_instance(struct qat_crypto_instance *inst); void qat_alg_callback(void *resp); @@ -168,6 +168,11 @@ void qat_algs_unregister(void); int qat_asym_algs_register(void); void qat_asym_algs_unregister(void); +int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev); +void adf_isr_resource_free(struct adf_accel_dev *accel_dev); +int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev); +void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev); + int qat_hal_init(struct adf_accel_dev *accel_dev); void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle); void qat_hal_start(struct icp_qat_fw_loader_handle *handle, unsigned char ae, @@ -178,6 +183,8 @@ void qat_hal_reset(struct icp_qat_fw_loader_handle *handle); int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle); void qat_hal_set_live_ctx(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned int ctx_mask); +int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle, + unsigned int ae); int qat_hal_set_ae_lm_mode(struct icp_qat_fw_loader_handle *handle, unsigned char ae, enum icp_qat_uof_regtype lm_type, unsigned char mode); @@ -216,10 +223,10 @@ int qat_hal_wr_lm(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned short lm_addr, unsigned int value); int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle); void qat_uclo_del_uof_obj(struct icp_qat_fw_loader_handle *handle); -int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle, - void *addr_ptr, int mem_size); -void qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, - void *addr_ptr, int mem_size); +int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, void *addr_ptr, + int mem_size); +int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, + void *addr_ptr, int mem_size); #if defined(CONFIG_PCI_IOV) int adf_sriov_configure(struct pci_dev *pdev, int numvfs); void adf_disable_sriov(struct adf_accel_dev *accel_dev); @@ -227,6 +234,8 @@ void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, uint32_t vf_mask); void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, uint32_t vf_mask); +void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); +void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev); #else static inline int adf_sriov_configure(struct pci_dev *pdev, int numvfs) { @@ -236,5 +245,13 @@ static inline int adf_sriov_configure(struct pci_dev *pdev, int numvfs) static inline void adf_disable_sriov(struct adf_accel_dev *accel_dev) { } + +static inline void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev) +{ +} + +static inline void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev) +{ +} #endif #endif diff --git a/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/qat/qat_common/adf_ctl_drv.c index 473d36d91644..5c897e6e7994 100644 --- a/drivers/crypto/qat/qat_common/adf_ctl_drv.c +++ b/drivers/crypto/qat/qat_common/adf_ctl_drv.c @@ -255,12 +255,9 @@ out: static int adf_ctl_is_device_in_use(int id) { - struct list_head *itr, *head = adf_devmgr_get_head(); - - list_for_each(itr, head) { - struct adf_accel_dev *dev = - list_entry(itr, struct adf_accel_dev, list); + struct adf_accel_dev *dev; + list_for_each_entry(dev, adf_devmgr_get_head(), list) { if (id == dev->accel_id || id == ADF_CFG_ALL_DEVICES) { if (adf_devmgr_in_reset(dev) || adf_dev_in_use(dev)) { dev_info(&GET_DEV(dev), @@ -275,12 +272,10 @@ static int adf_ctl_is_device_in_use(int id) static int adf_ctl_stop_devices(uint32_t id) { - struct list_head *itr, *head = adf_devmgr_get_head(); + struct adf_accel_dev *accel_dev; int ret = 0; - list_for_each(itr, head) { - struct adf_accel_dev *accel_dev = - list_entry(itr, struct adf_accel_dev, list); + list_for_each_entry_reverse(accel_dev, adf_devmgr_get_head(), list) { if (id == accel_dev->accel_id || id == ADF_CFG_ALL_DEVICES) { if (!adf_dev_started(accel_dev)) continue; @@ -342,12 +337,10 @@ static int adf_ctl_ioctl_dev_start(struct file *fp, unsigned int cmd, if (ret) return ret; + ret = -ENODEV; accel_dev = adf_devmgr_get_dev_by_id(ctl_data->device_id); - if (!accel_dev) { - pr_err("QAT: Device %d not found\n", ctl_data->device_id); - ret = -ENODEV; + if (!accel_dev) goto out; - } if (!adf_dev_started(accel_dev)) { dev_info(&GET_DEV(accel_dev), diff --git a/drivers/crypto/qat/qat_common/adf_dev_mgr.c b/drivers/crypto/qat/qat_common/adf_dev_mgr.c index 8dfdb8f90797..b3ebb25f9ca7 100644 --- a/drivers/crypto/qat/qat_common/adf_dev_mgr.c +++ b/drivers/crypto/qat/qat_common/adf_dev_mgr.c @@ -53,6 +53,7 @@ static LIST_HEAD(accel_table); static LIST_HEAD(vfs_table); static DEFINE_MUTEX(table_lock); static uint32_t num_devices; +static u8 id_map[ADF_MAX_DEVICES]; struct vf_id_map { u32 bdf; @@ -116,8 +117,10 @@ void adf_clean_vf_map(bool vf) mutex_lock(&table_lock); list_for_each_safe(ptr, tmp, &vfs_table) { map = list_entry(ptr, struct vf_id_map, list); - if (map->bdf != -1) + if (map->bdf != -1) { + id_map[map->id] = 0; num_devices--; + } if (vf && map->bdf == -1) continue; @@ -154,6 +157,19 @@ void adf_devmgr_update_class_index(struct adf_hw_device_data *hw_data) } EXPORT_SYMBOL_GPL(adf_devmgr_update_class_index); +static unsigned int adf_find_free_id(void) +{ + unsigned int i; + + for (i = 0; i < ADF_MAX_DEVICES; i++) { + if (!id_map[i]) { + id_map[i] = 1; + return i; + } + } + return ADF_MAX_DEVICES + 1; +} + /** * adf_devmgr_add_dev() - Add accel_dev to the acceleration framework * @accel_dev: Pointer to acceleration device. @@ -194,8 +210,12 @@ int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev, } list_add_tail(&accel_dev->list, &accel_table); - accel_dev->accel_id = num_devices++; - + accel_dev->accel_id = adf_find_free_id(); + if (accel_dev->accel_id > ADF_MAX_DEVICES) { + ret = -EFAULT; + goto unlock; + } + num_devices++; map = kzalloc(sizeof(*map), GFP_KERNEL); if (!map) { ret = -ENOMEM; @@ -236,8 +256,13 @@ int adf_devmgr_add_dev(struct adf_accel_dev *accel_dev, ret = -ENOMEM; goto unlock; } - - accel_dev->accel_id = num_devices++; + accel_dev->accel_id = adf_find_free_id(); + if (accel_dev->accel_id > ADF_MAX_DEVICES) { + kfree(map); + ret = -EFAULT; + goto unlock; + } + num_devices++; list_add_tail(&accel_dev->list, &accel_table); map->bdf = adf_get_vf_num(accel_dev); map->id = accel_dev->accel_id; @@ -271,6 +296,7 @@ void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev, { mutex_lock(&table_lock); if (!accel_dev->is_vf || (accel_dev->is_vf && !pf)) { + id_map[accel_dev->accel_id] = 0; num_devices--; } else if (accel_dev->is_vf && pf) { struct vf_id_map *map, *next; diff --git a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c index 6849422e04bb..f267d9e42e0b 100644 --- a/drivers/crypto/qat/qat_common/adf_hw_arbiter.c +++ b/drivers/crypto/qat/qat_common/adf_hw_arbiter.c @@ -45,6 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "adf_accel_devices.h" +#include "adf_common_drv.h" #include "adf_transport_internal.h" #define ADF_ARB_NUM 4 @@ -124,19 +125,12 @@ int adf_init_arb(struct adf_accel_dev *accel_dev) } EXPORT_SYMBOL_GPL(adf_init_arb); -/** - * adf_update_ring_arb() - update ring arbitration rgister - * @accel_dev: Pointer to ring data. - * - * Function enables or disables rings for/from arbitration. - */ void adf_update_ring_arb(struct adf_etr_ring_data *ring) { WRITE_CSR_ARB_RINGSRVARBEN(ring->bank->csr_addr, ring->bank->bank_number, ring->bank->ring_mask & 0xFF); } -EXPORT_SYMBOL_GPL(adf_update_ring_arb); void adf_exit_arb(struct adf_accel_dev *accel_dev) { diff --git a/drivers/crypto/qat/qat_common/adf_init.c b/drivers/crypto/qat/qat_common/adf_init.c index d873eeecc363..ef5575e4a215 100644 --- a/drivers/crypto/qat/qat_common/adf_init.c +++ b/drivers/crypto/qat/qat_common/adf_init.c @@ -62,15 +62,6 @@ static void adf_service_add(struct service_hndl *service) mutex_unlock(&service_lock); } -/** - * adf_service_register() - Register acceleration service in the accel framework - * @service: Pointer to the service - * - * Function adds the acceleration service to the acceleration framework. - * To be used by QAT device specific drivers. - * - * Return: 0 on success, error code otherwise. - */ int adf_service_register(struct service_hndl *service) { service->init_status = 0; @@ -78,7 +69,6 @@ int adf_service_register(struct service_hndl *service) adf_service_add(service); return 0; } -EXPORT_SYMBOL_GPL(adf_service_register); static void adf_service_remove(struct service_hndl *service) { @@ -87,15 +77,6 @@ static void adf_service_remove(struct service_hndl *service) mutex_unlock(&service_lock); } -/** - * adf_service_unregister() - Unregister acceleration service from the framework - * @service: Pointer to the service - * - * Function remove the acceleration service from the acceleration framework. - * To be used by QAT device specific drivers. - * - * Return: 0 on success, error code otherwise. - */ int adf_service_unregister(struct service_hndl *service) { if (service->init_status || service->start_status) { @@ -105,7 +86,6 @@ int adf_service_unregister(struct service_hndl *service) adf_service_remove(service); return 0; } -EXPORT_SYMBOL_GPL(adf_service_unregister); /** * adf_dev_init() - Init data structures and services for the given accel device @@ -366,6 +346,7 @@ void adf_dev_shutdown(struct adf_accel_dev *accel_dev) hw_data->disable_iov(accel_dev); adf_cleanup_etr_data(accel_dev); + adf_dev_restore(accel_dev); } EXPORT_SYMBOL_GPL(adf_dev_shutdown); diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_isr.c b/drivers/crypto/qat/qat_common/adf_isr.c index 5570f78795c1..b81f79acc4ea 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_isr.c @@ -51,15 +51,13 @@ #include <linux/slab.h> #include <linux/errno.h> #include <linux/interrupt.h> -#include <adf_accel_devices.h> -#include <adf_common_drv.h> -#include <adf_cfg.h> -#include <adf_cfg_strings.h> -#include <adf_cfg_common.h> -#include <adf_transport_access_macros.h> -#include <adf_transport_internal.h> -#include "adf_drv.h" -#include "adf_dh895xcc_hw_data.h" +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_cfg.h" +#include "adf_cfg_strings.h" +#include "adf_cfg_common.h" +#include "adf_transport_access_macros.h" +#include "adf_transport_internal.h" static int adf_enable_msix(struct adf_accel_dev *accel_dev) { @@ -109,14 +107,16 @@ static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr) #ifdef CONFIG_PCI_IOV /* If SR-IOV is enabled (vf_info is non-NULL), check for VF->PF ints */ if (accel_dev->pf.vf_info) { - void __iomem *pmisc_bar_addr = - (&GET_BARS(accel_dev)[ADF_DH895XCC_PMISC_BAR])->virt_addr; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_bar *pmisc = + &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; + void __iomem *pmisc_bar_addr = pmisc->virt_addr; u32 vf_mask; /* Get the interrupt sources triggered by VFs */ - vf_mask = ((ADF_CSR_RD(pmisc_bar_addr, ADF_DH895XCC_ERRSOU5) & + vf_mask = ((ADF_CSR_RD(pmisc_bar_addr, ADF_ERRSOU5) & 0x0000FFFF) << 16) | - ((ADF_CSR_RD(pmisc_bar_addr, ADF_DH895XCC_ERRSOU3) & + ((ADF_CSR_RD(pmisc_bar_addr, ADF_ERRSOU3) & 0x01FFFE00) >> 9); if (vf_mask) { @@ -301,6 +301,12 @@ static void adf_cleanup_bh(struct adf_accel_dev *accel_dev) } } +/** + * adf_vf_isr_resource_free() - Free IRQ for acceleration device + * @accel_dev: Pointer to acceleration device. + * + * Function frees interrupts for acceleration device. + */ void adf_isr_resource_free(struct adf_accel_dev *accel_dev) { adf_free_irqs(accel_dev); @@ -308,7 +314,16 @@ void adf_isr_resource_free(struct adf_accel_dev *accel_dev) adf_disable_msix(&accel_dev->accel_pci_dev); adf_isr_free_msix_entry_table(accel_dev); } - +EXPORT_SYMBOL_GPL(adf_isr_resource_free); + +/** + * adf_vf_isr_resource_alloc() - Allocate IRQ for acceleration device + * @accel_dev: Pointer to acceleration device. + * + * Function allocates interrupts for acceleration device. + * + * Return: 0 on success, error code otherwise. + */ int adf_isr_resource_alloc(struct adf_accel_dev *accel_dev) { int ret; @@ -330,3 +345,4 @@ err_out: adf_isr_resource_free(accel_dev); return -EFAULT; } +EXPORT_SYMBOL_GPL(adf_isr_resource_alloc); diff --git a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c index 5fdbad809343..b3875fdf6cd7 100644 --- a/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c +++ b/drivers/crypto/qat/qat_common/adf_pf2vf_msg.c @@ -45,8 +45,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include <linux/pci.h> -#include <linux/mutex.h> #include <linux/delay.h> #include "adf_accel_devices.h" #include "adf_common_drv.h" @@ -58,12 +56,6 @@ #define ADF_DH895XCC_ERRMSK5 (ADF_DH895XCC_EP_OFFSET + 0xDC) #define ADF_DH895XCC_ERRMSK5_VF2PF_U_MASK(vf_mask) (vf_mask >> 16) -/** - * adf_enable_pf2vf_interrupts() - Enable PF to VF interrupts - * @accel_dev: Pointer to acceleration device. - * - * Function enables PF to VF interrupts - */ void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev) { struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; @@ -73,14 +65,7 @@ void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev) ADF_CSR_WR(pmisc_bar_addr, hw_data->get_vintmsk_offset(0), 0x0); } -EXPORT_SYMBOL_GPL(adf_enable_pf2vf_interrupts); -/** - * adf_disable_pf2vf_interrupts() - Disable PF to VF interrupts - * @accel_dev: Pointer to acceleration device. - * - * Function disables PF to VF interrupts - */ void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev) { struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev; @@ -90,7 +75,6 @@ void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev) ADF_CSR_WR(pmisc_bar_addr, hw_data->get_vintmsk_offset(0), 0x2); } -EXPORT_SYMBOL_GPL(adf_disable_pf2vf_interrupts); void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask) @@ -116,12 +100,6 @@ void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, } } -/** - * adf_disable_pf2vf_interrupts() - Disable VF to PF interrupts - * @accel_dev: Pointer to acceleration device. - * - * Function disables VF to PF interrupts - */ void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask) { struct adf_hw_device_data *hw_data = accel_dev->hw_device; @@ -144,7 +122,6 @@ void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask) ADF_CSR_WR(pmisc_addr, ADF_DH895XCC_ERRMSK5, reg); } } -EXPORT_SYMBOL_GPL(adf_disable_vf2pf_interrupts); static int __adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr) { diff --git a/drivers/crypto/qat/qat_common/adf_transport.c b/drivers/crypto/qat/qat_common/adf_transport.c index 3865ae8d96d9..57d2622728a5 100644 --- a/drivers/crypto/qat/qat_common/adf_transport.c +++ b/drivers/crypto/qat/qat_common/adf_transport.c @@ -122,7 +122,7 @@ int adf_send_message(struct adf_etr_ring_data *ring, uint32_t *msg) return -EAGAIN; } spin_lock_bh(&ring->lock); - memcpy(ring->base_addr + ring->tail, msg, + memcpy((void *)((uintptr_t)ring->base_addr + ring->tail), msg, ADF_MSG_SIZE_TO_BYTES(ring->msg_size)); ring->tail = adf_modulo(ring->tail + @@ -137,23 +137,22 @@ int adf_send_message(struct adf_etr_ring_data *ring, uint32_t *msg) static int adf_handle_response(struct adf_etr_ring_data *ring) { uint32_t msg_counter = 0; - uint32_t *msg = (uint32_t *)(ring->base_addr + ring->head); + uint32_t *msg = (uint32_t *)((uintptr_t)ring->base_addr + ring->head); while (*msg != ADF_RING_EMPTY_SIG) { ring->callback((uint32_t *)msg); + atomic_dec(ring->inflights); *msg = ADF_RING_EMPTY_SIG; ring->head = adf_modulo(ring->head + ADF_MSG_SIZE_TO_BYTES(ring->msg_size), ADF_RING_SIZE_MODULO(ring->ring_size)); msg_counter++; - msg = (uint32_t *)(ring->base_addr + ring->head); + msg = (uint32_t *)((uintptr_t)ring->base_addr + ring->head); } - if (msg_counter > 0) { + if (msg_counter > 0) WRITE_CSR_RING_HEAD(ring->bank->csr_addr, ring->bank->bank_number, ring->ring_number, ring->head); - atomic_sub(msg_counter, ring->inflights); - } return 0; } @@ -342,27 +341,15 @@ static void adf_ring_response_handler(struct adf_etr_bank_data *bank) } } -/** - * adf_response_handler() - Bottom half handler response handler - * @bank_addr: Address of a ring bank for with the BH was scheduled. - * - * Function is the bottom half handler for the response from acceleration - * device. There is one handler for every ring bank. Function checks all - * communication rings in the bank. - * To be used by QAT device specific drivers. - * - * Return: void - */ -void adf_response_handler(unsigned long bank_addr) +void adf_response_handler(uintptr_t bank_addr) { struct adf_etr_bank_data *bank = (void *)bank_addr; - /* Handle all the responses nad reenable IRQs */ + /* Handle all the responses and reenable IRQs */ adf_ring_response_handler(bank); WRITE_CSR_INT_FLAG_AND_COL(bank->csr_addr, bank->bank_number, bank->irq_mask); } -EXPORT_SYMBOL_GPL(adf_response_handler); static inline int adf_get_cfg_int(struct adf_accel_dev *accel_dev, const char *section, const char *format, @@ -447,6 +434,7 @@ static int adf_init_bank(struct adf_accel_dev *accel_dev, goto err; } + WRITE_CSR_INT_FLAG(csr_addr, bank_num, ADF_BANK_INT_FLAG_CLEAR_MASK); WRITE_CSR_INT_SRCSEL(csr_addr, bank_num); return 0; err: diff --git a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h b/drivers/crypto/qat/qat_common/adf_transport_access_macros.h index 6ad7e4e1edca..80e02a2a0a09 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_access_macros.h +++ b/drivers/crypto/qat/qat_common/adf_transport_access_macros.h @@ -50,12 +50,14 @@ #include "adf_accel_devices.h" #define ADF_BANK_INT_SRC_SEL_MASK_0 0x4444444CUL #define ADF_BANK_INT_SRC_SEL_MASK_X 0x44444444UL +#define ADF_BANK_INT_FLAG_CLEAR_MASK 0xFFFF #define ADF_RING_CSR_RING_CONFIG 0x000 #define ADF_RING_CSR_RING_LBASE 0x040 #define ADF_RING_CSR_RING_UBASE 0x080 #define ADF_RING_CSR_RING_HEAD 0x0C0 #define ADF_RING_CSR_RING_TAIL 0x100 #define ADF_RING_CSR_E_STAT 0x14C +#define ADF_RING_CSR_INT_FLAG 0x170 #define ADF_RING_CSR_INT_SRCSEL 0x174 #define ADF_RING_CSR_INT_SRCSEL_2 0x178 #define ADF_RING_CSR_INT_COL_EN 0x17C @@ -144,6 +146,9 @@ do { \ #define WRITE_CSR_RING_TAIL(csr_base_addr, bank, ring, value) \ ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ ADF_RING_CSR_RING_TAIL + (ring << 2), value) +#define WRITE_CSR_INT_FLAG(csr_base_addr, bank, value) \ + ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * (bank)) + \ + ADF_RING_CSR_INT_FLAG, value) #define WRITE_CSR_INT_SRCSEL(csr_base_addr, bank) \ do { \ ADF_CSR_WR(csr_base_addr, (ADF_RING_BUNDLE_SIZE * bank) + \ diff --git a/drivers/crypto/qat/qat_common/adf_transport_internal.h b/drivers/crypto/qat/qat_common/adf_transport_internal.h index a4869627fd57..bb883368ac01 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_internal.h +++ b/drivers/crypto/qat/qat_common/adf_transport_internal.h @@ -91,7 +91,7 @@ struct adf_etr_data { struct dentry *debug; }; -void adf_response_handler(unsigned long bank_addr); +void adf_response_handler(uintptr_t bank_addr); #ifdef CONFIG_DEBUG_FS #include <linux/debugfs.h> int adf_bank_debugfs_add(struct adf_etr_bank_data *bank); diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_isr.c b/drivers/crypto/qat/qat_common/adf_vf_isr.c index 87c5d8adb125..09427b3d4d55 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_isr.c +++ b/drivers/crypto/qat/qat_common/adf_vf_isr.c @@ -51,16 +51,18 @@ #include <linux/slab.h> #include <linux/errno.h> #include <linux/interrupt.h> -#include <adf_accel_devices.h> -#include <adf_common_drv.h> -#include <adf_cfg.h> -#include <adf_cfg_strings.h> -#include <adf_cfg_common.h> -#include <adf_transport_access_macros.h> -#include <adf_transport_internal.h> -#include <adf_pf2vf_msg.h> -#include "adf_drv.h" -#include "adf_dh895xccvf_hw_data.h" +#include "adf_accel_devices.h" +#include "adf_common_drv.h" +#include "adf_cfg.h" +#include "adf_cfg_strings.h" +#include "adf_cfg_common.h" +#include "adf_transport_access_macros.h" +#include "adf_transport_internal.h" +#include "adf_pf2vf_msg.h" + +#define ADF_VINTSOU_OFFSET 0x204 +#define ADF_VINTSOU_BUN BIT(0) +#define ADF_VINTSOU_PF2VF BIT(1) static int adf_enable_msi(struct adf_accel_dev *accel_dev) { @@ -91,12 +93,14 @@ static void adf_disable_msi(struct adf_accel_dev *accel_dev) static void adf_pf2vf_bh_handler(void *data) { struct adf_accel_dev *accel_dev = data; - void __iomem *pmisc_bar_addr = - (&GET_BARS(accel_dev)[ADF_DH895XCCIOV_PMISC_BAR])->virt_addr; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_bar *pmisc = + &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; + void __iomem *pmisc_bar_addr = pmisc->virt_addr; u32 msg; /* Read the message from PF */ - msg = ADF_CSR_RD(pmisc_bar_addr, ADF_DH895XCCIOV_PF2VF_OFFSET); + msg = ADF_CSR_RD(pmisc_bar_addr, hw_data->get_pf2vf_offset(0)); if (!(msg & ADF_PF2VF_MSGORIGIN_SYSTEM)) /* Ignore legacy non-system (non-kernel) PF2VF messages */ @@ -124,8 +128,8 @@ static void adf_pf2vf_bh_handler(void *data) } /* To ack, clear the PF2VFINT bit */ - msg &= ~ADF_DH895XCC_PF2VF_PF2VFINT; - ADF_CSR_WR(pmisc_bar_addr, ADF_DH895XCCIOV_PF2VF_OFFSET, msg); + msg &= ~BIT(0); + ADF_CSR_WR(pmisc_bar_addr, hw_data->get_pf2vf_offset(0), msg); /* Re-enable PF2VF interrupts */ adf_enable_pf2vf_interrupts(accel_dev); @@ -155,15 +159,17 @@ static void adf_cleanup_pf2vf_bh(struct adf_accel_dev *accel_dev) static irqreturn_t adf_isr(int irq, void *privdata) { struct adf_accel_dev *accel_dev = privdata; - void __iomem *pmisc_bar_addr = - (&GET_BARS(accel_dev)[ADF_DH895XCCIOV_PMISC_BAR])->virt_addr; + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct adf_bar *pmisc = + &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)]; + void __iomem *pmisc_bar_addr = pmisc->virt_addr; u32 v_int; /* Read VF INT source CSR to determine the source of VF interrupt */ - v_int = ADF_CSR_RD(pmisc_bar_addr, ADF_DH895XCCIOV_VINTSOU_OFFSET); + v_int = ADF_CSR_RD(pmisc_bar_addr, ADF_VINTSOU_OFFSET); /* Check for PF2VF interrupt */ - if (v_int & ADF_DH895XCC_VINTSOU_PF2VF) { + if (v_int & ADF_VINTSOU_PF2VF) { /* Disable PF to VF interrupt */ adf_disable_pf2vf_interrupts(accel_dev); @@ -173,7 +179,7 @@ static irqreturn_t adf_isr(int irq, void *privdata) } /* Check bundle interrupt */ - if (v_int & ADF_DH895XCC_VINTSOU_BUN) { + if (v_int & ADF_VINTSOU_BUN) { struct adf_etr_data *etr_data = accel_dev->transport; struct adf_etr_bank_data *bank = &etr_data->banks[0]; @@ -226,6 +232,12 @@ static void adf_cleanup_bh(struct adf_accel_dev *accel_dev) tasklet_kill(&priv_data->banks[0].resp_handler); } +/** + * adf_vf_isr_resource_free() - Free IRQ for acceleration device + * @accel_dev: Pointer to acceleration device. + * + * Function frees interrupts for acceleration device virtual function. + */ void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev) { struct pci_dev *pdev = accel_to_pci_dev(accel_dev); @@ -236,7 +248,16 @@ void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev) adf_cleanup_pf2vf_bh(accel_dev); adf_disable_msi(accel_dev); } - +EXPORT_SYMBOL_GPL(adf_vf_isr_resource_free); + +/** + * adf_vf_isr_resource_alloc() - Allocate IRQ for acceleration device + * @accel_dev: Pointer to acceleration device. + * + * Function allocates interrupts for acceleration device virtual function. + * + * Return: 0 on success, error code otherwise. + */ int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev) { if (adf_enable_msi(accel_dev)) @@ -256,3 +277,4 @@ err_out: adf_vf_isr_resource_free(accel_dev); return -EFAULT; } +EXPORT_SYMBOL_GPL(adf_vf_isr_resource_alloc); diff --git a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h index 5e1aa40c0404..2ffef3e4fd68 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h +++ b/drivers/crypto/qat/qat_common/icp_qat_fw_loader_handle.h @@ -68,11 +68,21 @@ struct icp_qat_fw_loader_hal_handle { struct icp_qat_fw_loader_handle { struct icp_qat_fw_loader_hal_handle *hal_handle; + struct pci_dev *pci_dev; void *obj_handle; + void *sobj_handle; + bool fw_auth; void __iomem *hal_sram_addr_v; void __iomem *hal_cap_g_ctl_csr_addr_v; void __iomem *hal_cap_ae_xfer_csr_addr_v; void __iomem *hal_cap_ae_local_csr_addr_v; void __iomem *hal_ep_csr_addr_v; }; + +struct icp_firml_dram_desc { + void __iomem *dram_base_addr; + void *dram_base_addr_v; + dma_addr_t dram_bus_addr; + u64 dram_size; +}; #endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_hal.h b/drivers/crypto/qat/qat_common/icp_qat_hal.h index 85b6d241ea82..7187917533d0 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hal.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hal.h @@ -81,6 +81,31 @@ enum hal_ae_csr { LOCAL_CSR_STATUS = 0x180, }; +enum fcu_csr { + FCU_CONTROL = 0x8c0, + FCU_STATUS = 0x8c4, + FCU_STATUS1 = 0x8c8, + FCU_DRAM_ADDR_LO = 0x8cc, + FCU_DRAM_ADDR_HI = 0x8d0, + FCU_RAMBASE_ADDR_HI = 0x8d4, + FCU_RAMBASE_ADDR_LO = 0x8d8 +}; + +enum fcu_cmd { + FCU_CTRL_CMD_NOOP = 0, + FCU_CTRL_CMD_AUTH = 1, + FCU_CTRL_CMD_LOAD = 2, + FCU_CTRL_CMD_START = 3 +}; + +enum fcu_sts { + FCU_STS_NO_STS = 0, + FCU_STS_VERI_DONE = 1, + FCU_STS_LOAD_DONE = 2, + FCU_STS_VERI_FAIL = 3, + FCU_STS_LOAD_FAIL = 4, + FCU_STS_BUSY = 5 +}; #define UA_ECS (0x1 << 31) #define ACS_ABO_BITPOS 31 #define ACS_ACNO 0x7 @@ -98,6 +123,13 @@ enum hal_ae_csr { #define LCS_STATUS (0x1) #define MMC_SHARE_CS_BITPOS 2 #define GLOBAL_CSR 0xA00 +#define FCU_CTRL_AE_POS 0x8 +#define FCU_AUTH_STS_MASK 0x7 +#define FCU_STS_DONE_POS 0x9 +#define FCU_STS_AUTHFWLD_POS 0X8 +#define FCU_LOADED_AE_POS 0x16 +#define FW_AUTH_WAIT_PERIOD 10 +#define FW_AUTH_MAX_RETRY 300 #define SET_CAP_CSR(handle, csr, val) \ ADF_CSR_WR(handle->hal_cap_g_ctl_csr_addr_v, csr, val) @@ -106,14 +138,14 @@ enum hal_ae_csr { #define SET_GLB_CSR(handle, csr, val) SET_CAP_CSR(handle, csr + GLOBAL_CSR, val) #define GET_GLB_CSR(handle, csr) GET_CAP_CSR(handle, GLOBAL_CSR + csr) #define AE_CSR(handle, ae) \ - (handle->hal_cap_ae_local_csr_addr_v + \ + ((char __iomem *)handle->hal_cap_ae_local_csr_addr_v + \ ((ae & handle->hal_handle->ae_mask) << 12)) #define AE_CSR_ADDR(handle, ae, csr) (AE_CSR(handle, ae) + (0x3ff & csr)) #define SET_AE_CSR(handle, ae, csr, val) \ ADF_CSR_WR(AE_CSR_ADDR(handle, ae, csr), 0, val) #define GET_AE_CSR(handle, ae, csr) ADF_CSR_RD(AE_CSR_ADDR(handle, ae, csr), 0) #define AE_XFER(handle, ae) \ - (handle->hal_cap_ae_xfer_csr_addr_v + \ + ((char __iomem *)handle->hal_cap_ae_xfer_csr_addr_v + \ ((ae & handle->hal_handle->ae_mask) << 12)) #define AE_XFER_ADDR(handle, ae, reg) (AE_XFER(handle, ae) + \ ((reg & 0xff) << 2)) @@ -121,5 +153,4 @@ enum hal_ae_csr { ADF_CSR_WR(AE_XFER_ADDR(handle, ae, reg), 0, val) #define SRAM_WRITE(handle, addr, val) \ ADF_CSR_WR(handle->hal_sram_addr_v, addr, val) -#define SRAM_READ(handle, addr) ADF_CSR_RD(handle->hal_sram_addr_v, addr) #endif diff --git a/drivers/crypto/qat/qat_common/icp_qat_uclo.h b/drivers/crypto/qat/qat_common/icp_qat_uclo.h index 2132a8cbc4ec..d97db990955d 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_uclo.h +++ b/drivers/crypto/qat/qat_common/icp_qat_uclo.h @@ -47,32 +47,55 @@ #ifndef __ICP_QAT_UCLO_H__ #define __ICP_QAT_UCLO_H__ -#define ICP_QAT_AC_C_CPU_TYPE 0x00400000 +#define ICP_QAT_AC_895XCC_DEV_TYPE 0x00400000 +#define ICP_QAT_AC_C62X_DEV_TYPE 0x01000000 +#define ICP_QAT_AC_C3XXX_DEV_TYPE 0x02000000 #define ICP_QAT_UCLO_MAX_AE 12 #define ICP_QAT_UCLO_MAX_CTX 8 #define ICP_QAT_UCLO_MAX_UIMAGE (ICP_QAT_UCLO_MAX_AE * ICP_QAT_UCLO_MAX_CTX) #define ICP_QAT_UCLO_MAX_USTORE 0x4000 #define ICP_QAT_UCLO_MAX_XFER_REG 128 #define ICP_QAT_UCLO_MAX_GPR_REG 128 -#define ICP_QAT_UCLO_MAX_NN_REG 128 #define ICP_QAT_UCLO_MAX_LMEM_REG 1024 #define ICP_QAT_UCLO_AE_ALL_CTX 0xff #define ICP_QAT_UOF_OBJID_LEN 8 #define ICP_QAT_UOF_FID 0xc6c2 #define ICP_QAT_UOF_MAJVER 0x4 #define ICP_QAT_UOF_MINVER 0x11 -#define ICP_QAT_UOF_NN_MODE_NOTCARE 0xff #define ICP_QAT_UOF_OBJS "UOF_OBJS" #define ICP_QAT_UOF_STRT "UOF_STRT" -#define ICP_QAT_UOF_GTID "UOF_GTID" #define ICP_QAT_UOF_IMAG "UOF_IMAG" #define ICP_QAT_UOF_IMEM "UOF_IMEM" -#define ICP_QAT_UOF_MSEG "UOF_MSEG" #define ICP_QAT_UOF_LOCAL_SCOPE 1 #define ICP_QAT_UOF_INIT_EXPR 0 #define ICP_QAT_UOF_INIT_REG 1 #define ICP_QAT_UOF_INIT_REG_CTX 2 #define ICP_QAT_UOF_INIT_EXPR_ENDIAN_SWAP 3 +#define ICP_QAT_SUOF_OBJ_ID_LEN 8 +#define ICP_QAT_SUOF_FID 0x53554f46 +#define ICP_QAT_SUOF_MAJVER 0x0 +#define ICP_QAT_SUOF_MINVER 0x1 +#define ICP_QAT_SIMG_AE_INIT_SEQ_LEN (50 * sizeof(unsigned long long)) +#define ICP_QAT_SIMG_AE_INSTS_LEN (0x4000 * sizeof(unsigned long long)) +#define ICP_QAT_CSS_FWSK_MODULUS_LEN 256 +#define ICP_QAT_CSS_FWSK_EXPONENT_LEN 4 +#define ICP_QAT_CSS_FWSK_PAD_LEN 252 +#define ICP_QAT_CSS_FWSK_PUB_LEN (ICP_QAT_CSS_FWSK_MODULUS_LEN + \ + ICP_QAT_CSS_FWSK_EXPONENT_LEN + \ + ICP_QAT_CSS_FWSK_PAD_LEN) +#define ICP_QAT_CSS_SIGNATURE_LEN 256 +#define ICP_QAT_CSS_AE_IMG_LEN (sizeof(struct icp_qat_simg_ae_mode) + \ + ICP_QAT_SIMG_AE_INIT_SEQ_LEN + \ + ICP_QAT_SIMG_AE_INSTS_LEN) +#define ICP_QAT_CSS_AE_SIMG_LEN (sizeof(struct icp_qat_css_hdr) + \ + ICP_QAT_CSS_FWSK_PUB_LEN + \ + ICP_QAT_CSS_SIGNATURE_LEN + \ + ICP_QAT_CSS_AE_IMG_LEN) +#define ICP_QAT_AE_IMG_OFFSET (sizeof(struct icp_qat_css_hdr) + \ + ICP_QAT_CSS_FWSK_MODULUS_LEN + \ + ICP_QAT_CSS_FWSK_EXPONENT_LEN + \ + ICP_QAT_CSS_SIGNATURE_LEN) +#define ICP_QAT_CSS_MAX_IMAGE_LEN 0x40000 #define ICP_QAT_CTX_MODE(ae_mode) ((ae_mode) & 0xf) #define ICP_QAT_NN_MODE(ae_mode) (((ae_mode) >> 0x4) & 0xf) @@ -112,6 +135,11 @@ enum icp_qat_uof_regtype { ICP_NEIGH_REL, }; +enum icp_qat_css_fwtype { + CSS_AE_FIRMWARE = 0, + CSS_MMP_FIRMWARE = 1 +}; + struct icp_qat_uclo_page { struct icp_qat_uclo_encap_page *encap_page; struct icp_qat_uclo_region *region; @@ -235,7 +263,7 @@ struct icp_qat_uof_filechunkhdr { }; struct icp_qat_uof_objhdr { - unsigned int cpu_type; + unsigned int ac_dev_type; unsigned short min_cpu_ver; unsigned short max_cpu_ver; short max_chunks; @@ -326,7 +354,7 @@ struct icp_qat_uof_image { unsigned int img_name; unsigned int ae_assigned; unsigned int ctx_assigned; - unsigned int cpu_type; + unsigned int ac_dev_type; unsigned int entry_address; unsigned int fill_pattern[2]; unsigned int reloadable_size; @@ -374,4 +402,127 @@ struct icp_qat_uof_batch_init { unsigned int size; struct icp_qat_uof_batch_init *next; }; + +struct icp_qat_suof_img_hdr { + char *simg_buf; + unsigned long simg_len; + char *css_header; + char *css_key; + char *css_signature; + char *css_simg; + unsigned long simg_size; + unsigned int ae_num; + unsigned int ae_mask; + unsigned int fw_type; + unsigned long simg_name; + unsigned long appmeta_data; +}; + +struct icp_qat_suof_img_tbl { + unsigned int num_simgs; + struct icp_qat_suof_img_hdr *simg_hdr; +}; + +struct icp_qat_suof_handle { + unsigned int file_id; + unsigned int check_sum; + char min_ver; + char maj_ver; + char fw_type; + char *suof_buf; + unsigned int suof_size; + char *sym_str; + unsigned int sym_size; + struct icp_qat_suof_img_tbl img_table; +}; + +struct icp_qat_fw_auth_desc { + unsigned int img_len; + unsigned int reserved; + unsigned int css_hdr_high; + unsigned int css_hdr_low; + unsigned int img_high; + unsigned int img_low; + unsigned int signature_high; + unsigned int signature_low; + unsigned int fwsk_pub_high; + unsigned int fwsk_pub_low; + unsigned int img_ae_mode_data_high; + unsigned int img_ae_mode_data_low; + unsigned int img_ae_init_data_high; + unsigned int img_ae_init_data_low; + unsigned int img_ae_insts_high; + unsigned int img_ae_insts_low; +}; + +struct icp_qat_auth_chunk { + struct icp_qat_fw_auth_desc fw_auth_desc; + u64 chunk_size; + u64 chunk_bus_addr; +}; + +struct icp_qat_css_hdr { + unsigned int module_type; + unsigned int header_len; + unsigned int header_ver; + unsigned int module_id; + unsigned int module_vendor; + unsigned int date; + unsigned int size; + unsigned int key_size; + unsigned int module_size; + unsigned int exponent_size; + unsigned int fw_type; + unsigned int reserved[21]; +}; + +struct icp_qat_simg_ae_mode { + unsigned int file_id; + unsigned short maj_ver; + unsigned short min_ver; + unsigned int dev_type; + unsigned short devmax_ver; + unsigned short devmin_ver; + unsigned int ae_mask; + unsigned int ctx_enables; + char fw_type; + char ctx_mode; + char nn_mode; + char lm0_mode; + char lm1_mode; + char scs_mode; + char lm2_mode; + char lm3_mode; + char tindex_mode; + unsigned char reserved[7]; + char simg_name[256]; + char appmeta_data[256]; +}; + +struct icp_qat_suof_filehdr { + unsigned int file_id; + unsigned int check_sum; + char min_ver; + char maj_ver; + char fw_type; + char reserved; + unsigned short max_chunks; + unsigned short num_chunks; +}; + +struct icp_qat_suof_chunk_hdr { + char chunk_id[ICP_QAT_SUOF_OBJ_ID_LEN]; + u64 offset; + u64 size; +}; + +struct icp_qat_suof_strtable { + unsigned int tab_length; + unsigned int strings; +}; + +struct icp_qat_suof_objhdr { + unsigned int img_length; + unsigned int reserved; +}; #endif diff --git a/drivers/crypto/qat/qat_common/qat_crypto.c b/drivers/crypto/qat/qat_common/qat_crypto.c index 9cab15497f04..3852d31ce0a4 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.c +++ b/drivers/crypto/qat/qat_common/qat_crypto.c @@ -49,6 +49,7 @@ #include "adf_accel_devices.h" #include "adf_common_drv.h" #include "adf_transport.h" +#include "adf_transport_access_macros.h" #include "adf_cfg.h" #include "adf_cfg_strings.h" #include "qat_crypto.h" @@ -66,13 +67,10 @@ void qat_crypto_put_instance(struct qat_crypto_instance *inst) static int qat_crypto_free_instances(struct adf_accel_dev *accel_dev) { - struct qat_crypto_instance *inst; - struct list_head *list_ptr, *tmp; + struct qat_crypto_instance *inst, *tmp; int i; - list_for_each_safe(list_ptr, tmp, &accel_dev->crypto_list) { - inst = list_entry(list_ptr, struct qat_crypto_instance, list); - + list_for_each_entry_safe(inst, tmp, &accel_dev->crypto_list, list) { for (i = 0; i < atomic_read(&inst->refctr); i++) qat_crypto_put_instance(inst); @@ -88,7 +86,7 @@ static int qat_crypto_free_instances(struct adf_accel_dev *accel_dev) if (inst->pke_rx) adf_remove_ring(inst->pke_rx); - list_del(list_ptr); + list_del(&inst->list); kfree(inst); } return 0; @@ -96,17 +94,13 @@ static int qat_crypto_free_instances(struct adf_accel_dev *accel_dev) struct qat_crypto_instance *qat_crypto_get_instance_node(int node) { - struct adf_accel_dev *accel_dev = NULL; - struct qat_crypto_instance *inst = NULL; - struct list_head *itr; + struct adf_accel_dev *accel_dev = NULL, *tmp_dev; + struct qat_crypto_instance *inst = NULL, *tmp_inst; unsigned long best = ~0; - list_for_each(itr, adf_devmgr_get_head()) { - struct adf_accel_dev *tmp_dev; + list_for_each_entry(tmp_dev, adf_devmgr_get_head(), list) { unsigned long ctr; - tmp_dev = list_entry(itr, struct adf_accel_dev, list); - if ((node == dev_to_node(&GET_DEV(tmp_dev)) || dev_to_node(&GET_DEV(tmp_dev)) < 0) && adf_dev_started(tmp_dev) && @@ -118,19 +112,16 @@ struct qat_crypto_instance *qat_crypto_get_instance_node(int node) } } } - if (!accel_dev) - pr_info("QAT: Could not find a device on node %d\n", node); - - /* Get any started device */ - list_for_each(itr, adf_devmgr_get_head()) { - struct adf_accel_dev *tmp_dev; - tmp_dev = list_entry(itr, struct adf_accel_dev, list); - - if (adf_dev_started(tmp_dev) && - !list_empty(&tmp_dev->crypto_list)) { - accel_dev = tmp_dev; - break; + if (!accel_dev) { + pr_info("QAT: Could not find a device on node %d\n", node); + /* Get any started device */ + list_for_each_entry(tmp_dev, adf_devmgr_get_head(), list) { + if (adf_dev_started(tmp_dev) && + !list_empty(&tmp_dev->crypto_list)) { + accel_dev = tmp_dev; + break; + } } } @@ -138,11 +129,9 @@ struct qat_crypto_instance *qat_crypto_get_instance_node(int node) return NULL; best = ~0; - list_for_each(itr, &accel_dev->crypto_list) { - struct qat_crypto_instance *tmp_inst; + list_for_each_entry(tmp_inst, &accel_dev->crypto_list, list) { unsigned long ctr; - tmp_inst = list_entry(itr, struct qat_crypto_instance, list); ctr = atomic_read(&tmp_inst->refctr); if (best > ctr) { inst = tmp_inst; @@ -159,6 +148,97 @@ struct qat_crypto_instance *qat_crypto_get_instance_node(int node) return inst; } +/** + * qat_crypto_dev_config() - create dev config required to create crypto inst. + * + * @accel_dev: Pointer to acceleration device. + * + * Function creates device configuration required to create crypto instances + * + * Return: 0 on success, error code otherwise. + */ +int qat_crypto_dev_config(struct adf_accel_dev *accel_dev) +{ + int cpus = num_online_cpus(); + int banks = GET_MAX_BANKS(accel_dev); + int instances = min(cpus, banks); + char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; + int i; + unsigned long val; + + if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC)) + goto err; + if (adf_cfg_section_add(accel_dev, "Accelerator0")) + goto err; + for (i = 0; i < instances; i++) { + val = i; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, + i); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); + val = 128; + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = 512; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = 0; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = 2; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = 8; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = 10; + snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + key, (void *)&val, ADF_DEC)) + goto err; + + val = ADF_COALESCING_DEF_TIME; + snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); + if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0", + key, (void *)&val, ADF_DEC)) + goto err; + } + + val = i; + if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, + ADF_NUM_CY, (void *)&val, ADF_DEC)) + goto err; + + set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); + return 0; +err: + dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n"); + return -EINVAL; +} +EXPORT_SYMBOL_GPL(qat_crypto_dev_config); + static int qat_crypto_create_instances(struct adf_accel_dev *accel_dev) { int i; diff --git a/drivers/crypto/qat/qat_common/qat_hal.c b/drivers/crypto/qat/qat_common/qat_hal.c index 380e761801a7..0ac0ba867611 100644 --- a/drivers/crypto/qat/qat_common/qat_hal.c +++ b/drivers/crypto/qat/qat_common/qat_hal.c @@ -45,21 +45,22 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include <linux/slab.h> +#include <linux/delay.h> #include "adf_accel_devices.h" #include "adf_common_drv.h" #include "icp_qat_hal.h" #include "icp_qat_uclo.h" -#define BAD_REGADDR 0xffff -#define MAX_RETRY_TIMES 10000 -#define INIT_CTX_ARB_VALUE 0x0 +#define BAD_REGADDR 0xffff +#define MAX_RETRY_TIMES 10000 +#define INIT_CTX_ARB_VALUE 0x0 #define INIT_CTX_ENABLE_VALUE 0x0 -#define INIT_PC_VALUE 0x0 +#define INIT_PC_VALUE 0x0 #define INIT_WAKEUP_EVENTS_VALUE 0x1 #define INIT_SIG_EVENTS_VALUE 0x1 #define INIT_CCENABLE_VALUE 0x2000 -#define RST_CSR_QAT_LSB 20 +#define RST_CSR_QAT_LSB 20 #define RST_CSR_AE_LSB 0 #define MC_TIMESTAMP_ENABLE (0x1 << 7) @@ -185,7 +186,7 @@ static int qat_hal_wait_cycles(struct icp_qat_fw_loader_handle *handle, if (elapsed_cycles >= 8 && !(csr & (1 << ACS_ABO_BITPOS))) return 0; } - if (!times) { + if (times < 0) { pr_err("QAT: wait_num_cycles time out\n"); return -EFAULT; } @@ -391,9 +392,6 @@ static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle) unsigned int times = MAX_RETRY_TIMES; for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { - if (!(handle->hal_handle->ae_mask & (1 << ae))) - continue; - qat_hal_rd_ae_csr(handle, ae, PROFILE_COUNT, (unsigned int *)&base_cnt); base_cnt &= 0xffff; @@ -413,6 +411,20 @@ static int qat_hal_check_ae_alive(struct icp_qat_fw_loader_handle *handle) return 0; } +int qat_hal_check_ae_active(struct icp_qat_fw_loader_handle *handle, + unsigned int ae) +{ + unsigned int enable = 0, active = 0; + + qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES, &enable); + qat_hal_rd_ae_csr(handle, ae, ACTIVE_CTX_STATUS, &active); + if ((enable & (0xff << CE_ENABLE_BITPOS)) || + (active & (1 << ACS_ABO_BITPOS))) + return 1; + else + return 0; +} + static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) { unsigned int misc_ctl; @@ -425,8 +437,6 @@ static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) (~MC_TIMESTAMP_ENABLE)); for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { - if (!(handle->hal_handle->ae_mask & (1 << ae))) - continue; qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_LOW, 0); qat_hal_wr_ae_csr(handle, ae, TIMESTAMP_HIGH, 0); } @@ -440,8 +450,9 @@ static void qat_hal_reset_timestamp(struct icp_qat_fw_loader_handle *handle) #define ESRAM_AUTO_INIT_CSR_OFFSET 0xC1C static int qat_hal_init_esram(struct icp_qat_fw_loader_handle *handle) { - void __iomem *csr_addr = handle->hal_ep_csr_addr_v + - ESRAM_AUTO_INIT_CSR_OFFSET; + void __iomem *csr_addr = + (void __iomem *)((uintptr_t)handle->hal_ep_csr_addr_v + + ESRAM_AUTO_INIT_CSR_OFFSET); unsigned int csr_val, times = 30; csr_val = ADF_CSR_RD(csr_addr, 0); @@ -493,8 +504,6 @@ int qat_hal_clr_reset(struct icp_qat_fw_loader_handle *handle) /* Set undefined power-up/reset states to reasonable default values */ for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { - if (!(handle->hal_handle->ae_mask & (1 << ae))) - continue; qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, INIT_CTX_ENABLE_VALUE); qat_hal_wr_indr_csr(handle, ae, ICP_QAT_UCLO_AE_ALL_CTX, @@ -598,25 +607,31 @@ static void qat_hal_enable_ctx(struct icp_qat_fw_loader_handle *handle, qat_hal_wr_ae_csr(handle, ae, CTX_ENABLES, ctx); } -static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) +static void qat_hal_clear_xfer(struct icp_qat_fw_loader_handle *handle) { unsigned char ae; - unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX; - int times = MAX_RETRY_TIMES; - unsigned int csr_val = 0; unsigned short reg; - unsigned int savctx = 0; - int ret = 0; for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { - if (!(handle->hal_handle->ae_mask & (1 << ae))) - continue; for (reg = 0; reg < ICP_QAT_UCLO_MAX_GPR_REG; reg++) { qat_hal_init_rd_xfer(handle, ae, 0, ICP_SR_RD_ABS, reg, 0); qat_hal_init_rd_xfer(handle, ae, 0, ICP_DR_RD_ABS, reg, 0); } + } +} + +static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) +{ + unsigned char ae; + unsigned int ctx_mask = ICP_QAT_UCLO_AE_ALL_CTX; + int times = MAX_RETRY_TIMES; + unsigned int csr_val = 0; + unsigned int savctx = 0; + int ret = 0; + + for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { qat_hal_rd_ae_csr(handle, ae, AE_MISC_CONTROL, &csr_val); csr_val &= ~(1 << MMC_SHARE_CS_BITPOS); qat_hal_wr_ae_csr(handle, ae, AE_MISC_CONTROL, csr_val); @@ -638,8 +653,6 @@ static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) qat_hal_enable_ctx(handle, ae, ctx_mask); } for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { - if (!(handle->hal_handle->ae_mask & (1 << ae))) - continue; /* wait for AE to finish */ do { ret = qat_hal_wait_cycles(handle, ae, 20, 1); @@ -667,10 +680,10 @@ static int qat_hal_clear_gpr(struct icp_qat_fw_loader_handle *handle) return 0; } -#define ICP_DH895XCC_AE_OFFSET 0x20000 -#define ICP_DH895XCC_CAP_OFFSET (ICP_DH895XCC_AE_OFFSET + 0x10000) +#define ICP_QAT_AE_OFFSET 0x20000 +#define ICP_QAT_CAP_OFFSET (ICP_QAT_AE_OFFSET + 0x10000) #define LOCAL_TO_XFER_REG_OFFSET 0x800 -#define ICP_DH895XCC_EP_OFFSET 0x3a000 +#define ICP_QAT_EP_OFFSET 0x3a000 int qat_hal_init(struct adf_accel_dev *accel_dev) { unsigned char ae; @@ -687,15 +700,22 @@ int qat_hal_init(struct adf_accel_dev *accel_dev) if (!handle) return -ENOMEM; - handle->hal_cap_g_ctl_csr_addr_v = misc_bar->virt_addr + - ICP_DH895XCC_CAP_OFFSET; - handle->hal_cap_ae_xfer_csr_addr_v = misc_bar->virt_addr + - ICP_DH895XCC_AE_OFFSET; - handle->hal_ep_csr_addr_v = misc_bar->virt_addr + - ICP_DH895XCC_EP_OFFSET; - handle->hal_cap_ae_local_csr_addr_v = - handle->hal_cap_ae_xfer_csr_addr_v + LOCAL_TO_XFER_REG_OFFSET; handle->hal_sram_addr_v = sram_bar->virt_addr; + handle->hal_cap_g_ctl_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_CAP_OFFSET); + handle->hal_cap_ae_xfer_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_AE_OFFSET); + handle->hal_ep_csr_addr_v = + (void __iomem *)((uintptr_t)misc_bar->virt_addr + + ICP_QAT_EP_OFFSET); + handle->hal_cap_ae_local_csr_addr_v = + (void __iomem *)((uintptr_t)handle->hal_cap_ae_xfer_csr_addr_v + + LOCAL_TO_XFER_REG_OFFSET); + handle->pci_dev = pci_info->pci_dev; + handle->fw_auth = (handle->pci_dev->device == + ADF_DH895XCC_PCI_DEVICE_ID) ? false : true; handle->hal_handle = kzalloc(sizeof(*handle->hal_handle), GFP_KERNEL); if (!handle->hal_handle) goto out_hal_handle; @@ -723,14 +743,16 @@ int qat_hal_init(struct adf_accel_dev *accel_dev) dev_err(&GET_DEV(accel_dev), "qat_hal_clr_reset error\n"); goto out_err; } - if (qat_hal_clear_gpr(handle)) - goto out_err; + qat_hal_clear_xfer(handle); + if (!handle->fw_auth) { + if (qat_hal_clear_gpr(handle)) + goto out_err; + } + /* Set SIGNATURE_ENABLE[0] to 0x1 in order to enable ALU_OUT csr */ for (ae = 0; ae < handle->hal_handle->ae_max_num; ae++) { unsigned int csr_val = 0; - if (!(hw_data->ae_mask & (1 << ae))) - continue; qat_hal_rd_ae_csr(handle, ae, SIGNATURE_ENABLE, &csr_val); csr_val |= 0x1; qat_hal_wr_ae_csr(handle, ae, SIGNATURE_ENABLE, csr_val); @@ -756,15 +778,31 @@ void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle) void qat_hal_start(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned int ctx_mask) { - qat_hal_put_wakeup_event(handle, ae, (~ctx_mask) & + int retry = 0; + unsigned int fcu_sts = 0; + + if (handle->fw_auth) { + SET_CAP_CSR(handle, FCU_CONTROL, FCU_CTRL_CMD_START); + do { + msleep(FW_AUTH_WAIT_PERIOD); + fcu_sts = GET_CAP_CSR(handle, FCU_STATUS); + if (((fcu_sts >> FCU_STS_DONE_POS) & 0x1)) + return; + } while (retry++ < FW_AUTH_MAX_RETRY); + pr_err("QAT: start error (AE 0x%x FCU_STS = 0x%x)\n", ae, + fcu_sts); + } else { + qat_hal_put_wakeup_event(handle, ae, (~ctx_mask) & ICP_QAT_UCLO_AE_ALL_CTX, 0x10000); - qat_hal_enable_ctx(handle, ae, ctx_mask); + qat_hal_enable_ctx(handle, ae, ctx_mask); + } } void qat_hal_stop(struct icp_qat_fw_loader_handle *handle, unsigned char ae, unsigned int ctx_mask) { - qat_hal_disable_ctx(handle, ae, ctx_mask); + if (!handle->fw_auth) + qat_hal_disable_ctx(handle, ae, ctx_mask); } void qat_hal_set_pc(struct icp_qat_fw_loader_handle *handle, diff --git a/drivers/crypto/qat/qat_common/qat_uclo.c b/drivers/crypto/qat/qat_common/qat_uclo.c index c48f181e8941..25d15f19c2b3 100644 --- a/drivers/crypto/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/qat/qat_common/qat_uclo.c @@ -47,7 +47,7 @@ #include <linux/slab.h> #include <linux/ctype.h> #include <linux/kernel.h> - +#include <linux/delay.h> #include "adf_accel_devices.h" #include "adf_common_drv.h" #include "icp_qat_uclo.h" @@ -119,10 +119,10 @@ static char *qat_uclo_get_string(struct icp_qat_uof_strtable *str_table, { if ((!str_table->table_len) || (str_offset > str_table->table_len)) return NULL; - return (char *)(((unsigned long)(str_table->strings)) + str_offset); + return (char *)(((uintptr_t)(str_table->strings)) + str_offset); } -static int qat_uclo_check_format(struct icp_qat_uof_filehdr *hdr) +static int qat_uclo_check_uof_format(struct icp_qat_uof_filehdr *hdr) { int maj = hdr->maj_ver & 0xff; int min = hdr->min_ver & 0xff; @@ -139,6 +139,31 @@ static int qat_uclo_check_format(struct icp_qat_uof_filehdr *hdr) return 0; } +static int qat_uclo_check_suof_format(struct icp_qat_suof_filehdr *suof_hdr) +{ + int maj = suof_hdr->maj_ver & 0xff; + int min = suof_hdr->min_ver & 0xff; + + if (suof_hdr->file_id != ICP_QAT_SUOF_FID) { + pr_err("QAT: invalid header 0x%x\n", suof_hdr->file_id); + return -EINVAL; + } + if (suof_hdr->fw_type != 0) { + pr_err("QAT: unsupported firmware type\n"); + return -EINVAL; + } + if (suof_hdr->num_chunks <= 0x1) { + pr_err("QAT: SUOF chunk amount is incorrect\n"); + return -EINVAL; + } + if (maj != ICP_QAT_SUOF_MAJVER || min != ICP_QAT_SUOF_MINVER) { + pr_err("QAT: bad SUOF version, major 0x%x, minor 0x%x\n", + maj, min); + return -EINVAL; + } + return 0; +} + static void qat_uclo_wr_sram_by_words(struct icp_qat_fw_loader_handle *handle, unsigned int addr, unsigned int *val, unsigned int num_in_bytes) @@ -275,7 +300,7 @@ static int qat_uclo_create_batch_init_list(struct icp_qat_fw_loader_handle unsigned int i, flag = 0; mem_val_attr = - (struct icp_qat_uof_memvar_attr *)((unsigned long)init_mem + + (struct icp_qat_uof_memvar_attr *)((uintptr_t)init_mem + sizeof(struct icp_qat_uof_initmem)); init_header = *init_tab_base; @@ -425,8 +450,8 @@ static int qat_uclo_init_memory(struct icp_qat_fw_loader_handle *handle) if (qat_uclo_init_ae_memory(handle, initmem)) return -EINVAL; } - initmem = (struct icp_qat_uof_initmem *)((unsigned long)( - (unsigned long)initmem + + initmem = (struct icp_qat_uof_initmem *)((uintptr_t)( + (uintptr_t)initmem + sizeof(struct icp_qat_uof_initmem)) + (sizeof(struct icp_qat_uof_memvar_attr) * initmem->val_attr_num)); @@ -454,7 +479,7 @@ static void *qat_uclo_find_chunk(struct icp_qat_uof_objhdr *obj_hdr, int i; struct icp_qat_uof_chunkhdr *chunk_hdr = (struct icp_qat_uof_chunkhdr *) - ((unsigned long)obj_hdr + sizeof(struct icp_qat_uof_objhdr)); + ((uintptr_t)obj_hdr + sizeof(struct icp_qat_uof_objhdr)); for (i = 0; i < obj_hdr->num_chunks; i++) { if ((cur < (void *)&chunk_hdr[i]) && @@ -596,7 +621,7 @@ static void qat_uclo_map_image_page(struct icp_qat_uof_encap_obj page->uwblock = (struct icp_qat_uclo_encap_uwblock *)uwblock; for (i = 0; i < uword_block_tab->entry_num; i++) page->uwblock[i].micro_words = - (unsigned long)encap_uof_obj->beg_uof + uwblock[i].uword_offset; + (uintptr_t)encap_uof_obj->beg_uof + uwblock[i].uword_offset; } static int qat_uclo_map_uimage(struct icp_qat_uclo_objhandle *obj_handle, @@ -697,7 +722,7 @@ qat_uclo_map_str_table(struct icp_qat_uclo_objhdr *obj_hdr, memcpy(&str_table->table_len, obj_hdr->file_buff + chunk_hdr->offset, sizeof(str_table->table_len)); hdr_size = (char *)&str_table->strings - (char *)str_table; - str_table->strings = (unsigned long)obj_hdr->file_buff + + str_table->strings = (uintptr_t)obj_hdr->file_buff + chunk_hdr->offset + hdr_size; return str_table; } @@ -721,13 +746,31 @@ qat_uclo_map_initmem_table(struct icp_qat_uof_encap_obj *encap_uof_obj, } } +static unsigned int +qat_uclo_get_dev_type(struct icp_qat_fw_loader_handle *handle) +{ + switch (handle->pci_dev->device) { + case ADF_DH895XCC_PCI_DEVICE_ID: + return ICP_QAT_AC_895XCC_DEV_TYPE; + case ADF_C62X_PCI_DEVICE_ID: + return ICP_QAT_AC_C62X_DEV_TYPE; + case ADF_C3XXX_PCI_DEVICE_ID: + return ICP_QAT_AC_C3XXX_DEV_TYPE; + default: + pr_err("QAT: unsupported device 0x%x\n", + handle->pci_dev->device); + return 0; + } +} + static int qat_uclo_check_uof_compat(struct icp_qat_uclo_objhandle *obj_handle) { unsigned int maj_ver, prod_type = obj_handle->prod_type; - if (!(prod_type & obj_handle->encap_uof_obj.obj_hdr->cpu_type)) { - pr_err("QAT: UOF type 0x%x not match with cur platform 0x%x\n", - obj_handle->encap_uof_obj.obj_hdr->cpu_type, prod_type); + if (!(prod_type & obj_handle->encap_uof_obj.obj_hdr->ac_dev_type)) { + pr_err("QAT: UOF type 0x%x doesn't match with platform 0x%x\n", + obj_handle->encap_uof_obj.obj_hdr->ac_dev_type, + prod_type); return -EINVAL; } maj_ver = obj_handle->prod_rev & 0xff; @@ -932,7 +975,7 @@ static int qat_uclo_parse_uof_obj(struct icp_qat_fw_loader_handle *handle) obj_handle->encap_uof_obj.obj_hdr = (struct icp_qat_uof_objhdr *) obj_handle->obj_hdr->file_buff; obj_handle->uword_in_bytes = 6; - obj_handle->prod_type = ICP_QAT_AC_C_CPU_TYPE; + obj_handle->prod_type = qat_uclo_get_dev_type(handle); obj_handle->prod_rev = PID_MAJOR_REV | (PID_MINOR_REV & handle->hal_handle->revision_id); if (qat_uclo_check_uof_compat(obj_handle)) { @@ -969,23 +1012,435 @@ out_err: return -EFAULT; } -void qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, - void *addr_ptr, int mem_size) +static int qat_uclo_map_suof_file_hdr(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_suof_filehdr *suof_ptr, + int suof_size) { - qat_uclo_wr_sram_by_words(handle, 0, addr_ptr, ALIGN(mem_size, 4)); + unsigned int check_sum = 0; + unsigned int min_ver_offset = 0; + struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; + + suof_handle->file_id = ICP_QAT_SUOF_FID; + suof_handle->suof_buf = (char *)suof_ptr; + suof_handle->suof_size = suof_size; + min_ver_offset = suof_size - offsetof(struct icp_qat_suof_filehdr, + min_ver); + check_sum = qat_uclo_calc_str_checksum((char *)&suof_ptr->min_ver, + min_ver_offset); + if (check_sum != suof_ptr->check_sum) { + pr_err("QAT: incorrect SUOF checksum\n"); + return -EINVAL; + } + suof_handle->check_sum = suof_ptr->check_sum; + suof_handle->min_ver = suof_ptr->min_ver; + suof_handle->maj_ver = suof_ptr->maj_ver; + suof_handle->fw_type = suof_ptr->fw_type; + return 0; } -int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle, - void *addr_ptr, int mem_size) +static void qat_uclo_map_simg(struct icp_qat_suof_handle *suof_handle, + struct icp_qat_suof_img_hdr *suof_img_hdr, + struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) { - struct icp_qat_uof_filehdr *filehdr; - struct icp_qat_uclo_objhandle *objhdl; + struct icp_qat_simg_ae_mode *ae_mode; + struct icp_qat_suof_objhdr *suof_objhdr; + + suof_img_hdr->simg_buf = (suof_handle->suof_buf + + suof_chunk_hdr->offset + + sizeof(*suof_objhdr)); + suof_img_hdr->simg_len = ((struct icp_qat_suof_objhdr *)(uintptr_t) + (suof_handle->suof_buf + + suof_chunk_hdr->offset))->img_length; + + suof_img_hdr->css_header = suof_img_hdr->simg_buf; + suof_img_hdr->css_key = (suof_img_hdr->css_header + + sizeof(struct icp_qat_css_hdr)); + suof_img_hdr->css_signature = suof_img_hdr->css_key + + ICP_QAT_CSS_FWSK_MODULUS_LEN + + ICP_QAT_CSS_FWSK_EXPONENT_LEN; + suof_img_hdr->css_simg = suof_img_hdr->css_signature + + ICP_QAT_CSS_SIGNATURE_LEN; + + ae_mode = (struct icp_qat_simg_ae_mode *)(suof_img_hdr->css_simg); + suof_img_hdr->ae_mask = ae_mode->ae_mask; + suof_img_hdr->simg_name = (unsigned long)&ae_mode->simg_name; + suof_img_hdr->appmeta_data = (unsigned long)&ae_mode->appmeta_data; + suof_img_hdr->fw_type = ae_mode->fw_type; +} - BUILD_BUG_ON(ICP_QAT_UCLO_MAX_AE >= - (sizeof(handle->hal_handle->ae_mask) * 8)); +static void +qat_uclo_map_suof_symobjs(struct icp_qat_suof_handle *suof_handle, + struct icp_qat_suof_chunk_hdr *suof_chunk_hdr) +{ + char **sym_str = (char **)&suof_handle->sym_str; + unsigned int *sym_size = &suof_handle->sym_size; + struct icp_qat_suof_strtable *str_table_obj; + + *sym_size = *(unsigned int *)(uintptr_t) + (suof_chunk_hdr->offset + suof_handle->suof_buf); + *sym_str = (char *)(uintptr_t) + (suof_handle->suof_buf + suof_chunk_hdr->offset + + sizeof(str_table_obj->tab_length)); +} - if (!handle || !addr_ptr || mem_size < 24) +static int qat_uclo_check_simg_compat(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_suof_img_hdr *img_hdr) +{ + struct icp_qat_simg_ae_mode *img_ae_mode = NULL; + unsigned int prod_rev, maj_ver, prod_type; + + prod_type = qat_uclo_get_dev_type(handle); + img_ae_mode = (struct icp_qat_simg_ae_mode *)img_hdr->css_simg; + prod_rev = PID_MAJOR_REV | + (PID_MINOR_REV & handle->hal_handle->revision_id); + if (img_ae_mode->dev_type != prod_type) { + pr_err("QAT: incompatible product type %x\n", + img_ae_mode->dev_type); return -EINVAL; + } + maj_ver = prod_rev & 0xff; + if ((maj_ver > img_ae_mode->devmax_ver) || + (maj_ver < img_ae_mode->devmin_ver)) { + pr_err("QAT: incompatible device majver 0x%x\n", maj_ver); + return -EINVAL; + } + return 0; +} + +static void qat_uclo_del_suof(struct icp_qat_fw_loader_handle *handle) +{ + struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle; + + kfree(sobj_handle->img_table.simg_hdr); + sobj_handle->img_table.simg_hdr = NULL; + kfree(handle->sobj_handle); + handle->sobj_handle = NULL; +} + +static void qat_uclo_tail_img(struct icp_qat_suof_img_hdr *suof_img_hdr, + unsigned int img_id, unsigned int num_simgs) +{ + struct icp_qat_suof_img_hdr img_header; + + if (img_id != num_simgs - 1) { + memcpy(&img_header, &suof_img_hdr[num_simgs - 1], + sizeof(*suof_img_hdr)); + memcpy(&suof_img_hdr[num_simgs - 1], &suof_img_hdr[img_id], + sizeof(*suof_img_hdr)); + memcpy(&suof_img_hdr[img_id], &img_header, + sizeof(*suof_img_hdr)); + } +} + +static int qat_uclo_map_suof(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_suof_filehdr *suof_ptr, + int suof_size) +{ + struct icp_qat_suof_handle *suof_handle = handle->sobj_handle; + struct icp_qat_suof_chunk_hdr *suof_chunk_hdr = NULL; + struct icp_qat_suof_img_hdr *suof_img_hdr = NULL; + int ret = 0, ae0_img = ICP_QAT_UCLO_MAX_AE; + unsigned int i = 0; + struct icp_qat_suof_img_hdr img_header; + + if (!suof_ptr || (suof_size == 0)) { + pr_err("QAT: input parameter SUOF pointer/size is NULL\n"); + return -EINVAL; + } + if (qat_uclo_check_suof_format(suof_ptr)) + return -EINVAL; + ret = qat_uclo_map_suof_file_hdr(handle, suof_ptr, suof_size); + if (ret) + return ret; + suof_chunk_hdr = (struct icp_qat_suof_chunk_hdr *) + ((uintptr_t)suof_ptr + sizeof(*suof_ptr)); + + qat_uclo_map_suof_symobjs(suof_handle, suof_chunk_hdr); + suof_handle->img_table.num_simgs = suof_ptr->num_chunks - 1; + + if (suof_handle->img_table.num_simgs != 0) { + suof_img_hdr = kzalloc(suof_handle->img_table.num_simgs * + sizeof(img_header), GFP_KERNEL); + if (!suof_img_hdr) + return -ENOMEM; + suof_handle->img_table.simg_hdr = suof_img_hdr; + } + + for (i = 0; i < suof_handle->img_table.num_simgs; i++) { + qat_uclo_map_simg(handle->sobj_handle, &suof_img_hdr[i], + &suof_chunk_hdr[1 + i]); + ret = qat_uclo_check_simg_compat(handle, + &suof_img_hdr[i]); + if (ret) + return ret; + if ((suof_img_hdr[i].ae_mask & 0x1) != 0) + ae0_img = i; + } + qat_uclo_tail_img(suof_img_hdr, ae0_img, + suof_handle->img_table.num_simgs); + return 0; +} + +#define ADD_ADDR(high, low) ((((uint64_t)high) << 32) + low) +#define BITS_IN_DWORD 32 + +static int qat_uclo_auth_fw(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_fw_auth_desc *desc) +{ + unsigned int fcu_sts, retry = 0; + u64 bus_addr; + + bus_addr = ADD_ADDR(desc->css_hdr_high, desc->css_hdr_low) + - sizeof(struct icp_qat_auth_chunk); + SET_CAP_CSR(handle, FCU_DRAM_ADDR_HI, (bus_addr >> BITS_IN_DWORD)); + SET_CAP_CSR(handle, FCU_DRAM_ADDR_LO, bus_addr); + SET_CAP_CSR(handle, FCU_CONTROL, FCU_CTRL_CMD_AUTH); + + do { + msleep(FW_AUTH_WAIT_PERIOD); + fcu_sts = GET_CAP_CSR(handle, FCU_STATUS); + if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_FAIL) + goto auth_fail; + if (((fcu_sts >> FCU_STS_AUTHFWLD_POS) & 0x1)) + if ((fcu_sts & FCU_AUTH_STS_MASK) == FCU_STS_VERI_DONE) + return 0; + } while (retry++ < FW_AUTH_MAX_RETRY); +auth_fail: + pr_err("QAT: authentication error (FCU_STATUS = 0x%x),retry = %d\n", + fcu_sts & FCU_AUTH_STS_MASK, retry); + return -EINVAL; +} + +static int qat_uclo_simg_alloc(struct icp_qat_fw_loader_handle *handle, + struct icp_firml_dram_desc *dram_desc, + unsigned int size) +{ + void *vptr; + dma_addr_t ptr; + + vptr = dma_alloc_coherent(&handle->pci_dev->dev, + size, &ptr, GFP_KERNEL); + if (!vptr) + return -ENOMEM; + dram_desc->dram_base_addr_v = vptr; + dram_desc->dram_bus_addr = ptr; + dram_desc->dram_size = size; + return 0; +} + +static void qat_uclo_simg_free(struct icp_qat_fw_loader_handle *handle, + struct icp_firml_dram_desc *dram_desc) +{ + dma_free_coherent(&handle->pci_dev->dev, + (size_t)(dram_desc->dram_size), + (dram_desc->dram_base_addr_v), + dram_desc->dram_bus_addr); + memset(dram_desc, 0, sizeof(*dram_desc)); +} + +static void qat_uclo_ummap_auth_fw(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_fw_auth_desc **desc) +{ + struct icp_firml_dram_desc dram_desc; + + dram_desc.dram_base_addr_v = *desc; + dram_desc.dram_bus_addr = ((struct icp_qat_auth_chunk *) + (*desc))->chunk_bus_addr; + dram_desc.dram_size = ((struct icp_qat_auth_chunk *) + (*desc))->chunk_size; + qat_uclo_simg_free(handle, &dram_desc); +} + +static int qat_uclo_map_auth_fw(struct icp_qat_fw_loader_handle *handle, + char *image, unsigned int size, + struct icp_qat_fw_auth_desc **desc) +{ + struct icp_qat_css_hdr *css_hdr = (struct icp_qat_css_hdr *)image; + struct icp_qat_fw_auth_desc *auth_desc; + struct icp_qat_auth_chunk *auth_chunk; + u64 virt_addr, bus_addr, virt_base; + unsigned int length, simg_offset = sizeof(*auth_chunk); + struct icp_firml_dram_desc img_desc; + + if (size > (ICP_QAT_AE_IMG_OFFSET + ICP_QAT_CSS_MAX_IMAGE_LEN)) { + pr_err("QAT: error, input image size overflow %d\n", size); + return -EINVAL; + } + length = (css_hdr->fw_type == CSS_AE_FIRMWARE) ? + ICP_QAT_CSS_AE_SIMG_LEN + simg_offset : + size + ICP_QAT_CSS_FWSK_PAD_LEN + simg_offset; + if (qat_uclo_simg_alloc(handle, &img_desc, length)) { + pr_err("QAT: error, allocate continuous dram fail\n"); + return -ENOMEM; + } + + auth_chunk = img_desc.dram_base_addr_v; + auth_chunk->chunk_size = img_desc.dram_size; + auth_chunk->chunk_bus_addr = img_desc.dram_bus_addr; + virt_base = (uintptr_t)img_desc.dram_base_addr_v + simg_offset; + bus_addr = img_desc.dram_bus_addr + simg_offset; + auth_desc = img_desc.dram_base_addr_v; + auth_desc->css_hdr_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); + auth_desc->css_hdr_low = (unsigned int)bus_addr; + virt_addr = virt_base; + + memcpy((void *)(uintptr_t)virt_addr, image, sizeof(*css_hdr)); + /* pub key */ + bus_addr = ADD_ADDR(auth_desc->css_hdr_high, auth_desc->css_hdr_low) + + sizeof(*css_hdr); + virt_addr = virt_addr + sizeof(*css_hdr); + + auth_desc->fwsk_pub_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); + auth_desc->fwsk_pub_low = (unsigned int)bus_addr; + + memcpy((void *)(uintptr_t)virt_addr, + (void *)(image + sizeof(*css_hdr)), + ICP_QAT_CSS_FWSK_MODULUS_LEN); + /* padding */ + memset((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN), + 0, ICP_QAT_CSS_FWSK_PAD_LEN); + + /* exponent */ + memcpy((void *)(uintptr_t)(virt_addr + ICP_QAT_CSS_FWSK_MODULUS_LEN + + ICP_QAT_CSS_FWSK_PAD_LEN), + (void *)(image + sizeof(*css_hdr) + + ICP_QAT_CSS_FWSK_MODULUS_LEN), + sizeof(unsigned int)); + + /* signature */ + bus_addr = ADD_ADDR(auth_desc->fwsk_pub_high, + auth_desc->fwsk_pub_low) + + ICP_QAT_CSS_FWSK_PUB_LEN; + virt_addr = virt_addr + ICP_QAT_CSS_FWSK_PUB_LEN; + auth_desc->signature_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); + auth_desc->signature_low = (unsigned int)bus_addr; + + memcpy((void *)(uintptr_t)virt_addr, + (void *)(image + sizeof(*css_hdr) + + ICP_QAT_CSS_FWSK_MODULUS_LEN + + ICP_QAT_CSS_FWSK_EXPONENT_LEN), + ICP_QAT_CSS_SIGNATURE_LEN); + + bus_addr = ADD_ADDR(auth_desc->signature_high, + auth_desc->signature_low) + + ICP_QAT_CSS_SIGNATURE_LEN; + virt_addr += ICP_QAT_CSS_SIGNATURE_LEN; + + auth_desc->img_high = (unsigned int)(bus_addr >> BITS_IN_DWORD); + auth_desc->img_low = (unsigned int)bus_addr; + auth_desc->img_len = size - ICP_QAT_AE_IMG_OFFSET; + memcpy((void *)(uintptr_t)virt_addr, + (void *)(image + ICP_QAT_AE_IMG_OFFSET), + auth_desc->img_len); + virt_addr = virt_base; + /* AE firmware */ + if (((struct icp_qat_css_hdr *)(uintptr_t)virt_addr)->fw_type == + CSS_AE_FIRMWARE) { + auth_desc->img_ae_mode_data_high = auth_desc->img_high; + auth_desc->img_ae_mode_data_low = auth_desc->img_low; + bus_addr = ADD_ADDR(auth_desc->img_ae_mode_data_high, + auth_desc->img_ae_mode_data_low) + + sizeof(struct icp_qat_simg_ae_mode); + + auth_desc->img_ae_init_data_high = (unsigned int) + (bus_addr >> BITS_IN_DWORD); + auth_desc->img_ae_init_data_low = (unsigned int)bus_addr; + bus_addr += ICP_QAT_SIMG_AE_INIT_SEQ_LEN; + auth_desc->img_ae_insts_high = (unsigned int) + (bus_addr >> BITS_IN_DWORD); + auth_desc->img_ae_insts_low = (unsigned int)bus_addr; + } else { + auth_desc->img_ae_insts_high = auth_desc->img_high; + auth_desc->img_ae_insts_low = auth_desc->img_low; + } + *desc = auth_desc; + return 0; +} + +static int qat_uclo_load_fw(struct icp_qat_fw_loader_handle *handle, + struct icp_qat_fw_auth_desc *desc) +{ + unsigned int i; + unsigned int fcu_sts; + struct icp_qat_simg_ae_mode *virt_addr; + unsigned int fcu_loaded_ae_pos = FCU_LOADED_AE_POS; + + virt_addr = (void *)((uintptr_t)desc + + sizeof(struct icp_qat_auth_chunk) + + sizeof(struct icp_qat_css_hdr) + + ICP_QAT_CSS_FWSK_PUB_LEN + + ICP_QAT_CSS_SIGNATURE_LEN); + for (i = 0; i < handle->hal_handle->ae_max_num; i++) { + int retry = 0; + + if (!((virt_addr->ae_mask >> i) & 0x1)) + continue; + if (qat_hal_check_ae_active(handle, i)) { + pr_err("QAT: AE %d is active\n", i); + return -EINVAL; + } + SET_CAP_CSR(handle, FCU_CONTROL, + (FCU_CTRL_CMD_LOAD | (i << FCU_CTRL_AE_POS))); + + do { + msleep(FW_AUTH_WAIT_PERIOD); + fcu_sts = GET_CAP_CSR(handle, FCU_STATUS); + if (((fcu_sts & FCU_AUTH_STS_MASK) == + FCU_STS_LOAD_DONE) && + ((fcu_sts >> fcu_loaded_ae_pos) & (1 << i))) + break; + } while (retry++ < FW_AUTH_MAX_RETRY); + if (retry > FW_AUTH_MAX_RETRY) { + pr_err("QAT: firmware load failed timeout %x\n", retry); + return -EINVAL; + } + } + return 0; +} + +static int qat_uclo_map_suof_obj(struct icp_qat_fw_loader_handle *handle, + void *addr_ptr, int mem_size) +{ + struct icp_qat_suof_handle *suof_handle; + + suof_handle = kzalloc(sizeof(*suof_handle), GFP_KERNEL); + if (!suof_handle) + return -ENOMEM; + handle->sobj_handle = suof_handle; + if (qat_uclo_map_suof(handle, addr_ptr, mem_size)) { + qat_uclo_del_suof(handle); + pr_err("QAT: map SUOF failed\n"); + return -EINVAL; + } + return 0; +} + +int qat_uclo_wr_mimage(struct icp_qat_fw_loader_handle *handle, + void *addr_ptr, int mem_size) +{ + struct icp_qat_fw_auth_desc *desc = NULL; + int status = 0; + + if (handle->fw_auth) { + if (!qat_uclo_map_auth_fw(handle, addr_ptr, mem_size, &desc)) + status = qat_uclo_auth_fw(handle, desc); + qat_uclo_ummap_auth_fw(handle, &desc); + } else { + if (handle->pci_dev->device == ADF_C3XXX_PCI_DEVICE_ID) { + pr_err("QAT: C3XXX doesn't support unsigned MMP\n"); + return -EINVAL; + } + qat_uclo_wr_sram_by_words(handle, 0, addr_ptr, mem_size); + } + return status; +} + +static int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle, + void *addr_ptr, int mem_size) +{ + struct icp_qat_uof_filehdr *filehdr; + struct icp_qat_uclo_objhandle *objhdl; + objhdl = kzalloc(sizeof(*objhdl), GFP_KERNEL); if (!objhdl) return -ENOMEM; @@ -993,7 +1448,7 @@ int qat_uclo_map_uof_obj(struct icp_qat_fw_loader_handle *handle, if (!objhdl->obj_buf) goto out_objbuf_err; filehdr = (struct icp_qat_uof_filehdr *)objhdl->obj_buf; - if (qat_uclo_check_format(filehdr)) + if (qat_uclo_check_uof_format(filehdr)) goto out_objhdr_err; objhdl->obj_hdr = qat_uclo_map_chunk((char *)objhdl->obj_buf, filehdr, ICP_QAT_UOF_OBJS); @@ -1016,11 +1471,27 @@ out_objbuf_err: return -ENOMEM; } +int qat_uclo_map_obj(struct icp_qat_fw_loader_handle *handle, + void *addr_ptr, int mem_size) +{ + BUILD_BUG_ON(ICP_QAT_UCLO_MAX_AE >= + (sizeof(handle->hal_handle->ae_mask) * 8)); + + if (!handle || !addr_ptr || mem_size < 24) + return -EINVAL; + + return (handle->fw_auth) ? + qat_uclo_map_suof_obj(handle, addr_ptr, mem_size) : + qat_uclo_map_uof_obj(handle, addr_ptr, mem_size); +} + void qat_uclo_del_uof_obj(struct icp_qat_fw_loader_handle *handle) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned int a; + if (handle->sobj_handle) + qat_uclo_del_suof(handle); if (!obj_handle) return; @@ -1055,7 +1526,7 @@ static void qat_uclo_fill_uwords(struct icp_qat_uclo_objhandle *obj_handle, encap_page->uwblock[i].words_num - 1) { raddr -= encap_page->uwblock[i].start_addr; raddr *= obj_handle->uword_in_bytes; - memcpy(&uwrd, (void *)(((unsigned long) + memcpy(&uwrd, (void *)(((uintptr_t) encap_page->uwblock[i].micro_words) + raddr), obj_handle->uword_in_bytes); uwrd = uwrd & 0xbffffffffffull; @@ -1147,7 +1618,33 @@ static void qat_uclo_wr_uimage_page(struct icp_qat_fw_loader_handle *handle, } } -int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle) +static int qat_uclo_wr_suof_img(struct icp_qat_fw_loader_handle *handle) +{ + unsigned int i; + struct icp_qat_fw_auth_desc *desc = NULL; + struct icp_qat_suof_handle *sobj_handle = handle->sobj_handle; + struct icp_qat_suof_img_hdr *simg_hdr = sobj_handle->img_table.simg_hdr; + + for (i = 0; i < sobj_handle->img_table.num_simgs; i++) { + if (qat_uclo_map_auth_fw(handle, + (char *)simg_hdr[i].simg_buf, + (unsigned int) + (simg_hdr[i].simg_len), + &desc)) + goto wr_err; + if (qat_uclo_auth_fw(handle, desc)) + goto wr_err; + if (qat_uclo_load_fw(handle, desc)) + goto wr_err; + qat_uclo_ummap_auth_fw(handle, &desc); + } + return 0; +wr_err: + qat_uclo_ummap_auth_fw(handle, &desc); + return -EINVAL; +} + +static int qat_uclo_wr_uof_img(struct icp_qat_fw_loader_handle *handle) { struct icp_qat_uclo_objhandle *obj_handle = handle->obj_handle; unsigned int i; @@ -1164,3 +1661,9 @@ int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle) } return 0; } + +int qat_uclo_wr_all_uimage(struct icp_qat_fw_loader_handle *handle) +{ + return (handle->fw_auth) ? qat_uclo_wr_suof_img(handle) : + qat_uclo_wr_uof_img(handle); +} diff --git a/drivers/crypto/qat/qat_dh895xcc/Makefile b/drivers/crypto/qat/qat_dh895xcc/Makefile index 8c79c543740f..180a00ed7f89 100644 --- a/drivers/crypto/qat/qat_dh895xcc/Makefile +++ b/drivers/crypto/qat/qat_dh895xcc/Makefile @@ -1,5 +1,3 @@ ccflags-y := -I$(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCC) += qat_dh895xcc.o -qat_dh895xcc-objs := adf_drv.o \ - adf_isr.o \ - adf_dh895xcc_hw_data.o +qat_dh895xcc-objs := adf_drv.o adf_dh895xcc_hw_data.o diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index ff54257eced4..6e1d5e185526 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -48,7 +48,6 @@ #include <adf_pf2vf_msg.h> #include <adf_common_drv.h> #include "adf_dh895xcc_hw_data.h" -#include "adf_drv.h" /* Worker thread to service arbiter mappings based on dev SKUs */ static const uint32_t thrd_to_arb_map_sku4[] = { @@ -143,8 +142,8 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self) return DEV_SKU_UNKNOWN; } -void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, - uint32_t const **arb_map_config) +static void adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev, + u32 const **arb_map_config) { switch (accel_dev->accel_pci_dev.sku) { case DEV_SKU_1: diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h index 88dffb297346..092f7353ed23 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.h @@ -53,7 +53,6 @@ #define ADF_DH895XCC_ETR_BAR 2 #define ADF_DH895XCC_RX_RINGS_OFFSET 8 #define ADF_DH895XCC_TX_RINGS_MASK 0xFF -#define ADF_DH895XCC_FUSECTL_OFFSET 0x40 #define ADF_DH895XCC_FUSECTL_SKU_MASK 0x300000 #define ADF_DH895XCC_FUSECTL_SKU_SHIFT 20 #define ADF_DH895XCC_FUSECTL_SKU_1 0x0 @@ -65,7 +64,6 @@ #define ADF_DH895XCC_ACCELERATORS_REG_OFFSET 13 #define ADF_DH895XCC_ACCELERATORS_MASK 0x3F #define ADF_DH895XCC_ACCELENGINES_MASK 0xFFF -#define ADF_DH895XCC_LEGFUSE_OFFSET 0x4C #define ADF_DH895XCC_ETR_MAX_BANKS 32 #define ADF_DH895XCC_SMIAPF0_MASK_OFFSET (0x3A000 + 0x28) #define ADF_DH895XCC_SMIAPF1_MASK_OFFSET (0x3A000 + 0x30) @@ -80,11 +78,12 @@ #define ADF_DH895XCC_CERRSSMSH(i) (i * 0x4000 + 0x10) #define ADF_DH895XCC_ERRSSMSH_EN BIT(3) -#define ADF_DH895XCC_ERRSOU3 (0x3A000 + 0x00C) -#define ADF_DH895XCC_ERRSOU5 (0x3A000 + 0x0D8) #define ADF_DH895XCC_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) #define ADF_DH895XCC_VINTMSK_OFFSET(i) (0x3A000 + 0x200 + ((i) * 0x04)) /* FW names */ #define ADF_DH895XCC_FW "qat_895xcc.bin" -#define ADF_DH895XCC_MMP "qat_mmp.bin" +#define ADF_DH895XCC_MMP "qat_895xcc_mmp.bin" + +void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data); +void adf_clean_hw_data_dh895xcc(struct adf_hw_device_data *hw_data); #endif diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c index f8dd14f232c8..a8c4b92a7cbd 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c @@ -60,11 +60,7 @@ #include <adf_accel_devices.h> #include <adf_common_drv.h> #include <adf_cfg.h> -#include <adf_transport_access_macros.h> #include "adf_dh895xcc_hw_data.h" -#include "adf_drv.h" - -static const char adf_driver_name[] = ADF_DH895XCC_DEVICE_NAME; #define ADF_SYSTEM_DEVICE(device_id) \ {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} @@ -80,7 +76,7 @@ static void adf_remove(struct pci_dev *dev); static struct pci_driver adf_driver = { .id_table = adf_pci_tbl, - .name = adf_driver_name, + .name = ADF_DH895XCC_DEVICE_NAME, .probe = adf_probe, .remove = adf_remove, .sriov_configure = adf_sriov_configure, @@ -120,87 +116,6 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) adf_devmgr_rm_dev(accel_dev, NULL); } -static int adf_dev_configure(struct adf_accel_dev *accel_dev) -{ - int cpus = num_online_cpus(); - int banks = GET_MAX_BANKS(accel_dev); - int instances = min(cpus, banks); - char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; - int i; - unsigned long val; - - if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC)) - goto err; - if (adf_cfg_section_add(accel_dev, "Accelerator0")) - goto err; - for (i = 0; i < instances; i++) { - val = i; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) - goto err; - - snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, - i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) - goto err; - - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, i); - val = 128; - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) - goto err; - - val = 512; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) - goto err; - - val = 0; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) - goto err; - - val = 2; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) - goto err; - - val = 8; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) - goto err; - - val = 10; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, i); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) - goto err; - - val = ADF_COALESCING_DEF_TIME; - snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, i); - if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0", - key, (void *)&val, ADF_DEC)) - goto err; - } - - val = i; - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - ADF_NUM_CY, (void *)&val, ADF_DEC)) - goto err; - - set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); - return 0; -err: - dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n"); - return -EINVAL; -} - static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { struct adf_accel_dev *accel_dev; @@ -253,15 +168,9 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } accel_dev->hw_device = hw_data; - switch (ent->device) { - case ADF_DH895XCC_PCI_DEVICE_ID: - adf_init_hw_data_dh895xcc(accel_dev->hw_device); - break; - default: - return -ENODEV; - } + adf_init_hw_data_dh895xcc(accel_dev->hw_device); pci_read_config_byte(pdev, PCI_REVISION_ID, &accel_pci_dev->revid); - pci_read_config_dword(pdev, ADF_DH895XCC_FUSECTL_OFFSET, + pci_read_config_dword(pdev, ADF_DEVICE_FUSECTL_OFFSET, &hw_data->fuses); /* Get Accelerators and Accelerators Engines masks */ @@ -316,13 +225,13 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); } - if (pci_request_regions(pdev, adf_driver_name)) { + if (pci_request_regions(pdev, ADF_DH895XCC_DEVICE_NAME)) { ret = -EFAULT; goto out_err_disable; } /* Read accelerator capabilities mask */ - pci_read_config_dword(pdev, ADF_DH895XCC_LEGFUSE_OFFSET, + pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, &hw_data->accel_capabilities_mask); /* Find and map all the device's BARS */ @@ -357,7 +266,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err_free_reg; } - ret = adf_dev_configure(accel_dev); + ret = qat_crypto_dev_config(accel_dev); if (ret) goto out_err_free_reg; diff --git a/drivers/crypto/qat/qat_dh895xccvf/Makefile b/drivers/crypto/qat/qat_dh895xccvf/Makefile index 85399fcbbad4..5c3ccf8267eb 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/Makefile +++ b/drivers/crypto/qat/qat_dh895xccvf/Makefile @@ -1,5 +1,3 @@ ccflags-y := -I$(src)/../qat_common obj-$(CONFIG_CRYPTO_DEV_QAT_DH895xCCVF) += qat_dh895xccvf.o -qat_dh895xccvf-objs := adf_drv.o \ - adf_isr.o \ - adf_dh895xccvf_hw_data.o +qat_dh895xccvf-objs := adf_drv.o adf_dh895xccvf_hw_data.o diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c index a9a27eff41fb..dc04ab68d24d 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.c @@ -48,7 +48,6 @@ #include <adf_pf2vf_msg.h> #include <adf_common_drv.h> #include "adf_dh895xccvf_hw_data.h" -#include "adf_drv.h" static struct adf_hw_device_class dh895xcciov_class = { .name = ADF_DH895XCCVF_DEVICE_NAME, @@ -136,7 +135,6 @@ static void adf_vf2pf_shutdown(struct adf_accel_dev *accel_dev) void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &dh895xcciov_class; - hw_data->instance_id = dh895xcciov_class.instances++; hw_data->num_banks = ADF_DH895XCCIOV_ETR_MAX_BANKS; hw_data->num_accel = ADF_DH895XCCIOV_MAX_ACCELERATORS; hw_data->num_logical_accel = 1; @@ -164,9 +162,12 @@ void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) hw_data->enable_ints = adf_vf_void_noop; hw_data->enable_vf2pf_comms = adf_enable_vf2pf_comms; hw_data->min_iov_compat_ver = ADF_PFVF_COMPATIBILITY_VERSION; + hw_data->dev_class->instances++; + adf_devmgr_update_class_index(hw_data); } void adf_clean_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data) { hw_data->dev_class->instances--; + adf_devmgr_update_class_index(hw_data); } diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h index 8f6babfef629..6ddc19bd4410 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_dh895xccvf_hw_data.h @@ -56,13 +56,9 @@ #define ADF_DH895XCCIOV_TX_RINGS_MASK 0xFF #define ADF_DH895XCCIOV_ETR_BAR 0 #define ADF_DH895XCCIOV_ETR_MAX_BANKS 1 - #define ADF_DH895XCCIOV_PF2VF_OFFSET 0x200 -#define ADF_DH895XCC_PF2VF_PF2VFINT BIT(0) - -#define ADF_DH895XCCIOV_VINTSOU_OFFSET 0x204 -#define ADF_DH895XCC_VINTSOU_BUN BIT(0) -#define ADF_DH895XCC_VINTSOU_PF2VF BIT(1) - #define ADF_DH895XCCIOV_VINTMSK_OFFSET 0x208 + +void adf_init_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data); +void adf_clean_hw_data_dh895xcciov(struct adf_hw_device_data *hw_data); #endif diff --git a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c index 789426f21882..f8cc4bf0a50c 100644 --- a/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c +++ b/drivers/crypto/qat/qat_dh895xccvf/adf_drv.c @@ -60,11 +60,7 @@ #include <adf_accel_devices.h> #include <adf_common_drv.h> #include <adf_cfg.h> -#include <adf_transport_access_macros.h> #include "adf_dh895xccvf_hw_data.h" -#include "adf_drv.h" - -static const char adf_driver_name[] = ADF_DH895XCCVF_DEVICE_NAME; #define ADF_SYSTEM_DEVICE(device_id) \ {PCI_DEVICE(PCI_VENDOR_ID_INTEL, device_id)} @@ -80,7 +76,7 @@ static void adf_remove(struct pci_dev *dev); static struct pci_driver adf_driver = { .id_table = adf_pci_tbl, - .name = adf_driver_name, + .name = ADF_DH895XCCVF_DEVICE_NAME, .probe = adf_probe, .remove = adf_remove, }; @@ -121,83 +117,6 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) adf_devmgr_rm_dev(accel_dev, pf); } -static int adf_dev_configure(struct adf_accel_dev *accel_dev) -{ - char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; - unsigned long val, bank = 0; - - if (adf_cfg_section_add(accel_dev, ADF_KERNEL_SEC)) - goto err; - if (adf_cfg_section_add(accel_dev, "Accelerator0")) - goto err; - - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_BANK_NUM, 0); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, key, - (void *)&bank, ADF_DEC)) - goto err; - - val = bank; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_ETRMGR_CORE_AFFINITY, 0); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, key, - (void *)&val, ADF_DEC)) - goto err; - - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_SIZE, 0); - - val = 128; - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, key, - (void *)&val, ADF_DEC)) - goto err; - - val = 512; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_SIZE, 0); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) - goto err; - - val = 0; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_TX, 0); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) - goto err; - - val = 2; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_TX, 0); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) - goto err; - - val = 8; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_ASYM_RX, 0); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) - goto err; - - val = 10; - snprintf(key, sizeof(key), ADF_CY "%d" ADF_RING_SYM_RX, 0); - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - key, (void *)&val, ADF_DEC)) - goto err; - - val = ADF_COALESCING_DEF_TIME; - snprintf(key, sizeof(key), ADF_ETRMGR_COALESCE_TIMER_FORMAT, - (int)bank); - if (adf_cfg_add_key_value_param(accel_dev, "Accelerator0", - key, (void *)&val, ADF_DEC)) - goto err; - - val = 1; - if (adf_cfg_add_key_value_param(accel_dev, ADF_KERNEL_SEC, - ADF_NUM_CY, (void *)&val, ADF_DEC)) - goto err; - - set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); - return 0; -err: - dev_err(&GET_DEV(accel_dev), "Failed to configure QAT accel dev\n"); - return -EINVAL; -} - static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) { struct adf_accel_dev *accel_dev; @@ -243,14 +162,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err; } accel_dev->hw_device = hw_data; - switch (ent->device) { - case ADF_DH895XCCIOV_PCI_DEVICE_ID: - adf_init_hw_data_dh895xcciov(accel_dev->hw_device); - break; - default: - ret = -ENODEV; - goto out_err; - } + adf_init_hw_data_dh895xcciov(accel_dev->hw_device); /* Get Accelerators and Accelerators Engines masks */ hw_data->accel_mask = hw_data->get_accel_mask(hw_data->fuses); @@ -295,7 +207,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); } - if (pci_request_regions(pdev, adf_driver_name)) { + if (pci_request_regions(pdev, ADF_DH895XCCVF_DEVICE_NAME)) { ret = -EFAULT; goto out_err_disable; } @@ -322,7 +234,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* Completion for VF2PF request/response message exchange */ init_completion(&accel_dev->vf.iov_msg_completion); - ret = adf_dev_configure(accel_dev); + ret = qat_crypto_dev_config(accel_dev); if (ret) goto out_err_free_reg; diff --git a/drivers/crypto/qce/ablkcipher.c b/drivers/crypto/qce/ablkcipher.c index 2c0d63d48747..dbcbbe242bd6 100644 --- a/drivers/crypto/qce/ablkcipher.c +++ b/drivers/crypto/qce/ablkcipher.c @@ -83,6 +83,14 @@ qce_ablkcipher_async_req_handle(struct crypto_async_request *async_req) rctx->dst_nents = sg_nents_for_len(req->dst, req->nbytes); else rctx->dst_nents = rctx->src_nents; + if (rctx->src_nents < 0) { + dev_err(qce->dev, "Invalid numbers of src SG.\n"); + return rctx->src_nents; + } + if (rctx->dst_nents < 0) { + dev_err(qce->dev, "Invalid numbers of dst SG.\n"); + return -rctx->dst_nents; + } rctx->dst_nents += 1; diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c index 0c9973ec80eb..47e114ac09d0 100644 --- a/drivers/crypto/qce/sha.c +++ b/drivers/crypto/qce/sha.c @@ -92,6 +92,11 @@ static int qce_ahash_async_req_handle(struct crypto_async_request *async_req) } rctx->src_nents = sg_nents_for_len(req->src, req->nbytes); + if (rctx->src_nents < 0) { + dev_err(qce->dev, "Invalid numbers of src SG.\n"); + return rctx->src_nents; + } + ret = dma_map_sg(qce->dev, req->src, rctx->src_nents, DMA_TO_DEVICE); if (ret < 0) return ret; diff --git a/drivers/crypto/rockchip/Makefile b/drivers/crypto/rockchip/Makefile new file mode 100644 index 000000000000..7051c6c715f3 --- /dev/null +++ b/drivers/crypto/rockchip/Makefile @@ -0,0 +1,3 @@ +obj-$(CONFIG_CRYPTO_DEV_ROCKCHIP) += rk_crypto.o +rk_crypto-objs := rk3288_crypto.o \ + rk3288_crypto_ablkcipher.o \ diff --git a/drivers/crypto/rockchip/rk3288_crypto.c b/drivers/crypto/rockchip/rk3288_crypto.c new file mode 100644 index 000000000000..da9c73dce4af --- /dev/null +++ b/drivers/crypto/rockchip/rk3288_crypto.c @@ -0,0 +1,394 @@ +/* + * Crypto acceleration support for Rockchip RK3288 + * + * Copyright (c) 2015, Fuzhou Rockchip Electronics Co., Ltd + * + * Author: Zain Wang <zain.wang@rock-chips.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * Some ideas are from marvell-cesa.c and s5p-sss.c driver. + */ + +#include "rk3288_crypto.h" +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/of.h> +#include <linux/clk.h> +#include <linux/crypto.h> +#include <linux/reset.h> + +static int rk_crypto_enable_clk(struct rk_crypto_info *dev) +{ + int err; + + err = clk_prepare_enable(dev->sclk); + if (err) { + dev_err(dev->dev, "[%s:%d], Couldn't enable clock sclk\n", + __func__, __LINE__); + goto err_return; + } + err = clk_prepare_enable(dev->aclk); + if (err) { + dev_err(dev->dev, "[%s:%d], Couldn't enable clock aclk\n", + __func__, __LINE__); + goto err_aclk; + } + err = clk_prepare_enable(dev->hclk); + if (err) { + dev_err(dev->dev, "[%s:%d], Couldn't enable clock hclk\n", + __func__, __LINE__); + goto err_hclk; + } + err = clk_prepare_enable(dev->dmaclk); + if (err) { + dev_err(dev->dev, "[%s:%d], Couldn't enable clock dmaclk\n", + __func__, __LINE__); + goto err_dmaclk; + } + return err; +err_dmaclk: + clk_disable_unprepare(dev->hclk); +err_hclk: + clk_disable_unprepare(dev->aclk); +err_aclk: + clk_disable_unprepare(dev->sclk); +err_return: + return err; +} + +static void rk_crypto_disable_clk(struct rk_crypto_info *dev) +{ + clk_disable_unprepare(dev->dmaclk); + clk_disable_unprepare(dev->hclk); + clk_disable_unprepare(dev->aclk); + clk_disable_unprepare(dev->sclk); +} + +static int check_alignment(struct scatterlist *sg_src, + struct scatterlist *sg_dst, + int align_mask) +{ + int in, out, align; + + in = IS_ALIGNED((uint32_t)sg_src->offset, 4) && + IS_ALIGNED((uint32_t)sg_src->length, align_mask); + if (!sg_dst) + return in; + out = IS_ALIGNED((uint32_t)sg_dst->offset, 4) && + IS_ALIGNED((uint32_t)sg_dst->length, align_mask); + align = in && out; + + return (align && (sg_src->length == sg_dst->length)); +} + +static int rk_load_data(struct rk_crypto_info *dev, + struct scatterlist *sg_src, + struct scatterlist *sg_dst) +{ + unsigned int count; + + dev->aligned = dev->aligned ? + check_alignment(sg_src, sg_dst, dev->align_size) : + dev->aligned; + if (dev->aligned) { + count = min(dev->left_bytes, sg_src->length); + dev->left_bytes -= count; + + if (!dma_map_sg(dev->dev, sg_src, 1, DMA_TO_DEVICE)) { + dev_err(dev->dev, "[%s:%d] dma_map_sg(src) error\n", + __func__, __LINE__); + return -EINVAL; + } + dev->addr_in = sg_dma_address(sg_src); + + if (sg_dst) { + if (!dma_map_sg(dev->dev, sg_dst, 1, DMA_FROM_DEVICE)) { + dev_err(dev->dev, + "[%s:%d] dma_map_sg(dst) error\n", + __func__, __LINE__); + dma_unmap_sg(dev->dev, sg_src, 1, + DMA_TO_DEVICE); + return -EINVAL; + } + dev->addr_out = sg_dma_address(sg_dst); + } + } else { + count = (dev->left_bytes > PAGE_SIZE) ? + PAGE_SIZE : dev->left_bytes; + + if (!sg_pcopy_to_buffer(dev->first, dev->nents, + dev->addr_vir, count, + dev->total - dev->left_bytes)) { + dev_err(dev->dev, "[%s:%d] pcopy err\n", + __func__, __LINE__); + return -EINVAL; + } + dev->left_bytes -= count; + sg_init_one(&dev->sg_tmp, dev->addr_vir, count); + if (!dma_map_sg(dev->dev, &dev->sg_tmp, 1, DMA_TO_DEVICE)) { + dev_err(dev->dev, "[%s:%d] dma_map_sg(sg_tmp) error\n", + __func__, __LINE__); + return -ENOMEM; + } + dev->addr_in = sg_dma_address(&dev->sg_tmp); + + if (sg_dst) { + if (!dma_map_sg(dev->dev, &dev->sg_tmp, 1, + DMA_FROM_DEVICE)) { + dev_err(dev->dev, + "[%s:%d] dma_map_sg(sg_tmp) error\n", + __func__, __LINE__); + dma_unmap_sg(dev->dev, &dev->sg_tmp, 1, + DMA_TO_DEVICE); + return -ENOMEM; + } + dev->addr_out = sg_dma_address(&dev->sg_tmp); + } + } + dev->count = count; + return 0; +} + +static void rk_unload_data(struct rk_crypto_info *dev) +{ + struct scatterlist *sg_in, *sg_out; + + sg_in = dev->aligned ? dev->sg_src : &dev->sg_tmp; + dma_unmap_sg(dev->dev, sg_in, 1, DMA_TO_DEVICE); + + if (dev->sg_dst) { + sg_out = dev->aligned ? dev->sg_dst : &dev->sg_tmp; + dma_unmap_sg(dev->dev, sg_out, 1, DMA_FROM_DEVICE); + } +} + +static irqreturn_t rk_crypto_irq_handle(int irq, void *dev_id) +{ + struct rk_crypto_info *dev = platform_get_drvdata(dev_id); + u32 interrupt_status; + int err = 0; + + spin_lock(&dev->lock); + interrupt_status = CRYPTO_READ(dev, RK_CRYPTO_INTSTS); + CRYPTO_WRITE(dev, RK_CRYPTO_INTSTS, interrupt_status); + if (interrupt_status & 0x0a) { + dev_warn(dev->dev, "DMA Error\n"); + err = -EFAULT; + } else if (interrupt_status & 0x05) { + err = dev->update(dev); + } + if (err) + dev->complete(dev, err); + spin_unlock(&dev->lock); + return IRQ_HANDLED; +} + +static void rk_crypto_tasklet_cb(unsigned long data) +{ + struct rk_crypto_info *dev = (struct rk_crypto_info *)data; + struct crypto_async_request *async_req, *backlog; + unsigned long flags; + int err = 0; + + spin_lock_irqsave(&dev->lock, flags); + backlog = crypto_get_backlog(&dev->queue); + async_req = crypto_dequeue_request(&dev->queue); + spin_unlock_irqrestore(&dev->lock, flags); + if (!async_req) { + dev_err(dev->dev, "async_req is NULL !!\n"); + return; + } + if (backlog) { + backlog->complete(backlog, -EINPROGRESS); + backlog = NULL; + } + + if (crypto_tfm_alg_type(async_req->tfm) == CRYPTO_ALG_TYPE_ABLKCIPHER) + dev->ablk_req = ablkcipher_request_cast(async_req); + err = dev->start(dev); + if (err) + dev->complete(dev, err); +} + +static struct rk_crypto_tmp *rk_cipher_algs[] = { + &rk_ecb_aes_alg, + &rk_cbc_aes_alg, + &rk_ecb_des_alg, + &rk_cbc_des_alg, + &rk_ecb_des3_ede_alg, + &rk_cbc_des3_ede_alg, +}; + +static int rk_crypto_register(struct rk_crypto_info *crypto_info) +{ + unsigned int i, k; + int err = 0; + + for (i = 0; i < ARRAY_SIZE(rk_cipher_algs); i++) { + rk_cipher_algs[i]->dev = crypto_info; + err = crypto_register_alg(&rk_cipher_algs[i]->alg); + if (err) + goto err_cipher_algs; + } + return 0; + +err_cipher_algs: + for (k = 0; k < i; k++) + crypto_unregister_alg(&rk_cipher_algs[k]->alg); + return err; +} + +static void rk_crypto_unregister(void) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(rk_cipher_algs); i++) + crypto_unregister_alg(&rk_cipher_algs[i]->alg); +} + +static void rk_crypto_action(void *data) +{ + struct rk_crypto_info *crypto_info = data; + + reset_control_assert(crypto_info->rst); +} + +static const struct of_device_id crypto_of_id_table[] = { + { .compatible = "rockchip,rk3288-crypto" }, + {} +}; +MODULE_DEVICE_TABLE(of, crypto_of_id_table); + +static int rk_crypto_probe(struct platform_device *pdev) +{ + struct resource *res; + struct device *dev = &pdev->dev; + struct rk_crypto_info *crypto_info; + int err = 0; + + crypto_info = devm_kzalloc(&pdev->dev, + sizeof(*crypto_info), GFP_KERNEL); + if (!crypto_info) { + err = -ENOMEM; + goto err_crypto; + } + + crypto_info->rst = devm_reset_control_get(dev, "crypto-rst"); + if (IS_ERR(crypto_info->rst)) { + err = PTR_ERR(crypto_info->rst); + goto err_crypto; + } + + reset_control_assert(crypto_info->rst); + usleep_range(10, 20); + reset_control_deassert(crypto_info->rst); + + err = devm_add_action(dev, rk_crypto_action, crypto_info); + if (err) { + reset_control_assert(crypto_info->rst); + goto err_crypto; + } + + spin_lock_init(&crypto_info->lock); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + crypto_info->reg = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(crypto_info->reg)) { + err = PTR_ERR(crypto_info->reg); + goto err_crypto; + } + + crypto_info->aclk = devm_clk_get(&pdev->dev, "aclk"); + if (IS_ERR(crypto_info->aclk)) { + err = PTR_ERR(crypto_info->aclk); + goto err_crypto; + } + + crypto_info->hclk = devm_clk_get(&pdev->dev, "hclk"); + if (IS_ERR(crypto_info->hclk)) { + err = PTR_ERR(crypto_info->hclk); + goto err_crypto; + } + + crypto_info->sclk = devm_clk_get(&pdev->dev, "sclk"); + if (IS_ERR(crypto_info->sclk)) { + err = PTR_ERR(crypto_info->sclk); + goto err_crypto; + } + + crypto_info->dmaclk = devm_clk_get(&pdev->dev, "apb_pclk"); + if (IS_ERR(crypto_info->dmaclk)) { + err = PTR_ERR(crypto_info->dmaclk); + goto err_crypto; + } + + crypto_info->irq = platform_get_irq(pdev, 0); + if (crypto_info->irq < 0) { + dev_warn(crypto_info->dev, + "control Interrupt is not available.\n"); + err = crypto_info->irq; + goto err_crypto; + } + + err = devm_request_irq(&pdev->dev, crypto_info->irq, + rk_crypto_irq_handle, IRQF_SHARED, + "rk-crypto", pdev); + + if (err) { + dev_err(crypto_info->dev, "irq request failed.\n"); + goto err_crypto; + } + + crypto_info->dev = &pdev->dev; + platform_set_drvdata(pdev, crypto_info); + + tasklet_init(&crypto_info->crypto_tasklet, + rk_crypto_tasklet_cb, (unsigned long)crypto_info); + crypto_init_queue(&crypto_info->queue, 50); + + crypto_info->enable_clk = rk_crypto_enable_clk; + crypto_info->disable_clk = rk_crypto_disable_clk; + crypto_info->load_data = rk_load_data; + crypto_info->unload_data = rk_unload_data; + + err = rk_crypto_register(crypto_info); + if (err) { + dev_err(dev, "err in register alg"); + goto err_register_alg; + } + + dev_info(dev, "Crypto Accelerator successfully registered\n"); + return 0; + +err_register_alg: + tasklet_kill(&crypto_info->crypto_tasklet); +err_crypto: + return err; +} + +static int rk_crypto_remove(struct platform_device *pdev) +{ + struct rk_crypto_info *crypto_tmp = platform_get_drvdata(pdev); + + rk_crypto_unregister(); + tasklet_kill(&crypto_tmp->crypto_tasklet); + return 0; +} + +static struct platform_driver crypto_driver = { + .probe = rk_crypto_probe, + .remove = rk_crypto_remove, + .driver = { + .name = "rk3288-crypto", + .of_match_table = crypto_of_id_table, + }, +}; + +module_platform_driver(crypto_driver); + +MODULE_AUTHOR("Zain Wang <zain.wang@rock-chips.com>"); +MODULE_DESCRIPTION("Support for Rockchip's cryptographic engine"); +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/rockchip/rk3288_crypto.h b/drivers/crypto/rockchip/rk3288_crypto.h new file mode 100644 index 000000000000..e499c2c6c903 --- /dev/null +++ b/drivers/crypto/rockchip/rk3288_crypto.h @@ -0,0 +1,216 @@ +#ifndef __RK3288_CRYPTO_H__ +#define __RK3288_CRYPTO_H__ + +#include <crypto/aes.h> +#include <crypto/des.h> +#include <crypto/algapi.h> +#include <linux/interrupt.h> +#include <linux/delay.h> + +#define _SBF(v, f) ((v) << (f)) + +/* Crypto control registers*/ +#define RK_CRYPTO_INTSTS 0x0000 +#define RK_CRYPTO_PKA_DONE_INT BIT(5) +#define RK_CRYPTO_HASH_DONE_INT BIT(4) +#define RK_CRYPTO_HRDMA_ERR_INT BIT(3) +#define RK_CRYPTO_HRDMA_DONE_INT BIT(2) +#define RK_CRYPTO_BCDMA_ERR_INT BIT(1) +#define RK_CRYPTO_BCDMA_DONE_INT BIT(0) + +#define RK_CRYPTO_INTENA 0x0004 +#define RK_CRYPTO_PKA_DONE_ENA BIT(5) +#define RK_CRYPTO_HASH_DONE_ENA BIT(4) +#define RK_CRYPTO_HRDMA_ERR_ENA BIT(3) +#define RK_CRYPTO_HRDMA_DONE_ENA BIT(2) +#define RK_CRYPTO_BCDMA_ERR_ENA BIT(1) +#define RK_CRYPTO_BCDMA_DONE_ENA BIT(0) + +#define RK_CRYPTO_CTRL 0x0008 +#define RK_CRYPTO_WRITE_MASK _SBF(0xFFFF, 16) +#define RK_CRYPTO_TRNG_FLUSH BIT(9) +#define RK_CRYPTO_TRNG_START BIT(8) +#define RK_CRYPTO_PKA_FLUSH BIT(7) +#define RK_CRYPTO_HASH_FLUSH BIT(6) +#define RK_CRYPTO_BLOCK_FLUSH BIT(5) +#define RK_CRYPTO_PKA_START BIT(4) +#define RK_CRYPTO_HASH_START BIT(3) +#define RK_CRYPTO_BLOCK_START BIT(2) +#define RK_CRYPTO_TDES_START BIT(1) +#define RK_CRYPTO_AES_START BIT(0) + +#define RK_CRYPTO_CONF 0x000c +/* HASH Receive DMA Address Mode: fix | increment */ +#define RK_CRYPTO_HR_ADDR_MODE BIT(8) +/* Block Transmit DMA Address Mode: fix | increment */ +#define RK_CRYPTO_BT_ADDR_MODE BIT(7) +/* Block Receive DMA Address Mode: fix | increment */ +#define RK_CRYPTO_BR_ADDR_MODE BIT(6) +#define RK_CRYPTO_BYTESWAP_HRFIFO BIT(5) +#define RK_CRYPTO_BYTESWAP_BTFIFO BIT(4) +#define RK_CRYPTO_BYTESWAP_BRFIFO BIT(3) +/* AES = 0 OR DES = 1 */ +#define RK_CRYPTO_DESSEL BIT(2) +#define RK_CYYPTO_HASHINSEL_INDEPENDENT_SOURCE _SBF(0x00, 0) +#define RK_CYYPTO_HASHINSEL_BLOCK_CIPHER_INPUT _SBF(0x01, 0) +#define RK_CYYPTO_HASHINSEL_BLOCK_CIPHER_OUTPUT _SBF(0x02, 0) + +/* Block Receiving DMA Start Address Register */ +#define RK_CRYPTO_BRDMAS 0x0010 +/* Block Transmitting DMA Start Address Register */ +#define RK_CRYPTO_BTDMAS 0x0014 +/* Block Receiving DMA Length Register */ +#define RK_CRYPTO_BRDMAL 0x0018 +/* Hash Receiving DMA Start Address Register */ +#define RK_CRYPTO_HRDMAS 0x001c +/* Hash Receiving DMA Length Register */ +#define RK_CRYPTO_HRDMAL 0x0020 + +/* AES registers */ +#define RK_CRYPTO_AES_CTRL 0x0080 +#define RK_CRYPTO_AES_BYTESWAP_CNT BIT(11) +#define RK_CRYPTO_AES_BYTESWAP_KEY BIT(10) +#define RK_CRYPTO_AES_BYTESWAP_IV BIT(9) +#define RK_CRYPTO_AES_BYTESWAP_DO BIT(8) +#define RK_CRYPTO_AES_BYTESWAP_DI BIT(7) +#define RK_CRYPTO_AES_KEY_CHANGE BIT(6) +#define RK_CRYPTO_AES_ECB_MODE _SBF(0x00, 4) +#define RK_CRYPTO_AES_CBC_MODE _SBF(0x01, 4) +#define RK_CRYPTO_AES_CTR_MODE _SBF(0x02, 4) +#define RK_CRYPTO_AES_128BIT_key _SBF(0x00, 2) +#define RK_CRYPTO_AES_192BIT_key _SBF(0x01, 2) +#define RK_CRYPTO_AES_256BIT_key _SBF(0x02, 2) +/* Slave = 0 / fifo = 1 */ +#define RK_CRYPTO_AES_FIFO_MODE BIT(1) +/* Encryption = 0 , Decryption = 1 */ +#define RK_CRYPTO_AES_DEC BIT(0) + +#define RK_CRYPTO_AES_STS 0x0084 +#define RK_CRYPTO_AES_DONE BIT(0) + +/* AES Input Data 0-3 Register */ +#define RK_CRYPTO_AES_DIN_0 0x0088 +#define RK_CRYPTO_AES_DIN_1 0x008c +#define RK_CRYPTO_AES_DIN_2 0x0090 +#define RK_CRYPTO_AES_DIN_3 0x0094 + +/* AES output Data 0-3 Register */ +#define RK_CRYPTO_AES_DOUT_0 0x0098 +#define RK_CRYPTO_AES_DOUT_1 0x009c +#define RK_CRYPTO_AES_DOUT_2 0x00a0 +#define RK_CRYPTO_AES_DOUT_3 0x00a4 + +/* AES IV Data 0-3 Register */ +#define RK_CRYPTO_AES_IV_0 0x00a8 +#define RK_CRYPTO_AES_IV_1 0x00ac +#define RK_CRYPTO_AES_IV_2 0x00b0 +#define RK_CRYPTO_AES_IV_3 0x00b4 + +/* AES Key Data 0-3 Register */ +#define RK_CRYPTO_AES_KEY_0 0x00b8 +#define RK_CRYPTO_AES_KEY_1 0x00bc +#define RK_CRYPTO_AES_KEY_2 0x00c0 +#define RK_CRYPTO_AES_KEY_3 0x00c4 +#define RK_CRYPTO_AES_KEY_4 0x00c8 +#define RK_CRYPTO_AES_KEY_5 0x00cc +#define RK_CRYPTO_AES_KEY_6 0x00d0 +#define RK_CRYPTO_AES_KEY_7 0x00d4 + +/* des/tdes */ +#define RK_CRYPTO_TDES_CTRL 0x0100 +#define RK_CRYPTO_TDES_BYTESWAP_KEY BIT(8) +#define RK_CRYPTO_TDES_BYTESWAP_IV BIT(7) +#define RK_CRYPTO_TDES_BYTESWAP_DO BIT(6) +#define RK_CRYPTO_TDES_BYTESWAP_DI BIT(5) +/* 0: ECB, 1: CBC */ +#define RK_CRYPTO_TDES_CHAINMODE_CBC BIT(4) +/* TDES Key Mode, 0 : EDE, 1 : EEE */ +#define RK_CRYPTO_TDES_EEE BIT(3) +/* 0: DES, 1:TDES */ +#define RK_CRYPTO_TDES_SELECT BIT(2) +/* 0: Slave, 1:Fifo */ +#define RK_CRYPTO_TDES_FIFO_MODE BIT(1) +/* Encryption = 0 , Decryption = 1 */ +#define RK_CRYPTO_TDES_DEC BIT(0) + +#define RK_CRYPTO_TDES_STS 0x0104 +#define RK_CRYPTO_TDES_DONE BIT(0) + +#define RK_CRYPTO_TDES_DIN_0 0x0108 +#define RK_CRYPTO_TDES_DIN_1 0x010c +#define RK_CRYPTO_TDES_DOUT_0 0x0110 +#define RK_CRYPTO_TDES_DOUT_1 0x0114 +#define RK_CRYPTO_TDES_IV_0 0x0118 +#define RK_CRYPTO_TDES_IV_1 0x011c +#define RK_CRYPTO_TDES_KEY1_0 0x0120 +#define RK_CRYPTO_TDES_KEY1_1 0x0124 +#define RK_CRYPTO_TDES_KEY2_0 0x0128 +#define RK_CRYPTO_TDES_KEY2_1 0x012c +#define RK_CRYPTO_TDES_KEY3_0 0x0130 +#define RK_CRYPTO_TDES_KEY3_1 0x0134 + +#define CRYPTO_READ(dev, offset) \ + readl_relaxed(((dev)->reg + (offset))) +#define CRYPTO_WRITE(dev, offset, val) \ + writel_relaxed((val), ((dev)->reg + (offset))) + +struct rk_crypto_info { + struct device *dev; + struct clk *aclk; + struct clk *hclk; + struct clk *sclk; + struct clk *dmaclk; + struct reset_control *rst; + void __iomem *reg; + int irq; + struct crypto_queue queue; + struct tasklet_struct crypto_tasklet; + struct ablkcipher_request *ablk_req; + /* device lock */ + spinlock_t lock; + + /* the public variable */ + struct scatterlist *sg_src; + struct scatterlist *sg_dst; + struct scatterlist sg_tmp; + struct scatterlist *first; + unsigned int left_bytes; + void *addr_vir; + int aligned; + int align_size; + size_t nents; + unsigned int total; + unsigned int count; + u32 mode; + dma_addr_t addr_in; + dma_addr_t addr_out; + int (*start)(struct rk_crypto_info *dev); + int (*update)(struct rk_crypto_info *dev); + void (*complete)(struct rk_crypto_info *dev, int err); + int (*enable_clk)(struct rk_crypto_info *dev); + void (*disable_clk)(struct rk_crypto_info *dev); + int (*load_data)(struct rk_crypto_info *dev, + struct scatterlist *sg_src, + struct scatterlist *sg_dst); + void (*unload_data)(struct rk_crypto_info *dev); +}; + +/* the private variable of cipher */ +struct rk_cipher_ctx { + struct rk_crypto_info *dev; + unsigned int keylen; +}; + +struct rk_crypto_tmp { + struct rk_crypto_info *dev; + struct crypto_alg alg; +}; + +extern struct rk_crypto_tmp rk_ecb_aes_alg; +extern struct rk_crypto_tmp rk_cbc_aes_alg; +extern struct rk_crypto_tmp rk_ecb_des_alg; +extern struct rk_crypto_tmp rk_cbc_des_alg; +extern struct rk_crypto_tmp rk_ecb_des3_ede_alg; +extern struct rk_crypto_tmp rk_cbc_des3_ede_alg; + +#endif diff --git a/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c b/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c new file mode 100644 index 000000000000..d98b681f6c06 --- /dev/null +++ b/drivers/crypto/rockchip/rk3288_crypto_ablkcipher.c @@ -0,0 +1,505 @@ +/* + * Crypto acceleration support for Rockchip RK3288 + * + * Copyright (c) 2015, Fuzhou Rockchip Electronics Co., Ltd + * + * Author: Zain Wang <zain.wang@rock-chips.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * Some ideas are from marvell-cesa.c and s5p-sss.c driver. + */ +#include "rk3288_crypto.h" + +#define RK_CRYPTO_DEC BIT(0) + +static void rk_crypto_complete(struct rk_crypto_info *dev, int err) +{ + if (dev->ablk_req->base.complete) + dev->ablk_req->base.complete(&dev->ablk_req->base, err); +} + +static int rk_handle_req(struct rk_crypto_info *dev, + struct ablkcipher_request *req) +{ + unsigned long flags; + int err; + + if (!IS_ALIGNED(req->nbytes, dev->align_size)) + return -EINVAL; + + dev->left_bytes = req->nbytes; + dev->total = req->nbytes; + dev->sg_src = req->src; + dev->first = req->src; + dev->nents = sg_nents(req->src); + dev->sg_dst = req->dst; + dev->aligned = 1; + dev->ablk_req = req; + + spin_lock_irqsave(&dev->lock, flags); + err = ablkcipher_enqueue_request(&dev->queue, req); + spin_unlock_irqrestore(&dev->lock, flags); + tasklet_schedule(&dev->crypto_tasklet); + return err; +} + +static int rk_aes_setkey(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); + struct rk_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && + keylen != AES_KEYSIZE_256) { + crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + ctx->keylen = keylen; + memcpy_toio(ctx->dev->reg + RK_CRYPTO_AES_KEY_0, key, keylen); + return 0; +} + +static int rk_tdes_setkey(struct crypto_ablkcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); + struct rk_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + u32 tmp[DES_EXPKEY_WORDS]; + + if (keylen != DES_KEY_SIZE && keylen != DES3_EDE_KEY_SIZE) { + crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + if (keylen == DES_KEY_SIZE) { + if (!des_ekey(tmp, key) && + (tfm->crt_flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + tfm->crt_flags |= CRYPTO_TFM_RES_WEAK_KEY; + return -EINVAL; + } + } + + ctx->keylen = keylen; + memcpy_toio(ctx->dev->reg + RK_CRYPTO_TDES_KEY1_0, key, keylen); + return 0; +} + +static int rk_aes_ecb_encrypt(struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct rk_crypto_info *dev = ctx->dev; + + dev->mode = RK_CRYPTO_AES_ECB_MODE; + return rk_handle_req(dev, req); +} + +static int rk_aes_ecb_decrypt(struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct rk_crypto_info *dev = ctx->dev; + + dev->mode = RK_CRYPTO_AES_ECB_MODE | RK_CRYPTO_DEC; + return rk_handle_req(dev, req); +} + +static int rk_aes_cbc_encrypt(struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct rk_crypto_info *dev = ctx->dev; + + dev->mode = RK_CRYPTO_AES_CBC_MODE; + return rk_handle_req(dev, req); +} + +static int rk_aes_cbc_decrypt(struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct rk_crypto_info *dev = ctx->dev; + + dev->mode = RK_CRYPTO_AES_CBC_MODE | RK_CRYPTO_DEC; + return rk_handle_req(dev, req); +} + +static int rk_des_ecb_encrypt(struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct rk_crypto_info *dev = ctx->dev; + + dev->mode = 0; + return rk_handle_req(dev, req); +} + +static int rk_des_ecb_decrypt(struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct rk_crypto_info *dev = ctx->dev; + + dev->mode = RK_CRYPTO_DEC; + return rk_handle_req(dev, req); +} + +static int rk_des_cbc_encrypt(struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct rk_crypto_info *dev = ctx->dev; + + dev->mode = RK_CRYPTO_TDES_CHAINMODE_CBC; + return rk_handle_req(dev, req); +} + +static int rk_des_cbc_decrypt(struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct rk_crypto_info *dev = ctx->dev; + + dev->mode = RK_CRYPTO_TDES_CHAINMODE_CBC | RK_CRYPTO_DEC; + return rk_handle_req(dev, req); +} + +static int rk_des3_ede_ecb_encrypt(struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct rk_crypto_info *dev = ctx->dev; + + dev->mode = RK_CRYPTO_TDES_SELECT; + return rk_handle_req(dev, req); +} + +static int rk_des3_ede_ecb_decrypt(struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct rk_crypto_info *dev = ctx->dev; + + dev->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_DEC; + return rk_handle_req(dev, req); +} + +static int rk_des3_ede_cbc_encrypt(struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct rk_crypto_info *dev = ctx->dev; + + dev->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC; + return rk_handle_req(dev, req); +} + +static int rk_des3_ede_cbc_decrypt(struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct rk_crypto_info *dev = ctx->dev; + + dev->mode = RK_CRYPTO_TDES_SELECT | RK_CRYPTO_TDES_CHAINMODE_CBC | + RK_CRYPTO_DEC; + return rk_handle_req(dev, req); +} + +static void rk_ablk_hw_init(struct rk_crypto_info *dev) +{ + struct crypto_ablkcipher *cipher = + crypto_ablkcipher_reqtfm(dev->ablk_req); + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); + struct rk_cipher_ctx *ctx = crypto_ablkcipher_ctx(cipher); + u32 ivsize, block, conf_reg = 0; + + block = crypto_tfm_alg_blocksize(tfm); + ivsize = crypto_ablkcipher_ivsize(cipher); + + if (block == DES_BLOCK_SIZE) { + dev->mode |= RK_CRYPTO_TDES_FIFO_MODE | + RK_CRYPTO_TDES_BYTESWAP_KEY | + RK_CRYPTO_TDES_BYTESWAP_IV; + CRYPTO_WRITE(dev, RK_CRYPTO_TDES_CTRL, dev->mode); + memcpy_toio(dev->reg + RK_CRYPTO_TDES_IV_0, + dev->ablk_req->info, ivsize); + conf_reg = RK_CRYPTO_DESSEL; + } else { + dev->mode |= RK_CRYPTO_AES_FIFO_MODE | + RK_CRYPTO_AES_KEY_CHANGE | + RK_CRYPTO_AES_BYTESWAP_KEY | + RK_CRYPTO_AES_BYTESWAP_IV; + if (ctx->keylen == AES_KEYSIZE_192) + dev->mode |= RK_CRYPTO_AES_192BIT_key; + else if (ctx->keylen == AES_KEYSIZE_256) + dev->mode |= RK_CRYPTO_AES_256BIT_key; + CRYPTO_WRITE(dev, RK_CRYPTO_AES_CTRL, dev->mode); + memcpy_toio(dev->reg + RK_CRYPTO_AES_IV_0, + dev->ablk_req->info, ivsize); + } + conf_reg |= RK_CRYPTO_BYTESWAP_BTFIFO | + RK_CRYPTO_BYTESWAP_BRFIFO; + CRYPTO_WRITE(dev, RK_CRYPTO_CONF, conf_reg); + CRYPTO_WRITE(dev, RK_CRYPTO_INTENA, + RK_CRYPTO_BCDMA_ERR_ENA | RK_CRYPTO_BCDMA_DONE_ENA); +} + +static void crypto_dma_start(struct rk_crypto_info *dev) +{ + CRYPTO_WRITE(dev, RK_CRYPTO_BRDMAS, dev->addr_in); + CRYPTO_WRITE(dev, RK_CRYPTO_BRDMAL, dev->count / 4); + CRYPTO_WRITE(dev, RK_CRYPTO_BTDMAS, dev->addr_out); + CRYPTO_WRITE(dev, RK_CRYPTO_CTRL, RK_CRYPTO_BLOCK_START | + _SBF(RK_CRYPTO_BLOCK_START, 16)); +} + +static int rk_set_data_start(struct rk_crypto_info *dev) +{ + int err; + + err = dev->load_data(dev, dev->sg_src, dev->sg_dst); + if (!err) + crypto_dma_start(dev); + return err; +} + +static int rk_ablk_start(struct rk_crypto_info *dev) +{ + unsigned long flags; + int err; + + spin_lock_irqsave(&dev->lock, flags); + rk_ablk_hw_init(dev); + err = rk_set_data_start(dev); + spin_unlock_irqrestore(&dev->lock, flags); + return err; +} + +static void rk_iv_copyback(struct rk_crypto_info *dev) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(dev->ablk_req); + u32 ivsize = crypto_ablkcipher_ivsize(tfm); + + if (ivsize == DES_BLOCK_SIZE) + memcpy_fromio(dev->ablk_req->info, + dev->reg + RK_CRYPTO_TDES_IV_0, ivsize); + else if (ivsize == AES_BLOCK_SIZE) + memcpy_fromio(dev->ablk_req->info, + dev->reg + RK_CRYPTO_AES_IV_0, ivsize); +} + +/* return: + * true some err was occurred + * fault no err, continue + */ +static int rk_ablk_rx(struct rk_crypto_info *dev) +{ + int err = 0; + + dev->unload_data(dev); + if (!dev->aligned) { + if (!sg_pcopy_from_buffer(dev->ablk_req->dst, dev->nents, + dev->addr_vir, dev->count, + dev->total - dev->left_bytes - + dev->count)) { + err = -EINVAL; + goto out_rx; + } + } + if (dev->left_bytes) { + if (dev->aligned) { + if (sg_is_last(dev->sg_src)) { + dev_err(dev->dev, "[%s:%d] Lack of data\n", + __func__, __LINE__); + err = -ENOMEM; + goto out_rx; + } + dev->sg_src = sg_next(dev->sg_src); + dev->sg_dst = sg_next(dev->sg_dst); + } + err = rk_set_data_start(dev); + } else { + rk_iv_copyback(dev); + /* here show the calculation is over without any err */ + dev->complete(dev, 0); + } +out_rx: + return err; +} + +static int rk_ablk_cra_init(struct crypto_tfm *tfm) +{ + struct rk_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_alg *alg = tfm->__crt_alg; + struct rk_crypto_tmp *algt; + + algt = container_of(alg, struct rk_crypto_tmp, alg); + + ctx->dev = algt->dev; + ctx->dev->align_size = crypto_tfm_alg_alignmask(tfm) + 1; + ctx->dev->start = rk_ablk_start; + ctx->dev->update = rk_ablk_rx; + ctx->dev->complete = rk_crypto_complete; + ctx->dev->addr_vir = (char *)__get_free_page(GFP_KERNEL); + + return ctx->dev->addr_vir ? ctx->dev->enable_clk(ctx->dev) : -ENOMEM; +} + +static void rk_ablk_cra_exit(struct crypto_tfm *tfm) +{ + struct rk_cipher_ctx *ctx = crypto_tfm_ctx(tfm); + + free_page((unsigned long)ctx->dev->addr_vir); + ctx->dev->disable_clk(ctx->dev); +} + +struct rk_crypto_tmp rk_ecb_aes_alg = { + .alg = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-rk", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct rk_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = rk_ablk_cra_init, + .cra_exit = rk_ablk_cra_exit, + .cra_u.ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = rk_aes_setkey, + .encrypt = rk_aes_ecb_encrypt, + .decrypt = rk_aes_ecb_decrypt, + } + } +}; + +struct rk_crypto_tmp rk_cbc_aes_alg = { + .alg = { + .cra_name = "cbc(aes)", + .cra_driver_name = "cbc-aes-rk", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct rk_cipher_ctx), + .cra_alignmask = 0x0f, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = rk_ablk_cra_init, + .cra_exit = rk_ablk_cra_exit, + .cra_u.ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = rk_aes_setkey, + .encrypt = rk_aes_cbc_encrypt, + .decrypt = rk_aes_cbc_decrypt, + } + } +}; + +struct rk_crypto_tmp rk_ecb_des_alg = { + .alg = { + .cra_name = "ecb(des)", + .cra_driver_name = "ecb-des-rk", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct rk_cipher_ctx), + .cra_alignmask = 0x07, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = rk_ablk_cra_init, + .cra_exit = rk_ablk_cra_exit, + .cra_u.ablkcipher = { + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .setkey = rk_tdes_setkey, + .encrypt = rk_des_ecb_encrypt, + .decrypt = rk_des_ecb_decrypt, + } + } +}; + +struct rk_crypto_tmp rk_cbc_des_alg = { + .alg = { + .cra_name = "cbc(des)", + .cra_driver_name = "cbc-des-rk", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct rk_cipher_ctx), + .cra_alignmask = 0x07, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = rk_ablk_cra_init, + .cra_exit = rk_ablk_cra_exit, + .cra_u.ablkcipher = { + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = rk_tdes_setkey, + .encrypt = rk_des_cbc_encrypt, + .decrypt = rk_des_cbc_decrypt, + } + } +}; + +struct rk_crypto_tmp rk_ecb_des3_ede_alg = { + .alg = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "ecb-des3-ede-rk", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct rk_cipher_ctx), + .cra_alignmask = 0x07, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = rk_ablk_cra_init, + .cra_exit = rk_ablk_cra_exit, + .cra_u.ablkcipher = { + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = rk_tdes_setkey, + .encrypt = rk_des3_ede_ecb_encrypt, + .decrypt = rk_des3_ede_ecb_decrypt, + } + } +}; + +struct rk_crypto_tmp rk_cbc_des3_ede_alg = { + .alg = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "cbc-des3-ede-rk", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC, + .cra_blocksize = DES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct rk_cipher_ctx), + .cra_alignmask = 0x07, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = rk_ablk_cra_init, + .cra_exit = rk_ablk_cra_exit, + .cra_u.ablkcipher = { + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + .setkey = rk_tdes_setkey, + .encrypt = rk_des3_ede_cbc_encrypt, + .decrypt = rk_des3_ede_cbc_decrypt, + } + } +}; diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index f68c24a98277..6c4f91c5e6b3 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -130,18 +130,18 @@ #define SAHARA_REG_IDAR 0x20 struct sahara_hw_desc { - u32 hdr; - u32 len1; - dma_addr_t p1; - u32 len2; - dma_addr_t p2; - dma_addr_t next; + u32 hdr; + u32 len1; + u32 p1; + u32 len2; + u32 p2; + u32 next; }; struct sahara_hw_link { - u32 len; - dma_addr_t p; - dma_addr_t next; + u32 len; + u32 p; + u32 next; }; struct sahara_ctx { @@ -228,9 +228,9 @@ struct sahara_dev { size_t total; struct scatterlist *in_sg; - unsigned int nb_in_sg; + int nb_in_sg; struct scatterlist *out_sg; - unsigned int nb_out_sg; + int nb_out_sg; u32 error; }; @@ -416,8 +416,8 @@ static void sahara_dump_descriptors(struct sahara_dev *dev) return; for (i = 0; i < SAHARA_MAX_HW_DESC; i++) { - dev_dbg(dev->device, "Descriptor (%d) (0x%08x):\n", - i, dev->hw_phys_desc[i]); + dev_dbg(dev->device, "Descriptor (%d) (%pad):\n", + i, &dev->hw_phys_desc[i]); dev_dbg(dev->device, "\thdr = 0x%08x\n", dev->hw_desc[i]->hdr); dev_dbg(dev->device, "\tlen1 = %u\n", dev->hw_desc[i]->len1); dev_dbg(dev->device, "\tp1 = 0x%08x\n", dev->hw_desc[i]->p1); @@ -437,8 +437,8 @@ static void sahara_dump_links(struct sahara_dev *dev) return; for (i = 0; i < SAHARA_MAX_HW_LINK; i++) { - dev_dbg(dev->device, "Link (%d) (0x%08x):\n", - i, dev->hw_phys_link[i]); + dev_dbg(dev->device, "Link (%d) (%pad):\n", + i, &dev->hw_phys_link[i]); dev_dbg(dev->device, "\tlen = %u\n", dev->hw_link[i]->len); dev_dbg(dev->device, "\tp = 0x%08x\n", dev->hw_link[i]->p); dev_dbg(dev->device, "\tnext = 0x%08x\n", @@ -477,7 +477,15 @@ static int sahara_hw_descriptor_create(struct sahara_dev *dev) } dev->nb_in_sg = sg_nents_for_len(dev->in_sg, dev->total); + if (dev->nb_in_sg < 0) { + dev_err(dev->device, "Invalid numbers of src SG.\n"); + return dev->nb_in_sg; + } dev->nb_out_sg = sg_nents_for_len(dev->out_sg, dev->total); + if (dev->nb_out_sg < 0) { + dev_err(dev->device, "Invalid numbers of dst SG.\n"); + return dev->nb_out_sg; + } if ((dev->nb_in_sg + dev->nb_out_sg) > SAHARA_MAX_HW_LINK) { dev_err(dev->device, "not enough hw links (%d)\n", dev->nb_in_sg + dev->nb_out_sg); @@ -793,6 +801,10 @@ static int sahara_sha_hw_links_create(struct sahara_dev *dev, dev->in_sg = rctx->in_sg; dev->nb_in_sg = sg_nents_for_len(dev->in_sg, rctx->total); + if (dev->nb_in_sg < 0) { + dev_err(dev->device, "Invalid numbers of src SG.\n"); + return dev->nb_in_sg; + } if ((dev->nb_in_sg) > SAHARA_MAX_HW_LINK) { dev_err(dev->device, "not enough hw links (%d)\n", dev->nb_in_sg + dev->nb_out_sg); diff --git a/drivers/crypto/sunxi-ss/sun4i-ss-core.c b/drivers/crypto/sunxi-ss/sun4i-ss-core.c index eab6fe227fa0..107cd2a41cae 100644 --- a/drivers/crypto/sunxi-ss/sun4i-ss-core.c +++ b/drivers/crypto/sunxi-ss/sun4i-ss-core.c @@ -39,6 +39,7 @@ static struct sun4i_ss_alg_template ss_algs[] = { .import = sun4i_hash_import_md5, .halg = { .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct md5_state), .base = { .cra_name = "md5", .cra_driver_name = "md5-sun4i-ss", @@ -66,6 +67,7 @@ static struct sun4i_ss_alg_template ss_algs[] = { .import = sun4i_hash_import_sha1, .halg = { .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct sha1_state), .base = { .cra_name = "sha1", .cra_driver_name = "sha1-sun4i-ss", diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index b6f9f42e2985..a0d4a08313ae 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -1216,6 +1216,7 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, struct talitos_private *priv = dev_get_drvdata(dev); bool is_sec1 = has_ftr_sec1(priv); int max_len = is_sec1 ? TALITOS1_MAX_DATA_LEN : TALITOS2_MAX_DATA_LEN; + void *err; if (cryptlen + authsize > max_len) { dev_err(dev, "length exceeds h/w max limit\n"); @@ -1228,14 +1229,29 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, if (!dst || dst == src) { src_nents = sg_nents_for_len(src, assoclen + cryptlen + authsize); + if (src_nents < 0) { + dev_err(dev, "Invalid number of src SG.\n"); + err = ERR_PTR(-EINVAL); + goto error_sg; + } src_nents = (src_nents == 1) ? 0 : src_nents; dst_nents = dst ? src_nents : 0; } else { /* dst && dst != src*/ src_nents = sg_nents_for_len(src, assoclen + cryptlen + (encrypt ? 0 : authsize)); + if (src_nents < 0) { + dev_err(dev, "Invalid number of src SG.\n"); + err = ERR_PTR(-EINVAL); + goto error_sg; + } src_nents = (src_nents == 1) ? 0 : src_nents; dst_nents = sg_nents_for_len(dst, assoclen + cryptlen + (encrypt ? authsize : 0)); + if (dst_nents < 0) { + dev_err(dev, "Invalid number of dst SG.\n"); + err = ERR_PTR(-EINVAL); + goto error_sg; + } dst_nents = (dst_nents == 1) ? 0 : dst_nents; } @@ -1260,11 +1276,9 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, edesc = kmalloc(alloc_len, GFP_DMA | flags); if (!edesc) { - if (iv_dma) - dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); - dev_err(dev, "could not allocate edescriptor\n"); - return ERR_PTR(-ENOMEM); + err = ERR_PTR(-ENOMEM); + goto error_sg; } edesc->src_nents = src_nents; @@ -1277,6 +1291,10 @@ static struct talitos_edesc *talitos_edesc_alloc(struct device *dev, DMA_BIDIRECTIONAL); return edesc; +error_sg: + if (iv_dma) + dma_unmap_single(dev, iv_dma, ivsize, DMA_TO_DEVICE); + return err; } static struct talitos_edesc *aead_edesc_alloc(struct aead_request *areq, u8 *iv, @@ -1830,11 +1848,16 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) unsigned int nbytes_to_hash; unsigned int to_hash_later; unsigned int nsg; + int nents; if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) { /* Buffer up to one whole block */ - sg_copy_to_buffer(areq->src, - sg_nents_for_len(areq->src, nbytes), + nents = sg_nents_for_len(areq->src, nbytes); + if (nents < 0) { + dev_err(ctx->dev, "Invalid number of src SG.\n"); + return nents; + } + sg_copy_to_buffer(areq->src, nents, req_ctx->buf + req_ctx->nbuf, nbytes); req_ctx->nbuf += nbytes; return 0; @@ -1867,7 +1890,11 @@ static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes) req_ctx->psrc = areq->src; if (to_hash_later) { - int nents = sg_nents_for_len(areq->src, nbytes); + nents = sg_nents_for_len(areq->src, nbytes); + if (nents < 0) { + dev_err(ctx->dev, "Invalid number of src SG.\n"); + return nents; + } sg_pcopy_to_buffer(areq->src, nents, req_ctx->bufnext, to_hash_later, @@ -2297,6 +2324,22 @@ static struct talitos_alg_template driver_algs[] = { /* ABLKCIPHER algorithms. */ { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .alg.crypto = { + .cra_name = "ecb(aes)", + .cra_driver_name = "ecb-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC, + .cra_ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_AESU, + }, + { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .alg.crypto = { .cra_name = "cbc(aes)", .cra_driver_name = "cbc-aes-talitos", .cra_blocksize = AES_BLOCK_SIZE, @@ -2314,6 +2357,73 @@ static struct talitos_alg_template driver_algs[] = { }, { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .alg.crypto = { + .cra_name = "ctr(aes)", + .cra_driver_name = "ctr-aes-talitos", + .cra_blocksize = AES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC, + .cra_ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_AESU | + DESC_HDR_MODE0_AESU_CTR, + }, + { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .alg.crypto = { + .cra_name = "ecb(des)", + .cra_driver_name = "ecb-des-talitos", + .cra_blocksize = DES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC, + .cra_ablkcipher = { + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_DEU, + }, + { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .alg.crypto = { + .cra_name = "cbc(des)", + .cra_driver_name = "cbc-des-talitos", + .cra_blocksize = DES_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC, + .cra_ablkcipher = { + .min_keysize = DES_KEY_SIZE, + .max_keysize = DES_KEY_SIZE, + .ivsize = DES_BLOCK_SIZE, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_CBC, + }, + { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .alg.crypto = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "ecb-3des-talitos", + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | + CRYPTO_ALG_ASYNC, + .cra_ablkcipher = { + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + } + }, + .desc_hdr_template = DESC_HDR_TYPE_COMMON_NONSNOOP_NO_AFEU | + DESC_HDR_SEL0_DEU | + DESC_HDR_MODE0_DEU_3DES, + }, + { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, + .alg.crypto = { .cra_name = "cbc(des3_ede)", .cra_driver_name = "cbc-3des-talitos", .cra_blocksize = DES3_EDE_BLOCK_SIZE, diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h index 0090f3211d68..8dd8f40e2771 100644 --- a/drivers/crypto/talitos.h +++ b/drivers/crypto/talitos.h @@ -345,6 +345,7 @@ static inline bool has_ftr_sec1(struct talitos_private *priv) /* primary execution unit mode (MODE0) and derivatives */ #define DESC_HDR_MODE0_ENCRYPT cpu_to_be32(0x00100000) #define DESC_HDR_MODE0_AESU_CBC cpu_to_be32(0x00200000) +#define DESC_HDR_MODE0_AESU_CTR cpu_to_be32(0x00600000) #define DESC_HDR_MODE0_DEU_CBC cpu_to_be32(0x00400000) #define DESC_HDR_MODE0_DEU_3DES cpu_to_be32(0x00200000) #define DESC_HDR_MODE0_MDEU_CONT cpu_to_be32(0x08000000) diff --git a/drivers/crypto/ux500/Kconfig b/drivers/crypto/ux500/Kconfig index 30796441b0a6..0e338bf6dfb7 100644 --- a/drivers/crypto/ux500/Kconfig +++ b/drivers/crypto/ux500/Kconfig @@ -18,6 +18,8 @@ config CRYPTO_DEV_UX500_HASH tristate "UX500 crypto driver for HASH block" depends on CRYPTO_DEV_UX500 select CRYPTO_HASH + select CRYPTO_SHA1 + select CRYPTO_SHA256 help This selects the hash driver for the UX500_HASH hardware. Depends on UX500/STM DMA if running in DMA mode. diff --git a/drivers/crypto/ux500/hash/hash_core.c b/drivers/crypto/ux500/hash/hash_core.c index f47d112041b2..d6fdc583ce5d 100644 --- a/drivers/crypto/ux500/hash/hash_core.c +++ b/drivers/crypto/ux500/hash/hash_core.c @@ -41,22 +41,6 @@ static int hash_mode; module_param(hash_mode, int, 0); MODULE_PARM_DESC(hash_mode, "CPU or DMA mode. CPU = 0 (default), DMA = 1"); -/** - * Pre-calculated empty message digests. - */ -static const u8 zero_message_hash_sha1[SHA1_DIGEST_SIZE] = { - 0xda, 0x39, 0xa3, 0xee, 0x5e, 0x6b, 0x4b, 0x0d, - 0x32, 0x55, 0xbf, 0xef, 0x95, 0x60, 0x18, 0x90, - 0xaf, 0xd8, 0x07, 0x09 -}; - -static const u8 zero_message_hash_sha256[SHA256_DIGEST_SIZE] = { - 0xe3, 0xb0, 0xc4, 0x42, 0x98, 0xfc, 0x1c, 0x14, - 0x9a, 0xfb, 0xf4, 0xc8, 0x99, 0x6f, 0xb9, 0x24, - 0x27, 0xae, 0x41, 0xe4, 0x64, 0x9b, 0x93, 0x4c, - 0xa4, 0x95, 0x99, 0x1b, 0x78, 0x52, 0xb8, 0x55 -}; - /* HMAC-SHA1, no key */ static const u8 zero_message_hmac_sha1[SHA1_DIGEST_SIZE] = { 0xfb, 0xdb, 0x1d, 0x1b, 0x18, 0xaa, 0x6c, 0x08, @@ -242,13 +226,13 @@ static int get_empty_message_digest( if (HASH_OPER_MODE_HASH == ctx->config.oper_mode) { if (HASH_ALGO_SHA1 == ctx->config.algorithm) { - memcpy(zero_hash, &zero_message_hash_sha1[0], + memcpy(zero_hash, &sha1_zero_message_hash[0], SHA1_DIGEST_SIZE); *zero_hash_size = SHA1_DIGEST_SIZE; *zero_digest = true; } else if (HASH_ALGO_SHA256 == ctx->config.algorithm) { - memcpy(zero_hash, &zero_message_hash_sha256[0], + memcpy(zero_hash, &sha256_zero_message_hash[0], SHA256_DIGEST_SIZE); *zero_hash_size = SHA256_DIGEST_SIZE; *zero_digest = true; diff --git a/drivers/crypto/vmx/aes_cbc.c b/drivers/crypto/vmx/aes_cbc.c index 0b8fe2ec5315..78a978613ca8 100644 --- a/drivers/crypto/vmx/aes_cbc.c +++ b/drivers/crypto/vmx/aes_cbc.c @@ -191,7 +191,7 @@ struct crypto_alg p8_aes_cbc_alg = { .cra_init = p8_aes_cbc_init, .cra_exit = p8_aes_cbc_exit, .cra_blkcipher = { - .ivsize = 0, + .ivsize = AES_BLOCK_SIZE, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .setkey = p8_aes_cbc_setkey, diff --git a/drivers/crypto/vmx/aes_ctr.c b/drivers/crypto/vmx/aes_ctr.c index ee1306cd8f59..1febc4f1d9af 100644 --- a/drivers/crypto/vmx/aes_ctr.c +++ b/drivers/crypto/vmx/aes_ctr.c @@ -175,7 +175,7 @@ struct crypto_alg p8_aes_ctr_alg = { .cra_init = p8_aes_ctr_init, .cra_exit = p8_aes_ctr_exit, .cra_blkcipher = { - .ivsize = 0, + .ivsize = AES_BLOCK_SIZE, .min_keysize = AES_MIN_KEY_SIZE, .max_keysize = AES_MAX_KEY_SIZE, .setkey = p8_aes_ctr_setkey, |