diff options
Diffstat (limited to 'drivers/crypto/cavium/nitrox')
| -rw-r--r-- | drivers/crypto/cavium/nitrox/Kconfig | 21 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/Makefile | 8 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_algs.c | 457 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_common.h | 42 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_csr.h | 1084 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_dev.h | 179 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_hal.c | 401 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_isr.c | 467 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_lib.c | 210 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_main.c | 640 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_req.h | 445 | ||||
| -rw-r--r-- | drivers/crypto/cavium/nitrox/nitrox_reqmgr.c | 735 |
12 files changed, 4689 insertions, 0 deletions
diff --git a/drivers/crypto/cavium/nitrox/Kconfig b/drivers/crypto/cavium/nitrox/Kconfig new file mode 100644 index 000000000000..731e6a57218a --- /dev/null +++ b/drivers/crypto/cavium/nitrox/Kconfig @@ -0,0 +1,21 @@ +# +# Cavium NITROX Crypto Device configuration +# +config CRYPTO_DEV_NITROX + tristate + select CRYPTO_BLKCIPHER + select CRYPTO_AES + select CRYPTO_DES + select FW_LOADER + +config CRYPTO_DEV_NITROX_CNN55XX + tristate "Support for Cavium CNN55XX driver" + depends on PCI_MSI && 64BIT + select CRYPTO_DEV_NITROX + default m + help + Support for Cavium NITROX family CNN55XX driver + for accelerating crypto workloads. + + To compile this as a module, choose M here: the module + will be called n5pf. diff --git a/drivers/crypto/cavium/nitrox/Makefile b/drivers/crypto/cavium/nitrox/Makefile new file mode 100644 index 000000000000..5af2e4368267 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/Makefile @@ -0,0 +1,8 @@ +obj-$(CONFIG_CRYPTO_DEV_NITROX_CNN55XX) += n5pf.o + +n5pf-objs := nitrox_main.o \ + nitrox_isr.o \ + nitrox_lib.o \ + nitrox_hal.o \ + nitrox_reqmgr.o \ + nitrox_algs.o diff --git a/drivers/crypto/cavium/nitrox/nitrox_algs.c b/drivers/crypto/cavium/nitrox/nitrox_algs.c new file mode 100644 index 000000000000..ce330278ef8a --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_algs.c @@ -0,0 +1,457 @@ +#include <linux/crypto.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/printk.h> + +#include <crypto/aes.h> +#include <crypto/skcipher.h> +#include <crypto/ctr.h> +#include <crypto/des.h> +#include <crypto/xts.h> + +#include "nitrox_dev.h" +#include "nitrox_common.h" +#include "nitrox_req.h" + +#define PRIO 4001 + +struct nitrox_cipher { + const char *name; + enum flexi_cipher value; +}; + +/** + * supported cipher list + */ +static const struct nitrox_cipher flexi_cipher_table[] = { + { "null", CIPHER_NULL }, + { "cbc(des3_ede)", CIPHER_3DES_CBC }, + { "ecb(des3_ede)", CIPHER_3DES_ECB }, + { "cbc(aes)", CIPHER_AES_CBC }, + { "ecb(aes)", CIPHER_AES_ECB }, + { "cfb(aes)", CIPHER_AES_CFB }, + { "rfc3686(ctr(aes))", CIPHER_AES_CTR }, + { "xts(aes)", CIPHER_AES_XTS }, + { "cts(cbc(aes))", CIPHER_AES_CBC_CTS }, + { NULL, CIPHER_INVALID } +}; + +static enum flexi_cipher flexi_cipher_type(const char *name) +{ + const struct nitrox_cipher *cipher = flexi_cipher_table; + + while (cipher->name) { + if (!strcmp(cipher->name, name)) + break; + cipher++; + } + return cipher->value; +} + +static int flexi_aes_keylen(int keylen) +{ + int aes_keylen; + + switch (keylen) { + case AES_KEYSIZE_128: + aes_keylen = 1; + break; + case AES_KEYSIZE_192: + aes_keylen = 2; + break; + case AES_KEYSIZE_256: + aes_keylen = 3; + break; + default: + aes_keylen = -EINVAL; + break; + } + return aes_keylen; +} + +static int nitrox_skcipher_init(struct crypto_skcipher *tfm) +{ + struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm); + void *fctx; + + /* get the first device */ + nctx->ndev = nitrox_get_first_device(); + if (!nctx->ndev) + return -ENODEV; + + /* allocate nitrox crypto context */ + fctx = crypto_alloc_context(nctx->ndev); + if (!fctx) { + nitrox_put_device(nctx->ndev); + return -ENOMEM; + } + nctx->u.ctx_handle = (uintptr_t)fctx; + crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(tfm) + + sizeof(struct nitrox_kcrypt_request)); + return 0; +} + +static void nitrox_skcipher_exit(struct crypto_skcipher *tfm) +{ + struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(tfm); + + /* free the nitrox crypto context */ + if (nctx->u.ctx_handle) { + struct flexi_crypto_context *fctx = nctx->u.fctx; + + memset(&fctx->crypto, 0, sizeof(struct crypto_keys)); + memset(&fctx->auth, 0, sizeof(struct auth_keys)); + crypto_free_context((void *)fctx); + } + nitrox_put_device(nctx->ndev); + + nctx->u.ctx_handle = 0; + nctx->ndev = NULL; +} + +static inline int nitrox_skcipher_setkey(struct crypto_skcipher *cipher, + int aes_keylen, const u8 *key, + unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); + struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm); + struct flexi_crypto_context *fctx; + enum flexi_cipher cipher_type; + const char *name; + + name = crypto_tfm_alg_name(tfm); + cipher_type = flexi_cipher_type(name); + if (unlikely(cipher_type == CIPHER_INVALID)) { + pr_err("unsupported cipher: %s\n", name); + return -EINVAL; + } + + /* fill crypto context */ + fctx = nctx->u.fctx; + fctx->flags = 0; + fctx->w0.cipher_type = cipher_type; + fctx->w0.aes_keylen = aes_keylen; + fctx->w0.iv_source = IV_FROM_DPTR; + fctx->flags = cpu_to_be64(*(u64 *)&fctx->w0); + /* copy the key to context */ + memcpy(fctx->crypto.u.key, key, keylen); + + return 0; +} + +static int nitrox_aes_setkey(struct crypto_skcipher *cipher, const u8 *key, + unsigned int keylen) +{ + int aes_keylen; + + aes_keylen = flexi_aes_keylen(keylen); + if (aes_keylen < 0) { + crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen); +} + +static void nitrox_skcipher_callback(struct skcipher_request *skreq, + int err) +{ + if (err) { + pr_err_ratelimited("request failed status 0x%0x\n", err); + err = -EINVAL; + } + skcipher_request_complete(skreq, err); +} + +static int nitrox_skcipher_crypt(struct skcipher_request *skreq, bool enc) +{ + struct crypto_skcipher *cipher = crypto_skcipher_reqtfm(skreq); + struct nitrox_crypto_ctx *nctx = crypto_skcipher_ctx(cipher); + struct nitrox_kcrypt_request *nkreq = skcipher_request_ctx(skreq); + int ivsize = crypto_skcipher_ivsize(cipher); + struct se_crypto_request *creq; + + creq = &nkreq->creq; + creq->flags = skreq->base.flags; + creq->gfp = (skreq->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC; + + /* fill the request */ + creq->ctrl.value = 0; + creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC; + creq->ctrl.s.arg = (enc ? ENCRYPT : DECRYPT); + /* param0: length of the data to be encrypted */ + creq->gph.param0 = cpu_to_be16(skreq->cryptlen); + creq->gph.param1 = 0; + /* param2: encryption data offset */ + creq->gph.param2 = cpu_to_be16(ivsize); + creq->gph.param3 = 0; + + creq->ctx_handle = nctx->u.ctx_handle; + creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context); + + /* copy the iv */ + memcpy(creq->iv, skreq->iv, ivsize); + creq->ivsize = ivsize; + creq->src = skreq->src; + creq->dst = skreq->dst; + + nkreq->nctx = nctx; + nkreq->skreq = skreq; + + /* send the crypto request */ + return nitrox_process_se_request(nctx->ndev, creq, + nitrox_skcipher_callback, skreq); +} + +static int nitrox_aes_encrypt(struct skcipher_request *skreq) +{ + return nitrox_skcipher_crypt(skreq, true); +} + +static int nitrox_aes_decrypt(struct skcipher_request *skreq) +{ + return nitrox_skcipher_crypt(skreq, false); +} + +static int nitrox_3des_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + if (keylen != DES3_EDE_KEY_SIZE) { + crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + return nitrox_skcipher_setkey(cipher, 0, key, keylen); +} + +static int nitrox_3des_encrypt(struct skcipher_request *skreq) +{ + return nitrox_skcipher_crypt(skreq, true); +} + +static int nitrox_3des_decrypt(struct skcipher_request *skreq) +{ + return nitrox_skcipher_crypt(skreq, false); +} + +static int nitrox_aes_xts_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); + struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm); + struct flexi_crypto_context *fctx; + int aes_keylen, ret; + + ret = xts_check_key(tfm, key, keylen); + if (ret) + return ret; + + keylen /= 2; + + aes_keylen = flexi_aes_keylen(keylen); + if (aes_keylen < 0) { + crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + fctx = nctx->u.fctx; + /* copy KEY2 */ + memcpy(fctx->auth.u.key2, (key + keylen), keylen); + + return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen); +} + +static int nitrox_aes_ctr_rfc3686_setkey(struct crypto_skcipher *cipher, + const u8 *key, unsigned int keylen) +{ + struct crypto_tfm *tfm = crypto_skcipher_tfm(cipher); + struct nitrox_crypto_ctx *nctx = crypto_tfm_ctx(tfm); + struct flexi_crypto_context *fctx; + int aes_keylen; + + if (keylen < CTR_RFC3686_NONCE_SIZE) + return -EINVAL; + + fctx = nctx->u.fctx; + + memcpy(fctx->crypto.iv, key + (keylen - CTR_RFC3686_NONCE_SIZE), + CTR_RFC3686_NONCE_SIZE); + + keylen -= CTR_RFC3686_NONCE_SIZE; + + aes_keylen = flexi_aes_keylen(keylen); + if (aes_keylen < 0) { + crypto_skcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + return nitrox_skcipher_setkey(cipher, aes_keylen, key, keylen); +} + +static struct skcipher_alg nitrox_skciphers[] = { { + .base = { + .cra_name = "cbc(aes)", + .cra_driver_name = "n5_cbc(aes)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = nitrox_aes_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_aes_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "ecb(aes)", + .cra_driver_name = "n5_ecb(aes)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = nitrox_aes_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_aes_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "cfb(aes)", + .cra_driver_name = "n5_cfb(aes)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = nitrox_aes_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_aes_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "xts(aes)", + .cra_driver_name = "n5_xts(aes)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = 2 * AES_MIN_KEY_SIZE, + .max_keysize = 2 * AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = nitrox_aes_xts_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_aes_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "rfc3686(ctr(aes))", + .cra_driver_name = "n5_rfc3686(ctr(aes))", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .max_keysize = AES_MAX_KEY_SIZE + CTR_RFC3686_NONCE_SIZE, + .ivsize = CTR_RFC3686_IV_SIZE, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, + .setkey = nitrox_aes_ctr_rfc3686_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_aes_decrypt, +}, { + .base = { + .cra_name = "cts(cbc(aes))", + .cra_driver_name = "n5_cts(cbc(aes))", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + }, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + .setkey = nitrox_aes_setkey, + .encrypt = nitrox_aes_encrypt, + .decrypt = nitrox_aes_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "cbc(des3_ede)", + .cra_driver_name = "n5_cbc(des3_ede)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = nitrox_3des_setkey, + .encrypt = nitrox_3des_encrypt, + .decrypt = nitrox_3des_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +}, { + .base = { + .cra_name = "ecb(des3_ede)", + .cra_driver_name = "n5_ecb(des3_ede)", + .cra_priority = PRIO, + .cra_flags = CRYPTO_ALG_ASYNC, + .cra_blocksize = DES3_EDE_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct nitrox_crypto_ctx), + .cra_alignmask = 0, + .cra_module = THIS_MODULE, + }, + .min_keysize = DES3_EDE_KEY_SIZE, + .max_keysize = DES3_EDE_KEY_SIZE, + .ivsize = DES3_EDE_BLOCK_SIZE, + .setkey = nitrox_3des_setkey, + .encrypt = nitrox_3des_encrypt, + .decrypt = nitrox_3des_decrypt, + .init = nitrox_skcipher_init, + .exit = nitrox_skcipher_exit, +} + +}; + +int nitrox_crypto_register(void) +{ + return crypto_register_skciphers(nitrox_skciphers, + ARRAY_SIZE(nitrox_skciphers)); +} + +void nitrox_crypto_unregister(void) +{ + crypto_unregister_skciphers(nitrox_skciphers, + ARRAY_SIZE(nitrox_skciphers)); +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_common.h b/drivers/crypto/cavium/nitrox/nitrox_common.h new file mode 100644 index 000000000000..4888c7823a5f --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_common.h @@ -0,0 +1,42 @@ +#ifndef __NITROX_COMMON_H +#define __NITROX_COMMON_H + +#include "nitrox_dev.h" +#include "nitrox_req.h" + +int nitrox_crypto_register(void); +void nitrox_crypto_unregister(void); +void *crypto_alloc_context(struct nitrox_device *ndev); +void crypto_free_context(void *ctx); +struct nitrox_device *nitrox_get_first_device(void); +void nitrox_put_device(struct nitrox_device *ndev); + +void nitrox_pf_cleanup_isr(struct nitrox_device *ndev); +int nitrox_pf_init_isr(struct nitrox_device *ndev); + +int nitrox_common_sw_init(struct nitrox_device *ndev); +void nitrox_common_sw_cleanup(struct nitrox_device *ndev); + +void pkt_slc_resp_handler(unsigned long data); +int nitrox_process_se_request(struct nitrox_device *ndev, + struct se_crypto_request *req, + completion_t cb, + struct skcipher_request *skreq); +void backlog_qflush_work(struct work_struct *work); + +void nitrox_config_emu_unit(struct nitrox_device *ndev); +void nitrox_config_pkt_input_rings(struct nitrox_device *ndev); +void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev); +void nitrox_config_vfmode(struct nitrox_device *ndev, int mode); +void nitrox_config_nps_unit(struct nitrox_device *ndev); +void nitrox_config_pom_unit(struct nitrox_device *ndev); +void nitrox_config_rand_unit(struct nitrox_device *ndev); +void nitrox_config_efl_unit(struct nitrox_device *ndev); +void nitrox_config_bmi_unit(struct nitrox_device *ndev); +void nitrox_config_bmo_unit(struct nitrox_device *ndev); +void nitrox_config_lbc_unit(struct nitrox_device *ndev); +void invalidate_lbc(struct nitrox_device *ndev); +void enable_pkt_input_ring(struct nitrox_device *ndev, int ring); +void enable_pkt_solicit_port(struct nitrox_device *ndev, int port); + +#endif /* __NITROX_COMMON_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_csr.h b/drivers/crypto/cavium/nitrox/nitrox_csr.h new file mode 100644 index 000000000000..30b04c4c6076 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_csr.h @@ -0,0 +1,1084 @@ +#ifndef __NITROX_CSR_H +#define __NITROX_CSR_H + +#include <asm/byteorder.h> +#include <linux/types.h> + +/* EMU clusters */ +#define NR_CLUSTERS 4 +#define AE_CORES_PER_CLUSTER 20 +#define SE_CORES_PER_CLUSTER 16 + +/* BIST registers */ +#define EMU_BIST_STATUSX(_i) (0x1402700 + ((_i) * 0x40000)) +#define UCD_BIST_STATUS 0x12C0070 +#define NPS_CORE_BIST_REG 0x10000E8 +#define NPS_CORE_NPC_BIST_REG 0x1000128 +#define NPS_PKT_SLC_BIST_REG 0x1040088 +#define NPS_PKT_IN_BIST_REG 0x1040100 +#define POM_BIST_REG 0x11C0100 +#define BMI_BIST_REG 0x1140080 +#define EFL_CORE_BIST_REGX(_i) (0x1240100 + ((_i) * 0x400)) +#define EFL_TOP_BIST_STAT 0x1241090 +#define BMO_BIST_REG 0x1180080 +#define LBC_BIST_STATUS 0x1200020 +#define PEM_BIST_STATUSX(_i) (0x1080468 | ((_i) << 18)) + +/* EMU registers */ +#define EMU_SE_ENABLEX(_i) (0x1400000 + ((_i) * 0x40000)) +#define EMU_AE_ENABLEX(_i) (0x1400008 + ((_i) * 0x40000)) +#define EMU_WD_INT_ENA_W1SX(_i) (0x1402318 + ((_i) * 0x40000)) +#define EMU_GE_INT_ENA_W1SX(_i) (0x1402518 + ((_i) * 0x40000)) +#define EMU_FUSE_MAPX(_i) (0x1402708 + ((_i) * 0x40000)) + +/* UCD registers */ +#define UCD_UCODE_LOAD_BLOCK_NUM 0x12C0010 +#define UCD_UCODE_LOAD_IDX_DATAX(_i) (0x12C0018 + ((_i) * 0x20)) +#define UCD_SE_EID_UCODE_BLOCK_NUMX(_i) (0x12C0000 + ((_i) * 0x1000)) + +/* NPS core registers */ +#define NPS_CORE_GBL_VFCFG 0x1000000 +#define NPS_CORE_CONTROL 0x1000008 +#define NPS_CORE_INT_ACTIVE 0x1000080 +#define NPS_CORE_INT 0x10000A0 +#define NPS_CORE_INT_ENA_W1S 0x10000B8 +#define NPS_STATS_PKT_DMA_RD_CNT 0x1000180 +#define NPS_STATS_PKT_DMA_WR_CNT 0x1000190 + +/* NPS packet registers */ +#define NPS_PKT_INT 0x1040018 +#define NPS_PKT_IN_RERR_HI 0x1040108 +#define NPS_PKT_IN_RERR_HI_ENA_W1S 0x1040120 +#define NPS_PKT_IN_RERR_LO 0x1040128 +#define NPS_PKT_IN_RERR_LO_ENA_W1S 0x1040140 +#define NPS_PKT_IN_ERR_TYPE 0x1040148 +#define NPS_PKT_IN_ERR_TYPE_ENA_W1S 0x1040160 +#define NPS_PKT_IN_INSTR_CTLX(_i) (0x10060 + ((_i) * 0x40000)) +#define NPS_PKT_IN_INSTR_BADDRX(_i) (0x10068 + ((_i) * 0x40000)) +#define NPS_PKT_IN_INSTR_RSIZEX(_i) (0x10070 + ((_i) * 0x40000)) +#define NPS_PKT_IN_DONE_CNTSX(_i) (0x10080 + ((_i) * 0x40000)) +#define NPS_PKT_IN_INSTR_BAOFF_DBELLX(_i) (0x10078 + ((_i) * 0x40000)) +#define NPS_PKT_IN_INT_LEVELSX(_i) (0x10088 + ((_i) * 0x40000)) + +#define NPS_PKT_SLC_RERR_HI 0x1040208 +#define NPS_PKT_SLC_RERR_HI_ENA_W1S 0x1040220 +#define NPS_PKT_SLC_RERR_LO 0x1040228 +#define NPS_PKT_SLC_RERR_LO_ENA_W1S 0x1040240 +#define NPS_PKT_SLC_ERR_TYPE 0x1040248 +#define NPS_PKT_SLC_ERR_TYPE_ENA_W1S 0x1040260 +#define NPS_PKT_SLC_CTLX(_i) (0x10000 + ((_i) * 0x40000)) +#define NPS_PKT_SLC_CNTSX(_i) (0x10008 + ((_i) * 0x40000)) +#define NPS_PKT_SLC_INT_LEVELSX(_i) (0x10010 + ((_i) * 0x40000)) + +/* POM registers */ +#define POM_INT_ENA_W1S 0x11C0018 +#define POM_GRP_EXECMASKX(_i) (0x11C1100 | ((_i) * 8)) +#define POM_INT 0x11C0000 +#define POM_PERF_CTL 0x11CC400 + +/* BMI registers */ +#define BMI_INT 0x1140000 +#define BMI_CTL 0x1140020 +#define BMI_INT_ENA_W1S 0x1140018 +#define BMI_NPS_PKT_CNT 0x1140070 + +/* EFL registers */ +#define EFL_CORE_INT_ENA_W1SX(_i) (0x1240018 + ((_i) * 0x400)) +#define EFL_CORE_VF_ERR_INT0X(_i) (0x1240050 + ((_i) * 0x400)) +#define EFL_CORE_VF_ERR_INT0_ENA_W1SX(_i) (0x1240068 + ((_i) * 0x400)) +#define EFL_CORE_VF_ERR_INT1X(_i) (0x1240070 + ((_i) * 0x400)) +#define EFL_CORE_VF_ERR_INT1_ENA_W1SX(_i) (0x1240088 + ((_i) * 0x400)) +#define EFL_CORE_SE_ERR_INTX(_i) (0x12400A0 + ((_i) * 0x400)) +#define EFL_RNM_CTL_STATUS 0x1241800 +#define EFL_CORE_INTX(_i) (0x1240000 + ((_i) * 0x400)) + +/* BMO registers */ +#define BMO_CTL2 0x1180028 +#define BMO_NPS_SLC_PKT_CNT 0x1180078 + +/* LBC registers */ +#define LBC_INT 0x1200000 +#define LBC_INVAL_CTL 0x1201010 +#define LBC_PLM_VF1_64_INT 0x1202008 +#define LBC_INVAL_STATUS 0x1202010 +#define LBC_INT_ENA_W1S 0x1203000 +#define LBC_PLM_VF1_64_INT_ENA_W1S 0x1205008 +#define LBC_PLM_VF65_128_INT 0x1206008 +#define LBC_ELM_VF1_64_INT 0x1208000 +#define LBC_PLM_VF65_128_INT_ENA_W1S 0x1209008 +#define LBC_ELM_VF1_64_INT_ENA_W1S 0x120B000 +#define LBC_ELM_VF65_128_INT 0x120C000 +#define LBC_ELM_VF65_128_INT_ENA_W1S 0x120F000 + +/* PEM registers */ +#define PEM0_INT 0x1080428 + +/** + * struct emu_fuse_map - EMU Fuse Map Registers + * @ae_fuse: Fuse settings for AE 19..0 + * @se_fuse: Fuse settings for SE 15..0 + * + * A set bit indicates the unit is fuse disabled. + */ +union emu_fuse_map { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 valid : 1; + u64 raz_52_62 : 11; + u64 ae_fuse : 20; + u64 raz_16_31 : 16; + u64 se_fuse : 16; +#else + u64 se_fuse : 16; + u64 raz_16_31 : 16; + u64 ae_fuse : 20; + u64 raz_52_62 : 11; + u64 valid : 1; +#endif + } s; +}; + +/** + * struct emu_se_enable - Symmetric Engine Enable Registers + * @enable: Individual enables for each of the clusters + * 16 symmetric engines. + */ +union emu_se_enable { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 48; + u64 enable : 16; +#else + u64 enable : 16; + u64 raz : 48; +#endif + } s; +}; + +/** + * struct emu_ae_enable - EMU Asymmetric engines. + * @enable: Individual enables for each of the cluster's + * 20 Asymmetric Engines. + */ +union emu_ae_enable { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 44; + u64 enable : 20; +#else + u64 enable : 20; + u64 raz : 44; +#endif + } s; +}; + +/** + * struct emu_wd_int_ena_w1s - EMU Interrupt Enable Registers + * @ae_wd: Reads or sets enable for EMU(0..3)_WD_INT[AE_WD] + * @se_wd: Reads or sets enable for EMU(0..3)_WD_INT[SE_WD] + */ +union emu_wd_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz2 : 12; + u64 ae_wd : 20; + u64 raz1 : 16; + u64 se_wd : 16; +#else + u64 se_wd : 16; + u64 raz1 : 16; + u64 ae_wd : 20; + u64 raz2 : 12; +#endif + } s; +}; + +/** + * struct emu_ge_int_ena_w1s - EMU Interrupt Enable set registers + * @ae_ge: Reads or sets enable for EMU(0..3)_GE_INT[AE_GE] + * @se_ge: Reads or sets enable for EMU(0..3)_GE_INT[SE_GE] + */ +union emu_ge_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_52_63 : 12; + u64 ae_ge : 20; + u64 raz_16_31: 16; + u64 se_ge : 16; +#else + u64 se_ge : 16; + u64 raz_16_31: 16; + u64 ae_ge : 20; + u64 raz_52_63 : 12; +#endif + } s; +}; + +/** + * struct nps_pkt_slc_ctl - Solicited Packet Out Control Registers + * @rh: Indicates whether to remove or include the response header + * 1 = Include, 0 = Remove + * @z: If set, 8 trailing 0x00 bytes will be added to the end of the + * outgoing packet. + * @enb: Enable for this port. + */ +union nps_pkt_slc_ctl { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 raz : 61; + u64 rh : 1; + u64 z : 1; + u64 enb : 1; +#else + u64 enb : 1; + u64 z : 1; + u64 rh : 1; + u64 raz : 61; +#endif + } s; +}; + +/** + * struct nps_pkt_slc_cnts - Solicited Packet Out Count Registers + * @slc_int: Returns a 1 when: + * NPS_PKT_SLC(i)_CNTS[CNT] > NPS_PKT_SLC(i)_INT_LEVELS[CNT], or + * NPS_PKT_SLC(i)_CNTS[TIMER] > NPS_PKT_SLC(i)_INT_LEVELS[TIMET]. + * To clear the bit, the CNTS register must be written to clear. + * @in_int: Returns a 1 when: + * NPS_PKT_IN(i)_DONE_CNTS[CNT] > NPS_PKT_IN(i)_INT_LEVELS[CNT]. + * To clear the bit, the DONE_CNTS register must be written to clear. + * @mbox_int: Returns a 1 when: + * NPS_PKT_MBOX_PF_VF(i)_INT[INTR] is set. To clear the bit, + * write NPS_PKT_MBOX_PF_VF(i)_INT[INTR] with 1. + * @timer: Timer, incremented every 2048 coprocessor clock cycles + * when [CNT] is not zero. The hardware clears both [TIMER] and + * [INT] when [CNT] goes to 0. + * @cnt: Packet counter. Hardware adds to [CNT] as it sends packets out. + * On a write to this CSR, hardware subtracts the amount written to the + * [CNT] field from [CNT]. + */ +union nps_pkt_slc_cnts { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 slc_int : 1; + u64 uns_int : 1; + u64 in_int : 1; + u64 mbox_int : 1; + u64 resend : 1; + u64 raz : 5; + u64 timer : 22; + u64 cnt : 32; +#else + u64 cnt : 32; + u64 timer : 22; + u64 raz : 5; + u64 resend : 1; + u64 mbox_int : 1; + u64 in_int : 1; + u64 uns_int : 1; + u64 slc_int : 1; +#endif + } s; +}; + +/** + * struct nps_pkt_slc_int_levels - Solicited Packet Out Interrupt Levels + * Registers. + * @bmode: Determines whether NPS_PKT_SLC_CNTS[CNT] is a byte or + * packet counter. + * @timet: Output port counter time interrupt threshold. + * @cnt: Output port counter interrupt threshold. + */ +union nps_pkt_slc_int_levels { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 bmode : 1; + u64 raz : 9; + u64 timet : 22; + u64 cnt : 32; +#else + u64 cnt : 32; + u64 timet : 22; + u64 raz : 9; + u64 bmode : 1; +#endif + } s; +}; + +/** + * struct nps_pkt_inst - NPS Packet Interrupt Register + * @in_err: Set when any NPS_PKT_IN_RERR_HI/LO bit and + * corresponding NPS_PKT_IN_RERR_*_ENA_* bit are bot set. + * @uns_err: Set when any NSP_PKT_UNS_RERR_HI/LO bit and + * corresponding NPS_PKT_UNS_RERR_*_ENA_* bit are both set. + * @slc_er: Set when any NSP_PKT_SLC_RERR_HI/LO bit and + * corresponding NPS_PKT_SLC_RERR_*_ENA_* bit are both set. + */ +union nps_pkt_int { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 raz : 54; + u64 uns_wto : 1; + u64 in_err : 1; + u64 uns_err : 1; + u64 slc_err : 1; + u64 in_dbe : 1; + u64 in_sbe : 1; + u64 uns_dbe : 1; + u64 uns_sbe : 1; + u64 slc_dbe : 1; + u64 slc_sbe : 1; +#else + u64 slc_sbe : 1; + u64 slc_dbe : 1; + u64 uns_sbe : 1; + u64 uns_dbe : 1; + u64 in_sbe : 1; + u64 in_dbe : 1; + u64 slc_err : 1; + u64 uns_err : 1; + u64 in_err : 1; + u64 uns_wto : 1; + u64 raz : 54; +#endif + } s; +}; + +/** + * struct nps_pkt_in_done_cnts - Input instruction ring counts registers + * @slc_cnt: Returns a 1 when: + * NPS_PKT_SLC(i)_CNTS[CNT] > NPS_PKT_SLC(i)_INT_LEVELS[CNT], or + * NPS_PKT_SLC(i)_CNTS[TIMER] > NPS_PKT_SCL(i)_INT_LEVELS[TIMET] + * To clear the bit, the CNTS register must be + * written to clear the underlying condition + * @uns_int: Return a 1 when: + * NPS_PKT_UNS(i)_CNTS[CNT] > NPS_PKT_UNS(i)_INT_LEVELS[CNT], or + * NPS_PKT_UNS(i)_CNTS[TIMER] > NPS_PKT_UNS(i)_INT_LEVELS[TIMET] + * To clear the bit, the CNTS register must be + * written to clear the underlying condition + * @in_int: Returns a 1 when: + * NPS_PKT_IN(i)_DONE_CNTS[CNT] > NPS_PKT_IN(i)_INT_LEVELS[CNT] + * To clear the bit, the DONE_CNTS register + * must be written to clear the underlying condition + * @mbox_int: Returns a 1 when: + * NPS_PKT_MBOX_PF_VF(i)_INT[INTR] is set. + * To clear the bit, write NPS_PKT_MBOX_PF_VF(i)_INT[INTR] + * with 1. + * @resend: A write of 1 will resend an MSI-X interrupt message if any + * of the following conditions are true for this ring "i". + * NPS_PKT_SLC(i)_CNTS[CNT] > NPS_PKT_SLC(i)_INT_LEVELS[CNT] + * NPS_PKT_SLC(i)_CNTS[TIMER] > NPS_PKT_SLC(i)_INT_LEVELS[TIMET] + * NPS_PKT_UNS(i)_CNTS[CNT] > NPS_PKT_UNS(i)_INT_LEVELS[CNT] + * NPS_PKT_UNS(i)_CNTS[TIMER] > NPS_PKT_UNS(i)_INT_LEVELS[TIMET] + * NPS_PKT_IN(i)_DONE_CNTS[CNT] > NPS_PKT_IN(i)_INT_LEVELS[CNT] + * NPS_PKT_MBOX_PF_VF(i)_INT[INTR] is set + * @cnt: Packet counter. Hardware adds to [CNT] as it reads + * packets. On a write to this CSR, hardware substracts the + * amount written to the [CNT] field from [CNT], which will + * clear PKT_IN(i)_INT_STATUS[INTR] if [CNT] becomes <= + * NPS_PKT_IN(i)_INT_LEVELS[CNT]. This register should be + * cleared before enabling a ring by reading the current + * value and writing it back. + */ +union nps_pkt_in_done_cnts { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 slc_int : 1; + u64 uns_int : 1; + u64 in_int : 1; + u64 mbox_int : 1; + u64 resend : 1; + u64 raz : 27; + u64 cnt : 32; +#else + u64 cnt : 32; + u64 raz : 27; + u64 resend : 1; + u64 mbox_int : 1; + u64 in_int : 1; + u64 uns_int : 1; + u64 slc_int : 1; +#endif + } s; +}; + +/** + * struct nps_pkt_in_instr_ctl - Input Instruction Ring Control Registers. + * @is64b: If 1, the ring uses 64-byte instructions. If 0, the + * ring uses 32-byte instructions. + * @enb: Enable for the input ring. + */ +union nps_pkt_in_instr_ctl { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 62; + u64 is64b : 1; + u64 enb : 1; +#else + u64 enb : 1; + u64 is64b : 1; + u64 raz : 62; +#endif + } s; +}; + +/** + * struct nps_pkt_in_instr_rsize - Input instruction ring size registers + * @rsize: Ring size (number of instructions) + */ +union nps_pkt_in_instr_rsize { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 32; + u64 rsize : 32; +#else + u64 rsize : 32; + u64 raz : 32; +#endif + } s; +}; + +/** + * struct nps_pkt_in_instr_baoff_dbell - Input instruction ring + * base address offset and doorbell registers + * @aoff: Address offset. The offset from the NPS_PKT_IN_INSTR_BADDR + * where the next pointer is read. + * @dbell: Pointer list doorbell count. Write operations to this field + * increments the present value here. Read operations return the + * present value. + */ +union nps_pkt_in_instr_baoff_dbell { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 aoff : 32; + u64 dbell : 32; +#else + u64 dbell : 32; + u64 aoff : 32; +#endif + } s; +}; + +/** + * struct nps_core_int_ena_w1s - NPS core interrupt enable set register + * @host_nps_wr_err: Reads or sets enable for + * NPS_CORE_INT[HOST_NPS_WR_ERR]. + * @npco_dma_malform: Reads or sets enable for + * NPS_CORE_INT[NPCO_DMA_MALFORM]. + * @exec_wr_timeout: Reads or sets enable for + * NPS_CORE_INT[EXEC_WR_TIMEOUT]. + * @host_wr_timeout: Reads or sets enable for + * NPS_CORE_INT[HOST_WR_TIMEOUT]. + * @host_wr_err: Reads or sets enable for + * NPS_CORE_INT[HOST_WR_ERR] + */ +union nps_core_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz4 : 55; + u64 host_nps_wr_err : 1; + u64 npco_dma_malform : 1; + u64 exec_wr_timeout : 1; + u64 host_wr_timeout : 1; + u64 host_wr_err : 1; + u64 raz3 : 1; + u64 raz2 : 1; + u64 raz1 : 1; + u64 raz0 : 1; +#else + u64 raz0 : 1; + u64 raz1 : 1; + u64 raz2 : 1; + u64 raz3 : 1; + u64 host_wr_err : 1; + u64 host_wr_timeout : 1; + u64 exec_wr_timeout : 1; + u64 npco_dma_malform : 1; + u64 host_nps_wr_err : 1; + u64 raz4 : 55; +#endif + } s; +}; + +/** + * struct nps_core_gbl_vfcfg - Global VF Configuration Register. + * @ilk_disable: When set, this bit indicates that the ILK interface has + * been disabled. + * @obaf: BMO allocation control + * 0 = allocate per queue + * 1 = allocate per VF + * @ibaf: BMI allocation control + * 0 = allocate per queue + * 1 = allocate per VF + * @zaf: ZIP allocation control + * 0 = allocate per queue + * 1 = allocate per VF + * @aeaf: AE allocation control + * 0 = allocate per queue + * 1 = allocate per VF + * @seaf: SE allocation control + * 0 = allocation per queue + * 1 = allocate per VF + * @cfg: VF/PF mode. + */ +union nps_core_gbl_vfcfg { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz :55; + u64 ilk_disable :1; + u64 obaf :1; + u64 ibaf :1; + u64 zaf :1; + u64 aeaf :1; + u64 seaf :1; + u64 cfg :3; +#else + u64 cfg :3; + u64 seaf :1; + u64 aeaf :1; + u64 zaf :1; + u64 ibaf :1; + u64 obaf :1; + u64 ilk_disable :1; + u64 raz :55; +#endif + } s; +}; + +/** + * struct nps_core_int_active - NPS Core Interrupt Active Register + * @resend: Resend MSI-X interrupt if needs to handle interrupts + * Sofware can set this bit and then exit the ISR. + * @ocla: Set when any OCLA(0)_INT and corresponding OCLA(0_INT_ENA_W1C + * bit are set + * @mbox: Set when any NPS_PKT_MBOX_INT_LO/HI and corresponding + * NPS_PKT_MBOX_INT_LO_ENA_W1C/HI_ENA_W1C bits are set + * @emu: bit i is set in [EMU] when any EMU(i)_INT bit is set + * @bmo: Set when any BMO_INT bit is set + * @bmi: Set when any BMI_INT bit is set or when any non-RO + * BMI_INT and corresponding BMI_INT_ENA_W1C bits are both set + * @aqm: Set when any AQM_INT bit is set + * @zqm: Set when any ZQM_INT bit is set + * @efl: Set when any EFL_INT RO bit is set or when any non-RO EFL_INT + * and corresponding EFL_INT_ENA_W1C bits are both set + * @ilk: Set when any ILK_INT bit is set + * @lbc: Set when any LBC_INT RO bit is set or when any non-RO LBC_INT + * and corresponding LBC_INT_ENA_W1C bits are bot set + * @pem: Set when any PEM(0)_INT RO bit is set or when any non-RO + * PEM(0)_INT and corresponding PEM(0)_INT_ENA_W1C bit are both set + * @ucd: Set when any UCD_INT bit is set + * @zctl: Set when any ZIP_INT RO bit is set or when any non-RO ZIP_INT + * and corresponding ZIP_INT_ENA_W1C bits are both set + * @lbm: Set when any LBM_INT bit is set + * @nps_pkt: Set when any NPS_PKT_INT bit is set + * @nps_core: Set when any NPS_CORE_INT RO bit is set or when non-RO + * NPS_CORE_INT and corresponding NSP_CORE_INT_ENA_W1C bits are both set + */ +union nps_core_int_active { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 resend : 1; + u64 raz : 43; + u64 ocla : 1; + u64 mbox : 1; + u64 emu : 4; + u64 bmo : 1; + u64 bmi : 1; + u64 aqm : 1; + u64 zqm : 1; + u64 efl : 1; + u64 ilk : 1; + u64 lbc : 1; + u64 pem : 1; + u64 pom : 1; + u64 ucd : 1; + u64 zctl : 1; + u64 lbm : 1; + u64 nps_pkt : 1; + u64 nps_core : 1; +#else + u64 nps_core : 1; + u64 nps_pkt : 1; + u64 lbm : 1; + u64 zctl: 1; + u64 ucd : 1; + u64 pom : 1; + u64 pem : 1; + u64 lbc : 1; + u64 ilk : 1; + u64 efl : 1; + u64 zqm : 1; + u64 aqm : 1; + u64 bmi : 1; + u64 bmo : 1; + u64 emu : 4; + u64 mbox : 1; + u64 ocla : 1; + u64 raz : 43; + u64 resend : 1; +#endif + } s; +}; + +/** + * struct efl_core_int - EFL Interrupt Registers + * @epci_decode_err: EPCI decoded a transacation that was unknown + * This error should only occurred when there is a micrcode/SE error + * and should be considered fatal + * @ae_err: An AE uncorrectable error occurred. + * See EFL_CORE(0..3)_AE_ERR_INT + * @se_err: An SE uncorrectable error occurred. + * See EFL_CORE(0..3)_SE_ERR_INT + * @dbe: Double-bit error occurred in EFL + * @sbe: Single-bit error occurred in EFL + * @d_left: Asserted when new POM-Header-BMI-data is + * being sent to an Exec, and that Exec has Not read all BMI + * data associated with the previous POM header + * @len_ovr: Asserted when an Exec-Read is issued that is more than + * 14 greater in length that the BMI data left to be read + */ +union efl_core_int { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz : 57; + u64 epci_decode_err : 1; + u64 ae_err : 1; + u64 se_err : 1; + u64 dbe : 1; + u64 sbe : 1; + u64 d_left : 1; + u64 len_ovr : 1; +#else + u64 len_ovr : 1; + u64 d_left : 1; + u64 sbe : 1; + u64 dbe : 1; + u64 se_err : 1; + u64 ae_err : 1; + u64 epci_decode_err : 1; + u64 raz : 57; +#endif + } s; +}; + +/** + * struct efl_core_int_ena_w1s - EFL core interrupt enable set register + * @epci_decode_err: Reads or sets enable for + * EFL_CORE(0..3)_INT[EPCI_DECODE_ERR]. + * @d_left: Reads or sets enable for + * EFL_CORE(0..3)_INT[D_LEFT]. + * @len_ovr: Reads or sets enable for + * EFL_CORE(0..3)_INT[LEN_OVR]. + */ +union efl_core_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_7_63 : 57; + u64 epci_decode_err : 1; + u64 raz_2_5 : 4; + u64 d_left : 1; + u64 len_ovr : 1; +#else + u64 len_ovr : 1; + u64 d_left : 1; + u64 raz_2_5 : 4; + u64 epci_decode_err : 1; + u64 raz_7_63 : 57; +#endif + } s; +}; + +/** + * struct efl_rnm_ctl_status - RNM Control and Status Register + * @ent_sel: Select input to RNM FIFO + * @exp_ent: Exported entropy enable for random number generator + * @rng_rst: Reset to RNG. Setting this bit to 1 cancels the generation + * of the current random number. + * @rnm_rst: Reset the RNM. Setting this bit to 1 clears all sorted numbers + * in the random number memory. + * @rng_en: Enabled the output of the RNG. + * @ent_en: Entropy enable for random number generator. + */ +union efl_rnm_ctl_status { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_9_63 : 55; + u64 ent_sel : 4; + u64 exp_ent : 1; + u64 rng_rst : 1; + u64 rnm_rst : 1; + u64 rng_en : 1; + u64 ent_en : 1; +#else + u64 ent_en : 1; + u64 rng_en : 1; + u64 rnm_rst : 1; + u64 rng_rst : 1; + u64 exp_ent : 1; + u64 ent_sel : 4; + u64 raz_9_63 : 55; +#endif + } s; +}; + +/** + * struct bmi_ctl - BMI control register + * @ilk_hdrq_thrsh: Maximum number of header queue locations + * that ILK packets may consume. When the threshold is + * exceeded ILK_XOFF is sent to the BMI_X2P_ARB. + * @nps_hdrq_thrsh: Maximum number of header queue locations + * that NPS packets may consume. When the threshold is + * exceeded NPS_XOFF is sent to the BMI_X2P_ARB. + * @totl_hdrq_thrsh: Maximum number of header queue locations + * that the sum of ILK and NPS packets may consume. + * @ilk_free_thrsh: Maximum number of buffers that ILK packet + * flows may consume before ILK_XOFF is sent to the BMI_X2P_ARB. + * @nps_free_thrsh: Maximum number of buffers that NPS packet + * flows may consume before NPS XOFF is sent to the BMI_X2p_ARB. + * @totl_free_thrsh: Maximum number of buffers that bot ILK and NPS + * packet flows may consume before both NPS_XOFF and ILK_XOFF + * are asserted to the BMI_X2P_ARB. + * @max_pkt_len: Maximum packet length, integral number of 256B + * buffers. + */ +union bmi_ctl { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_56_63 : 8; + u64 ilk_hdrq_thrsh : 8; + u64 nps_hdrq_thrsh : 8; + u64 totl_hdrq_thrsh : 8; + u64 ilk_free_thrsh : 8; + u64 nps_free_thrsh : 8; + u64 totl_free_thrsh : 8; + u64 max_pkt_len : 8; +#else + u64 max_pkt_len : 8; + u64 totl_free_thrsh : 8; + u64 nps_free_thrsh : 8; + u64 ilk_free_thrsh : 8; + u64 totl_hdrq_thrsh : 8; + u64 nps_hdrq_thrsh : 8; + u64 ilk_hdrq_thrsh : 8; + u64 raz_56_63 : 8; +#endif + } s; +}; + +/** + * struct bmi_int_ena_w1s - BMI interrupt enable set register + * @ilk_req_oflw: Reads or sets enable for + * BMI_INT[ILK_REQ_OFLW]. + * @nps_req_oflw: Reads or sets enable for + * BMI_INT[NPS_REQ_OFLW]. + * @fpf_undrrn: Reads or sets enable for + * BMI_INT[FPF_UNDRRN]. + * @eop_err_ilk: Reads or sets enable for + * BMI_INT[EOP_ERR_ILK]. + * @eop_err_nps: Reads or sets enable for + * BMI_INT[EOP_ERR_NPS]. + * @sop_err_ilk: Reads or sets enable for + * BMI_INT[SOP_ERR_ILK]. + * @sop_err_nps: Reads or sets enable for + * BMI_INT[SOP_ERR_NPS]. + * @pkt_rcv_err_ilk: Reads or sets enable for + * BMI_INT[PKT_RCV_ERR_ILK]. + * @pkt_rcv_err_nps: Reads or sets enable for + * BMI_INT[PKT_RCV_ERR_NPS]. + * @max_len_err_ilk: Reads or sets enable for + * BMI_INT[MAX_LEN_ERR_ILK]. + * @max_len_err_nps: Reads or sets enable for + * BMI_INT[MAX_LEN_ERR_NPS]. + */ +union bmi_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_13_63 : 51; + u64 ilk_req_oflw : 1; + u64 nps_req_oflw : 1; + u64 raz_10 : 1; + u64 raz_9 : 1; + u64 fpf_undrrn : 1; + u64 eop_err_ilk : 1; + u64 eop_err_nps : 1; + u64 sop_err_ilk : 1; + u64 sop_err_nps : 1; + u64 pkt_rcv_err_ilk : 1; + u64 pkt_rcv_err_nps : 1; + u64 max_len_err_ilk : 1; + u64 max_len_err_nps : 1; +#else + u64 max_len_err_nps : 1; + u64 max_len_err_ilk : 1; + u64 pkt_rcv_err_nps : 1; + u64 pkt_rcv_err_ilk : 1; + u64 sop_err_nps : 1; + u64 sop_err_ilk : 1; + u64 eop_err_nps : 1; + u64 eop_err_ilk : 1; + u64 fpf_undrrn : 1; + u64 raz_9 : 1; + u64 raz_10 : 1; + u64 nps_req_oflw : 1; + u64 ilk_req_oflw : 1; + u64 raz_13_63 : 51; +#endif + } s; +}; + +/** + * struct bmo_ctl2 - BMO Control2 Register + * @arb_sel: Determines P2X Arbitration + * @ilk_buf_thrsh: Maximum number of buffers that the + * ILK packet flows may consume before ILK XOFF is + * asserted to the POM. + * @nps_slc_buf_thrsh: Maximum number of buffers that the + * NPS_SLC packet flow may consume before NPS_SLC XOFF is + * asserted to the POM. + * @nps_uns_buf_thrsh: Maximum number of buffers that the + * NPS_UNS packet flow may consume before NPS_UNS XOFF is + * asserted to the POM. + * @totl_buf_thrsh: Maximum number of buffers that ILK, NPS_UNS and + * NPS_SLC packet flows may consume before NPS_UNS XOFF, NSP_SLC and + * ILK_XOFF are all asserted POM. + */ +union bmo_ctl2 { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 arb_sel : 1; + u64 raz_32_62 : 31; + u64 ilk_buf_thrsh : 8; + u64 nps_slc_buf_thrsh : 8; + u64 nps_uns_buf_thrsh : 8; + u64 totl_buf_thrsh : 8; +#else + u64 totl_buf_thrsh : 8; + u64 nps_uns_buf_thrsh : 8; + u64 nps_slc_buf_thrsh : 8; + u64 ilk_buf_thrsh : 8; + u64 raz_32_62 : 31; + u64 arb_sel : 1; +#endif + } s; +}; + +/** + * struct pom_int_ena_w1s - POM interrupt enable set register + * @illegal_intf: Reads or sets enable for POM_INT[ILLEGAL_INTF]. + * @illegal_dport: Reads or sets enable for POM_INT[ILLEGAL_DPORT]. + */ +union pom_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz2 : 60; + u64 illegal_intf : 1; + u64 illegal_dport : 1; + u64 raz1 : 1; + u64 raz0 : 1; +#else + u64 raz0 : 1; + u64 raz1 : 1; + u64 illegal_dport : 1; + u64 illegal_intf : 1; + u64 raz2 : 60; +#endif + } s; +}; + +/** + * struct lbc_inval_ctl - LBC invalidation control register + * @wait_timer: Wait timer for wait state. [WAIT_TIMER] must + * always be written with its reset value. + * @cam_inval_start: Software should write [CAM_INVAL_START]=1 + * to initiate an LBC cache invalidation. After this, software + * should read LBC_INVAL_STATUS until LBC_INVAL_STATUS[DONE] is set. + * LBC hardware clears [CAVM_INVAL_START] before software can + * observed LBC_INVAL_STATUS[DONE] to be set + */ +union lbc_inval_ctl { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz2 : 48; + u64 wait_timer : 8; + u64 raz1 : 6; + u64 cam_inval_start : 1; + u64 raz0 : 1; +#else + u64 raz0 : 1; + u64 cam_inval_start : 1; + u64 raz1 : 6; + u64 wait_timer : 8; + u64 raz2 : 48; +#endif + } s; +}; + +/** + * struct lbc_int_ena_w1s - LBC interrupt enable set register + * @cam_hard_err: Reads or sets enable for LBC_INT[CAM_HARD_ERR]. + * @cam_inval_abort: Reads or sets enable for LBC_INT[CAM_INVAL_ABORT]. + * @over_fetch_err: Reads or sets enable for LBC_INT[OVER_FETCH_ERR]. + * @cache_line_to_err: Reads or sets enable for + * LBC_INT[CACHE_LINE_TO_ERR]. + * @cam_soft_err: Reads or sets enable for + * LBC_INT[CAM_SOFT_ERR]. + * @dma_rd_err: Reads or sets enable for + * LBC_INT[DMA_RD_ERR]. + */ +union lbc_int_ena_w1s { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_10_63 : 54; + u64 cam_hard_err : 1; + u64 cam_inval_abort : 1; + u64 over_fetch_err : 1; + u64 cache_line_to_err : 1; + u64 raz_2_5 : 4; + u64 cam_soft_err : 1; + u64 dma_rd_err : 1; +#else + u64 dma_rd_err : 1; + u64 cam_soft_err : 1; + u64 raz_2_5 : 4; + u64 cache_line_to_err : 1; + u64 over_fetch_err : 1; + u64 cam_inval_abort : 1; + u64 cam_hard_err : 1; + u64 raz_10_63 : 54; +#endif + } s; +}; + +/** + * struct lbc_int - LBC interrupt summary register + * @cam_hard_err: indicates a fatal hardware error. + * It requires system reset. + * When [CAM_HARD_ERR] is set, LBC stops logging any new information in + * LBC_POM_MISS_INFO_LOG, + * LBC_POM_MISS_ADDR_LOG, + * LBC_EFL_MISS_INFO_LOG, and + * LBC_EFL_MISS_ADDR_LOG. + * Software should sample them. + * @cam_inval_abort: indicates a fatal hardware error. + * System reset is required. + * @over_fetch_err: indicates a fatal hardware error + * System reset is required + * @cache_line_to_err: is a debug feature. + * This timeout interrupt bit tells the software that + * a cacheline in LBC has non-zero usage and the context + * has not been used for greater than the + * LBC_TO_CNT[TO_CNT] time interval. + * @sbe: Memory SBE error. This is recoverable via ECC. + * See LBC_ECC_INT for more details. + * @dbe: Memory DBE error. This is a fatal and requires a + * system reset. + * @pref_dat_len_mismatch_err: Summary bit for context length + * mismatch errors. + * @rd_dat_len_mismatch_err: Summary bit for SE read data length + * greater than data prefect length errors. + * @cam_soft_err: is recoverable. Software must complete a + * LBC_INVAL_CTL[CAM_INVAL_START] invalidation sequence and + * then clear [CAM_SOFT_ERR]. + * @dma_rd_err: A context prefect read of host memory returned with + * a read error. + */ +union lbc_int { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz_10_63 : 54; + u64 cam_hard_err : 1; + u64 cam_inval_abort : 1; + u64 over_fetch_err : 1; + u64 cache_line_to_err : 1; + u64 sbe : 1; + u64 dbe : 1; + u64 pref_dat_len_mismatch_err : 1; + u64 rd_dat_len_mismatch_err : 1; + u64 cam_soft_err : 1; + u64 dma_rd_err : 1; +#else + u64 dma_rd_err : 1; + u64 cam_soft_err : 1; + u64 rd_dat_len_mismatch_err : 1; + u64 pref_dat_len_mismatch_err : 1; + u64 dbe : 1; + u64 sbe : 1; + u64 cache_line_to_err : 1; + u64 over_fetch_err : 1; + u64 cam_inval_abort : 1; + u64 cam_hard_err : 1; + u64 raz_10_63 : 54; +#endif + } s; +}; + +/** + * struct lbc_inval_status: LBC Invalidation status register + * @cam_clean_entry_complete_cnt: The number of entries that are + * cleaned up successfully. + * @cam_clean_entry_cnt: The number of entries that have the CAM + * inval command issued. + * @cam_inval_state: cam invalidation FSM state + * @cam_inval_abort: cam invalidation abort + * @cam_rst_rdy: lbc_cam reset ready + * @done: LBC clears [DONE] when + * LBC_INVAL_CTL[CAM_INVAL_START] is written with a one, + * and sets [DONE] when it completes the invalidation + * sequence. + */ +union lbc_inval_status { + u64 value; + struct { +#if (defined(__BIG_ENDIAN_BITFIELD)) + u64 raz3 : 23; + u64 cam_clean_entry_complete_cnt : 9; + u64 raz2 : 7; + u64 cam_clean_entry_cnt : 9; + u64 raz1 : 5; + u64 cam_inval_state : 3; + u64 raz0 : 5; + u64 cam_inval_abort : 1; + u64 cam_rst_rdy : 1; + u64 done : 1; +#else + u64 done : 1; + u64 cam_rst_rdy : 1; + u64 cam_inval_abort : 1; + u64 raz0 : 5; + u64 cam_inval_state : 3; + u64 raz1 : 5; + u64 cam_clean_entry_cnt : 9; + u64 raz2 : 7; + u64 cam_clean_entry_complete_cnt : 9; + u64 raz3 : 23; +#endif + } s; +}; + +#endif /* __NITROX_CSR_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_dev.h b/drivers/crypto/cavium/nitrox/nitrox_dev.h new file mode 100644 index 000000000000..57858b04f165 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_dev.h @@ -0,0 +1,179 @@ +#ifndef __NITROX_DEV_H +#define __NITROX_DEV_H + +#include <linux/dma-mapping.h> +#include <linux/interrupt.h> +#include <linux/pci.h> + +#define VERSION_LEN 32 + +struct nitrox_cmdq { + /* command queue lock */ + spinlock_t cmdq_lock; + /* response list lock */ + spinlock_t response_lock; + /* backlog list lock */ + spinlock_t backlog_lock; + + /* request submitted to chip, in progress */ + struct list_head response_head; + /* hw queue full, hold in backlog list */ + struct list_head backlog_head; + + /* doorbell address */ + u8 __iomem *dbell_csr_addr; + /* base address of the queue */ + u8 *head; + + struct nitrox_device *ndev; + /* flush pending backlog commands */ + struct work_struct backlog_qflush; + + /* requests posted waiting for completion */ + atomic_t pending_count; + /* requests in backlog queues */ + atomic_t backlog_count; + + /* command size 32B/64B */ + u8 instr_size; + u8 qno; + u32 qsize; + + /* unaligned addresses */ + u8 *head_unaligned; + dma_addr_t dma_unaligned; + /* dma address of the base */ + dma_addr_t dma; +}; + +struct nitrox_hw { + /* firmware version */ + char fw_name[VERSION_LEN]; + + u16 vendor_id; + u16 device_id; + u8 revision_id; + + /* CNN55XX cores */ + u8 se_cores; + u8 ae_cores; + u8 zip_cores; +}; + +#define MAX_MSIX_VECTOR_NAME 20 +/** + * vectors for queues (64 AE, 64 SE and 64 ZIP) and + * error condition/mailbox. + */ +#define MAX_MSIX_VECTORS 192 + +struct nitrox_msix { + struct msix_entry *entries; + char **names; + DECLARE_BITMAP(irqs, MAX_MSIX_VECTORS); + u32 nr_entries; +}; + +struct bh_data { + /* slc port completion count address */ + u8 __iomem *completion_cnt_csr_addr; + + struct nitrox_cmdq *cmdq; + struct tasklet_struct resp_handler; +}; + +struct nitrox_bh { + struct bh_data *slc; +}; + +/* NITROX-5 driver state */ +#define NITROX_UCODE_LOADED 0 +#define NITROX_READY 1 + +/* command queue size */ +#define DEFAULT_CMD_QLEN 2048 +/* command timeout in milliseconds */ +#define CMD_TIMEOUT 2000 + +#define DEV(ndev) ((struct device *)(&(ndev)->pdev->dev)) +#define PF_MODE 0 + +#define NITROX_CSR_ADDR(ndev, offset) \ + ((ndev)->bar_addr + (offset)) + +/** + * struct nitrox_device - NITROX Device Information. + * @list: pointer to linked list of devices + * @bar_addr: iomap address + * @pdev: PCI device information + * @status: NITROX status + * @timeout: Request timeout in jiffies + * @refcnt: Device usage count + * @idx: device index (0..N) + * @node: NUMA node id attached + * @qlen: Command queue length + * @nr_queues: Number of command queues + * @ctx_pool: DMA pool for crypto context + * @pkt_cmdqs: SE Command queues + * @msix: MSI-X information + * @bh: post processing work + * @hw: hardware information + * @debugfs_dir: debugfs directory + */ +struct nitrox_device { + struct list_head list; + + u8 __iomem *bar_addr; + struct pci_dev *pdev; + + unsigned long status; + unsigned long timeout; + refcount_t refcnt; + + u8 idx; + int node; + u16 qlen; + u16 nr_queues; + + struct dma_pool *ctx_pool; + struct nitrox_cmdq *pkt_cmdqs; + + struct nitrox_msix msix; + struct nitrox_bh bh; + + struct nitrox_hw hw; +#if IS_ENABLED(CONFIG_DEBUG_FS) + struct dentry *debugfs_dir; +#endif +}; + +/** + * nitrox_read_csr - Read from device register + * @ndev: NITROX device + * @offset: offset of the register to read + * + * Returns: value read + */ +static inline u64 nitrox_read_csr(struct nitrox_device *ndev, u64 offset) +{ + return readq(ndev->bar_addr + offset); +} + +/** + * nitrox_write_csr - Write to device register + * @ndev: NITROX device + * @offset: offset of the register to write + * @value: value to write + */ +static inline void nitrox_write_csr(struct nitrox_device *ndev, u64 offset, + u64 value) +{ + writeq(value, (ndev->bar_addr + offset)); +} + +static inline int nitrox_ready(struct nitrox_device *ndev) +{ + return test_bit(NITROX_READY, &ndev->status); +} + +#endif /* __NITROX_DEV_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_hal.c b/drivers/crypto/cavium/nitrox/nitrox_hal.c new file mode 100644 index 000000000000..f0655f82fa7d --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_hal.c @@ -0,0 +1,401 @@ +#include <linux/delay.h> + +#include "nitrox_dev.h" +#include "nitrox_csr.h" + +/** + * emu_enable_cores - Enable EMU cluster cores. + * @ndev: N5 device + */ +static void emu_enable_cores(struct nitrox_device *ndev) +{ + union emu_se_enable emu_se; + union emu_ae_enable emu_ae; + int i; + + /* AE cores 20 per cluster */ + emu_ae.value = 0; + emu_ae.s.enable = 0xfffff; + + /* SE cores 16 per cluster */ + emu_se.value = 0; + emu_se.s.enable = 0xffff; + + /* enable per cluster cores */ + for (i = 0; i < NR_CLUSTERS; i++) { + nitrox_write_csr(ndev, EMU_AE_ENABLEX(i), emu_ae.value); + nitrox_write_csr(ndev, EMU_SE_ENABLEX(i), emu_se.value); + } +} + +/** + * nitrox_config_emu_unit - configure EMU unit. + * @ndev: N5 device + */ +void nitrox_config_emu_unit(struct nitrox_device *ndev) +{ + union emu_wd_int_ena_w1s emu_wd_int; + union emu_ge_int_ena_w1s emu_ge_int; + u64 offset; + int i; + + /* enable cores */ + emu_enable_cores(ndev); + + /* enable general error and watch dog interrupts */ + emu_ge_int.value = 0; + emu_ge_int.s.se_ge = 0xffff; + emu_ge_int.s.ae_ge = 0xfffff; + emu_wd_int.value = 0; + emu_wd_int.s.se_wd = 1; + + for (i = 0; i < NR_CLUSTERS; i++) { + offset = EMU_WD_INT_ENA_W1SX(i); + nitrox_write_csr(ndev, offset, emu_wd_int.value); + offset = EMU_GE_INT_ENA_W1SX(i); + nitrox_write_csr(ndev, offset, emu_ge_int.value); + } +} + +static void reset_pkt_input_ring(struct nitrox_device *ndev, int ring) +{ + union nps_pkt_in_instr_ctl pkt_in_ctl; + union nps_pkt_in_instr_baoff_dbell pkt_in_dbell; + union nps_pkt_in_done_cnts pkt_in_cnts; + u64 offset; + + offset = NPS_PKT_IN_INSTR_CTLX(ring); + /* disable the ring */ + pkt_in_ctl.value = nitrox_read_csr(ndev, offset); + pkt_in_ctl.s.enb = 0; + nitrox_write_csr(ndev, offset, pkt_in_ctl.value); + usleep_range(100, 150); + + /* wait to clear [ENB] */ + do { + pkt_in_ctl.value = nitrox_read_csr(ndev, offset); + } while (pkt_in_ctl.s.enb); + + /* clear off door bell counts */ + offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(ring); + pkt_in_dbell.value = 0; + pkt_in_dbell.s.dbell = 0xffffffff; + nitrox_write_csr(ndev, offset, pkt_in_dbell.value); + + /* clear done counts */ + offset = NPS_PKT_IN_DONE_CNTSX(ring); + pkt_in_cnts.value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, pkt_in_cnts.value); + usleep_range(50, 100); +} + +void enable_pkt_input_ring(struct nitrox_device *ndev, int ring) +{ + union nps_pkt_in_instr_ctl pkt_in_ctl; + u64 offset; + + /* 64-byte instruction size */ + offset = NPS_PKT_IN_INSTR_CTLX(ring); + pkt_in_ctl.value = nitrox_read_csr(ndev, offset); + pkt_in_ctl.s.is64b = 1; + pkt_in_ctl.s.enb = 1; + nitrox_write_csr(ndev, offset, pkt_in_ctl.value); + + /* wait for set [ENB] */ + do { + pkt_in_ctl.value = nitrox_read_csr(ndev, offset); + } while (!pkt_in_ctl.s.enb); +} + +/** + * nitrox_config_pkt_input_rings - configure Packet Input Rings + * @ndev: N5 device + */ +void nitrox_config_pkt_input_rings(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < ndev->nr_queues; i++) { + struct nitrox_cmdq *cmdq = &ndev->pkt_cmdqs[i]; + union nps_pkt_in_instr_rsize pkt_in_rsize; + u64 offset; + + reset_pkt_input_ring(ndev, i); + + /* configure ring base address 16-byte aligned, + * size and interrupt threshold. + */ + offset = NPS_PKT_IN_INSTR_BADDRX(i); + nitrox_write_csr(ndev, NPS_PKT_IN_INSTR_BADDRX(i), cmdq->dma); + + /* configure ring size */ + offset = NPS_PKT_IN_INSTR_RSIZEX(i); + pkt_in_rsize.value = 0; + pkt_in_rsize.s.rsize = ndev->qlen; + nitrox_write_csr(ndev, offset, pkt_in_rsize.value); + + /* set high threshold for pkt input ring interrupts */ + offset = NPS_PKT_IN_INT_LEVELSX(i); + nitrox_write_csr(ndev, offset, 0xffffffff); + + enable_pkt_input_ring(ndev, i); + } +} + +static void reset_pkt_solicit_port(struct nitrox_device *ndev, int port) +{ + union nps_pkt_slc_ctl pkt_slc_ctl; + union nps_pkt_slc_cnts pkt_slc_cnts; + u64 offset; + + /* disable slc port */ + offset = NPS_PKT_SLC_CTLX(port); + pkt_slc_ctl.value = nitrox_read_csr(ndev, offset); + pkt_slc_ctl.s.enb = 0; + nitrox_write_csr(ndev, offset, pkt_slc_ctl.value); + usleep_range(100, 150); + + /* wait to clear [ENB] */ + do { + pkt_slc_ctl.value = nitrox_read_csr(ndev, offset); + } while (pkt_slc_ctl.s.enb); + + /* clear slc counters */ + offset = NPS_PKT_SLC_CNTSX(port); + pkt_slc_cnts.value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, pkt_slc_cnts.value); + usleep_range(50, 100); +} + +void enable_pkt_solicit_port(struct nitrox_device *ndev, int port) +{ + union nps_pkt_slc_ctl pkt_slc_ctl; + u64 offset; + + offset = NPS_PKT_SLC_CTLX(port); + pkt_slc_ctl.value = 0; + pkt_slc_ctl.s.enb = 1; + + /* + * 8 trailing 0x00 bytes will be added + * to the end of the outgoing packet. + */ + pkt_slc_ctl.s.z = 1; + /* enable response header */ + pkt_slc_ctl.s.rh = 1; + nitrox_write_csr(ndev, offset, pkt_slc_ctl.value); + + /* wait to set [ENB] */ + do { + pkt_slc_ctl.value = nitrox_read_csr(ndev, offset); + } while (!pkt_slc_ctl.s.enb); +} + +static void config_single_pkt_solicit_port(struct nitrox_device *ndev, + int port) +{ + union nps_pkt_slc_int_levels pkt_slc_int; + u64 offset; + + reset_pkt_solicit_port(ndev, port); + + offset = NPS_PKT_SLC_INT_LEVELSX(port); + pkt_slc_int.value = 0; + /* time interrupt threshold */ + pkt_slc_int.s.timet = 0x3fffff; + nitrox_write_csr(ndev, offset, pkt_slc_int.value); + + enable_pkt_solicit_port(ndev, port); +} + +void nitrox_config_pkt_solicit_ports(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < ndev->nr_queues; i++) + config_single_pkt_solicit_port(ndev, i); +} + +/** + * enable_nps_interrupts - enable NPS interrutps + * @ndev: N5 device. + * + * This includes NPS core, packet in and slc interrupts. + */ +static void enable_nps_interrupts(struct nitrox_device *ndev) +{ + union nps_core_int_ena_w1s core_int; + + /* NPS core interrutps */ + core_int.value = 0; + core_int.s.host_wr_err = 1; + core_int.s.host_wr_timeout = 1; + core_int.s.exec_wr_timeout = 1; + core_int.s.npco_dma_malform = 1; + core_int.s.host_nps_wr_err = 1; + nitrox_write_csr(ndev, NPS_CORE_INT_ENA_W1S, core_int.value); + + /* NPS packet in ring interrupts */ + nitrox_write_csr(ndev, NPS_PKT_IN_RERR_LO_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, NPS_PKT_IN_RERR_HI_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, NPS_PKT_IN_ERR_TYPE_ENA_W1S, (~0ULL)); + /* NPS packet slc port interrupts */ + nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_HI_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, NPS_PKT_SLC_RERR_LO_ENA_W1S, (~0ULL)); + nitrox_write_csr(ndev, NPS_PKT_SLC_ERR_TYPE_ENA_W1S, (~0uLL)); +} + +void nitrox_config_nps_unit(struct nitrox_device *ndev) +{ + union nps_core_gbl_vfcfg core_gbl_vfcfg; + + /* endian control information */ + nitrox_write_csr(ndev, NPS_CORE_CONTROL, 1ULL); + + /* disable ILK interface */ + core_gbl_vfcfg.value = 0; + core_gbl_vfcfg.s.ilk_disable = 1; + core_gbl_vfcfg.s.cfg = PF_MODE; + nitrox_write_csr(ndev, NPS_CORE_GBL_VFCFG, core_gbl_vfcfg.value); + /* config input and solicit ports */ + nitrox_config_pkt_input_rings(ndev); + nitrox_config_pkt_solicit_ports(ndev); + + /* enable interrupts */ + enable_nps_interrupts(ndev); +} + +void nitrox_config_pom_unit(struct nitrox_device *ndev) +{ + union pom_int_ena_w1s pom_int; + int i; + + /* enable pom interrupts */ + pom_int.value = 0; + pom_int.s.illegal_dport = 1; + nitrox_write_csr(ndev, POM_INT_ENA_W1S, pom_int.value); + + /* enable perf counters */ + for (i = 0; i < ndev->hw.se_cores; i++) + nitrox_write_csr(ndev, POM_PERF_CTL, BIT_ULL(i)); +} + +/** + * nitrox_config_rand_unit - enable N5 random number unit + * @ndev: N5 device + */ +void nitrox_config_rand_unit(struct nitrox_device *ndev) +{ + union efl_rnm_ctl_status efl_rnm_ctl; + u64 offset; + + offset = EFL_RNM_CTL_STATUS; + efl_rnm_ctl.value = nitrox_read_csr(ndev, offset); + efl_rnm_ctl.s.ent_en = 1; + efl_rnm_ctl.s.rng_en = 1; + nitrox_write_csr(ndev, offset, efl_rnm_ctl.value); +} + +void nitrox_config_efl_unit(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < NR_CLUSTERS; i++) { + union efl_core_int_ena_w1s efl_core_int; + u64 offset; + + /* EFL core interrupts */ + offset = EFL_CORE_INT_ENA_W1SX(i); + efl_core_int.value = 0; + efl_core_int.s.len_ovr = 1; + efl_core_int.s.d_left = 1; + efl_core_int.s.epci_decode_err = 1; + nitrox_write_csr(ndev, offset, efl_core_int.value); + + offset = EFL_CORE_VF_ERR_INT0_ENA_W1SX(i); + nitrox_write_csr(ndev, offset, (~0ULL)); + offset = EFL_CORE_VF_ERR_INT1_ENA_W1SX(i); + nitrox_write_csr(ndev, offset, (~0ULL)); + } +} + +void nitrox_config_bmi_unit(struct nitrox_device *ndev) +{ + union bmi_ctl bmi_ctl; + union bmi_int_ena_w1s bmi_int_ena; + u64 offset; + + /* no threshold limits for PCIe */ + offset = BMI_CTL; + bmi_ctl.value = nitrox_read_csr(ndev, offset); + bmi_ctl.s.max_pkt_len = 0xff; + bmi_ctl.s.nps_free_thrsh = 0xff; + bmi_ctl.s.nps_hdrq_thrsh = 0x7a; + nitrox_write_csr(ndev, offset, bmi_ctl.value); + + /* enable interrupts */ + offset = BMI_INT_ENA_W1S; + bmi_int_ena.value = 0; + bmi_int_ena.s.max_len_err_nps = 1; + bmi_int_ena.s.pkt_rcv_err_nps = 1; + bmi_int_ena.s.fpf_undrrn = 1; + nitrox_write_csr(ndev, offset, bmi_int_ena.value); +} + +void nitrox_config_bmo_unit(struct nitrox_device *ndev) +{ + union bmo_ctl2 bmo_ctl2; + u64 offset; + + /* no threshold limits for PCIe */ + offset = BMO_CTL2; + bmo_ctl2.value = nitrox_read_csr(ndev, offset); + bmo_ctl2.s.nps_slc_buf_thrsh = 0xff; + nitrox_write_csr(ndev, offset, bmo_ctl2.value); +} + +void invalidate_lbc(struct nitrox_device *ndev) +{ + union lbc_inval_ctl lbc_ctl; + union lbc_inval_status lbc_stat; + u64 offset; + + /* invalidate LBC */ + offset = LBC_INVAL_CTL; + lbc_ctl.value = nitrox_read_csr(ndev, offset); + lbc_ctl.s.cam_inval_start = 1; + nitrox_write_csr(ndev, offset, lbc_ctl.value); + + offset = LBC_INVAL_STATUS; + + do { + lbc_stat.value = nitrox_read_csr(ndev, offset); + } while (!lbc_stat.s.done); +} + +void nitrox_config_lbc_unit(struct nitrox_device *ndev) +{ + union lbc_int_ena_w1s lbc_int_ena; + u64 offset; + + invalidate_lbc(ndev); + + /* enable interrupts */ + offset = LBC_INT_ENA_W1S; + lbc_int_ena.value = 0; + lbc_int_ena.s.dma_rd_err = 1; + lbc_int_ena.s.over_fetch_err = 1; + lbc_int_ena.s.cam_inval_abort = 1; + lbc_int_ena.s.cam_hard_err = 1; + nitrox_write_csr(ndev, offset, lbc_int_ena.value); + + offset = LBC_PLM_VF1_64_INT_ENA_W1S; + nitrox_write_csr(ndev, offset, (~0ULL)); + offset = LBC_PLM_VF65_128_INT_ENA_W1S; + nitrox_write_csr(ndev, offset, (~0ULL)); + + offset = LBC_ELM_VF1_64_INT_ENA_W1S; + nitrox_write_csr(ndev, offset, (~0ULL)); + offset = LBC_ELM_VF65_128_INT_ENA_W1S; + nitrox_write_csr(ndev, offset, (~0ULL)); +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_isr.c b/drivers/crypto/cavium/nitrox/nitrox_isr.c new file mode 100644 index 000000000000..71f934871a89 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_isr.c @@ -0,0 +1,467 @@ +#include <linux/pci.h> +#include <linux/printk.h> +#include <linux/slab.h> + +#include "nitrox_dev.h" +#include "nitrox_csr.h" +#include "nitrox_common.h" + +#define NR_RING_VECTORS 3 +#define NPS_CORE_INT_ACTIVE_ENTRY 192 + +/** + * nps_pkt_slc_isr - IRQ handler for NPS solicit port + * @irq: irq number + * @data: argument + */ +static irqreturn_t nps_pkt_slc_isr(int irq, void *data) +{ + struct bh_data *slc = data; + union nps_pkt_slc_cnts pkt_slc_cnts; + + pkt_slc_cnts.value = readq(slc->completion_cnt_csr_addr); + /* New packet on SLC output port */ + if (pkt_slc_cnts.s.slc_int) + tasklet_hi_schedule(&slc->resp_handler); + + return IRQ_HANDLED; +} + +static void clear_nps_core_err_intr(struct nitrox_device *ndev) +{ + u64 value; + + /* Write 1 to clear */ + value = nitrox_read_csr(ndev, NPS_CORE_INT); + nitrox_write_csr(ndev, NPS_CORE_INT, value); + + dev_err_ratelimited(DEV(ndev), "NSP_CORE_INT 0x%016llx\n", value); +} + +static void clear_nps_pkt_err_intr(struct nitrox_device *ndev) +{ + union nps_pkt_int pkt_int; + unsigned long value, offset; + int i; + + pkt_int.value = nitrox_read_csr(ndev, NPS_PKT_INT); + dev_err_ratelimited(DEV(ndev), "NPS_PKT_INT 0x%016llx\n", + pkt_int.value); + + if (pkt_int.s.slc_err) { + offset = NPS_PKT_SLC_ERR_TYPE; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_SLC_ERR_TYPE 0x%016lx\n", value); + + offset = NPS_PKT_SLC_RERR_LO; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + /* enable the solicit ports */ + for_each_set_bit(i, &value, BITS_PER_LONG) + enable_pkt_solicit_port(ndev, i); + + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_SLC_RERR_LO 0x%016lx\n", value); + + offset = NPS_PKT_SLC_RERR_HI; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_SLC_RERR_HI 0x%016lx\n", value); + } + + if (pkt_int.s.in_err) { + offset = NPS_PKT_IN_ERR_TYPE; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_IN_ERR_TYPE 0x%016lx\n", value); + offset = NPS_PKT_IN_RERR_LO; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + /* enable the input ring */ + for_each_set_bit(i, &value, BITS_PER_LONG) + enable_pkt_input_ring(ndev, i); + + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_IN_RERR_LO 0x%016lx\n", value); + + offset = NPS_PKT_IN_RERR_HI; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + dev_err_ratelimited(DEV(ndev), + "NPS_PKT_IN_RERR_HI 0x%016lx\n", value); + } +} + +static void clear_pom_err_intr(struct nitrox_device *ndev) +{ + u64 value; + + value = nitrox_read_csr(ndev, POM_INT); + nitrox_write_csr(ndev, POM_INT, value); + dev_err_ratelimited(DEV(ndev), "POM_INT 0x%016llx\n", value); +} + +static void clear_pem_err_intr(struct nitrox_device *ndev) +{ + u64 value; + + value = nitrox_read_csr(ndev, PEM0_INT); + nitrox_write_csr(ndev, PEM0_INT, value); + dev_err_ratelimited(DEV(ndev), "PEM(0)_INT 0x%016llx\n", value); +} + +static void clear_lbc_err_intr(struct nitrox_device *ndev) +{ + union lbc_int lbc_int; + u64 value, offset; + int i; + + lbc_int.value = nitrox_read_csr(ndev, LBC_INT); + dev_err_ratelimited(DEV(ndev), "LBC_INT 0x%016llx\n", lbc_int.value); + + if (lbc_int.s.dma_rd_err) { + for (i = 0; i < NR_CLUSTERS; i++) { + offset = EFL_CORE_VF_ERR_INT0X(i); + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + offset = EFL_CORE_VF_ERR_INT1X(i); + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + } + } + + if (lbc_int.s.cam_soft_err) { + dev_err_ratelimited(DEV(ndev), "CAM_SOFT_ERR, invalidating LBC\n"); + invalidate_lbc(ndev); + } + + if (lbc_int.s.pref_dat_len_mismatch_err) { + offset = LBC_PLM_VF1_64_INT; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + offset = LBC_PLM_VF65_128_INT; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + } + + if (lbc_int.s.rd_dat_len_mismatch_err) { + offset = LBC_ELM_VF1_64_INT; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + offset = LBC_ELM_VF65_128_INT; + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + } + nitrox_write_csr(ndev, LBC_INT, lbc_int.value); +} + +static void clear_efl_err_intr(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < NR_CLUSTERS; i++) { + union efl_core_int core_int; + u64 value, offset; + + offset = EFL_CORE_INTX(i); + core_int.value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, core_int.value); + dev_err_ratelimited(DEV(ndev), "ELF_CORE(%d)_INT 0x%016llx\n", + i, core_int.value); + if (core_int.s.se_err) { + offset = EFL_CORE_SE_ERR_INTX(i); + value = nitrox_read_csr(ndev, offset); + nitrox_write_csr(ndev, offset, value); + } + } +} + +static void clear_bmi_err_intr(struct nitrox_device *ndev) +{ + u64 value; + + value = nitrox_read_csr(ndev, BMI_INT); + nitrox_write_csr(ndev, BMI_INT, value); + dev_err_ratelimited(DEV(ndev), "BMI_INT 0x%016llx\n", value); +} + +/** + * clear_nps_core_int_active - clear NPS_CORE_INT_ACTIVE interrupts + * @ndev: NITROX device + */ +static void clear_nps_core_int_active(struct nitrox_device *ndev) +{ + union nps_core_int_active core_int_active; + + core_int_active.value = nitrox_read_csr(ndev, NPS_CORE_INT_ACTIVE); + + if (core_int_active.s.nps_core) + clear_nps_core_err_intr(ndev); + + if (core_int_active.s.nps_pkt) + clear_nps_pkt_err_intr(ndev); + + if (core_int_active.s.pom) + clear_pom_err_intr(ndev); + + if (core_int_active.s.pem) + clear_pem_err_intr(ndev); + + if (core_int_active.s.lbc) + clear_lbc_err_intr(ndev); + + if (core_int_active.s.efl) + clear_efl_err_intr(ndev); + + if (core_int_active.s.bmi) + clear_bmi_err_intr(ndev); + + /* If more work callback the ISR, set resend */ + core_int_active.s.resend = 1; + nitrox_write_csr(ndev, NPS_CORE_INT_ACTIVE, core_int_active.value); +} + +static irqreturn_t nps_core_int_isr(int irq, void *data) +{ + struct nitrox_device *ndev = data; + + clear_nps_core_int_active(ndev); + + return IRQ_HANDLED; +} + +static int nitrox_enable_msix(struct nitrox_device *ndev) +{ + struct msix_entry *entries; + char **names; + int i, nr_entries, ret; + + /* + * PF MSI-X vectors + * + * Entry 0: NPS PKT ring 0 + * Entry 1: AQMQ ring 0 + * Entry 2: ZQM ring 0 + * Entry 3: NPS PKT ring 1 + * Entry 4: AQMQ ring 1 + * Entry 5: ZQM ring 1 + * .... + * Entry 192: NPS_CORE_INT_ACTIVE + */ + nr_entries = (ndev->nr_queues * NR_RING_VECTORS) + 1; + entries = kzalloc_node(nr_entries * sizeof(struct msix_entry), + GFP_KERNEL, ndev->node); + if (!entries) + return -ENOMEM; + + names = kcalloc(nr_entries, sizeof(char *), GFP_KERNEL); + if (!names) { + kfree(entries); + return -ENOMEM; + } + + /* fill entires */ + for (i = 0; i < (nr_entries - 1); i++) + entries[i].entry = i; + + entries[i].entry = NPS_CORE_INT_ACTIVE_ENTRY; + + for (i = 0; i < nr_entries; i++) { + *(names + i) = kzalloc(MAX_MSIX_VECTOR_NAME, GFP_KERNEL); + if (!(*(names + i))) { + ret = -ENOMEM; + goto msix_fail; + } + } + ndev->msix.entries = entries; + ndev->msix.names = names; + ndev->msix.nr_entries = nr_entries; + + ret = pci_enable_msix_exact(ndev->pdev, ndev->msix.entries, + ndev->msix.nr_entries); + if (ret) { + dev_err(&ndev->pdev->dev, "Failed to enable MSI-X IRQ(s) %d\n", + ret); + goto msix_fail; + } + return 0; + +msix_fail: + for (i = 0; i < nr_entries; i++) + kfree(*(names + i)); + + kfree(entries); + kfree(names); + return ret; +} + +static void nitrox_cleanup_pkt_slc_bh(struct nitrox_device *ndev) +{ + int i; + + if (!ndev->bh.slc) + return; + + for (i = 0; i < ndev->nr_queues; i++) { + struct bh_data *bh = &ndev->bh.slc[i]; + + tasklet_disable(&bh->resp_handler); + tasklet_kill(&bh->resp_handler); + } + kfree(ndev->bh.slc); + ndev->bh.slc = NULL; +} + +static int nitrox_setup_pkt_slc_bh(struct nitrox_device *ndev) +{ + u32 size; + int i; + + size = ndev->nr_queues * sizeof(struct bh_data); + ndev->bh.slc = kzalloc(size, GFP_KERNEL); + if (!ndev->bh.slc) + return -ENOMEM; + + for (i = 0; i < ndev->nr_queues; i++) { + struct bh_data *bh = &ndev->bh.slc[i]; + u64 offset; + + offset = NPS_PKT_SLC_CNTSX(i); + /* pre calculate completion count address */ + bh->completion_cnt_csr_addr = NITROX_CSR_ADDR(ndev, offset); + bh->cmdq = &ndev->pkt_cmdqs[i]; + + tasklet_init(&bh->resp_handler, pkt_slc_resp_handler, + (unsigned long)bh); + } + + return 0; +} + +static int nitrox_request_irqs(struct nitrox_device *ndev) +{ + struct pci_dev *pdev = ndev->pdev; + struct msix_entry *msix_ent = ndev->msix.entries; + int nr_ring_vectors, i = 0, ring, cpu, ret; + char *name; + + /* + * PF MSI-X vectors + * + * Entry 0: NPS PKT ring 0 + * Entry 1: AQMQ ring 0 + * Entry 2: ZQM ring 0 + * Entry 3: NPS PKT ring 1 + * .... + * Entry 192: NPS_CORE_INT_ACTIVE + */ + nr_ring_vectors = ndev->nr_queues * NR_RING_VECTORS; + + /* request irq for pkt ring/ports only */ + while (i < nr_ring_vectors) { + name = *(ndev->msix.names + i); + ring = (i / NR_RING_VECTORS); + snprintf(name, MAX_MSIX_VECTOR_NAME, "n5(%d)-slc-ring%d", + ndev->idx, ring); + + ret = request_irq(msix_ent[i].vector, nps_pkt_slc_isr, 0, + name, &ndev->bh.slc[ring]); + if (ret) { + dev_err(&pdev->dev, "failed to get irq %d for %s\n", + msix_ent[i].vector, name); + return ret; + } + cpu = ring % num_online_cpus(); + irq_set_affinity_hint(msix_ent[i].vector, get_cpu_mask(cpu)); + + set_bit(i, ndev->msix.irqs); + i += NR_RING_VECTORS; + } + + /* Request IRQ for NPS_CORE_INT_ACTIVE */ + name = *(ndev->msix.names + i); + snprintf(name, MAX_MSIX_VECTOR_NAME, "n5(%d)-nps-core-int", ndev->idx); + ret = request_irq(msix_ent[i].vector, nps_core_int_isr, 0, name, ndev); + if (ret) { + dev_err(&pdev->dev, "failed to get irq %d for %s\n", + msix_ent[i].vector, name); + return ret; + } + set_bit(i, ndev->msix.irqs); + + return 0; +} + +static void nitrox_disable_msix(struct nitrox_device *ndev) +{ + struct msix_entry *msix_ent = ndev->msix.entries; + char **names = ndev->msix.names; + int i = 0, ring, nr_ring_vectors; + + nr_ring_vectors = ndev->msix.nr_entries - 1; + + /* clear pkt ring irqs */ + while (i < nr_ring_vectors) { + if (test_and_clear_bit(i, ndev->msix.irqs)) { + ring = (i / NR_RING_VECTORS); + irq_set_affinity_hint(msix_ent[i].vector, NULL); + free_irq(msix_ent[i].vector, &ndev->bh.slc[ring]); + } + i += NR_RING_VECTORS; + } + irq_set_affinity_hint(msix_ent[i].vector, NULL); + free_irq(msix_ent[i].vector, ndev); + clear_bit(i, ndev->msix.irqs); + + kfree(ndev->msix.entries); + for (i = 0; i < ndev->msix.nr_entries; i++) + kfree(*(names + i)); + + kfree(names); + pci_disable_msix(ndev->pdev); +} + +/** + * nitrox_pf_cleanup_isr: Cleanup PF MSI-X and IRQ + * @ndev: NITROX device + */ +void nitrox_pf_cleanup_isr(struct nitrox_device *ndev) +{ + nitrox_disable_msix(ndev); + nitrox_cleanup_pkt_slc_bh(ndev); +} + +/** + * nitrox_init_isr - Initialize PF MSI-X vectors and IRQ + * @ndev: NITROX device + * + * Return: 0 on success, a negative value on failure. + */ +int nitrox_pf_init_isr(struct nitrox_device *ndev) +{ + int err; + + err = nitrox_setup_pkt_slc_bh(ndev); + if (err) + return err; + + err = nitrox_enable_msix(ndev); + if (err) + goto msix_fail; + + err = nitrox_request_irqs(ndev); + if (err) + goto irq_fail; + + return 0; + +irq_fail: + nitrox_disable_msix(ndev); +msix_fail: + nitrox_cleanup_pkt_slc_bh(ndev); + return err; +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_lib.c b/drivers/crypto/cavium/nitrox/nitrox_lib.c new file mode 100644 index 000000000000..b4a391adb9b6 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_lib.c @@ -0,0 +1,210 @@ +#include <linux/cpumask.h> +#include <linux/dma-mapping.h> +#include <linux/dmapool.h> +#include <linux/delay.h> +#include <linux/gfp.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/pci_regs.h> +#include <linux/vmalloc.h> +#include <linux/pci.h> + +#include "nitrox_dev.h" +#include "nitrox_common.h" +#include "nitrox_req.h" +#include "nitrox_csr.h" + +#define CRYPTO_CTX_SIZE 256 + +/* command queue alignments */ +#define PKT_IN_ALIGN 16 + +static int cmdq_common_init(struct nitrox_cmdq *cmdq) +{ + struct nitrox_device *ndev = cmdq->ndev; + u32 qsize; + + qsize = (ndev->qlen) * cmdq->instr_size; + cmdq->head_unaligned = dma_zalloc_coherent(DEV(ndev), + (qsize + PKT_IN_ALIGN), + &cmdq->dma_unaligned, + GFP_KERNEL); + if (!cmdq->head_unaligned) + return -ENOMEM; + + cmdq->head = PTR_ALIGN(cmdq->head_unaligned, PKT_IN_ALIGN); + cmdq->dma = PTR_ALIGN(cmdq->dma_unaligned, PKT_IN_ALIGN); + cmdq->qsize = (qsize + PKT_IN_ALIGN); + + spin_lock_init(&cmdq->response_lock); + spin_lock_init(&cmdq->cmdq_lock); + spin_lock_init(&cmdq->backlog_lock); + + INIT_LIST_HEAD(&cmdq->response_head); + INIT_LIST_HEAD(&cmdq->backlog_head); + INIT_WORK(&cmdq->backlog_qflush, backlog_qflush_work); + + atomic_set(&cmdq->pending_count, 0); + atomic_set(&cmdq->backlog_count, 0); + return 0; +} + +static void cmdq_common_cleanup(struct nitrox_cmdq *cmdq) +{ + struct nitrox_device *ndev = cmdq->ndev; + + cancel_work_sync(&cmdq->backlog_qflush); + + dma_free_coherent(DEV(ndev), cmdq->qsize, + cmdq->head_unaligned, cmdq->dma_unaligned); + + atomic_set(&cmdq->pending_count, 0); + atomic_set(&cmdq->backlog_count, 0); + + cmdq->dbell_csr_addr = NULL; + cmdq->head = NULL; + cmdq->dma = 0; + cmdq->qsize = 0; + cmdq->instr_size = 0; +} + +static void nitrox_cleanup_pkt_cmdqs(struct nitrox_device *ndev) +{ + int i; + + for (i = 0; i < ndev->nr_queues; i++) { + struct nitrox_cmdq *cmdq = &ndev->pkt_cmdqs[i]; + + cmdq_common_cleanup(cmdq); + } + kfree(ndev->pkt_cmdqs); + ndev->pkt_cmdqs = NULL; +} + +static int nitrox_init_pkt_cmdqs(struct nitrox_device *ndev) +{ + int i, err, size; + + size = ndev->nr_queues * sizeof(struct nitrox_cmdq); + ndev->pkt_cmdqs = kzalloc(size, GFP_KERNEL); + if (!ndev->pkt_cmdqs) + return -ENOMEM; + + for (i = 0; i < ndev->nr_queues; i++) { + struct nitrox_cmdq *cmdq; + u64 offset; + + cmdq = &ndev->pkt_cmdqs[i]; + cmdq->ndev = ndev; + cmdq->qno = i; + cmdq->instr_size = sizeof(struct nps_pkt_instr); + + offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(i); + /* SE ring doorbell address for this queue */ + cmdq->dbell_csr_addr = NITROX_CSR_ADDR(ndev, offset); + + err = cmdq_common_init(cmdq); + if (err) + goto pkt_cmdq_fail; + } + return 0; + +pkt_cmdq_fail: + nitrox_cleanup_pkt_cmdqs(ndev); + return err; +} + +static int create_crypto_dma_pool(struct nitrox_device *ndev) +{ + size_t size; + + /* Crypto context pool, 16 byte aligned */ + size = CRYPTO_CTX_SIZE + sizeof(struct ctx_hdr); + ndev->ctx_pool = dma_pool_create("crypto-context", + DEV(ndev), size, 16, 0); + if (!ndev->ctx_pool) + return -ENOMEM; + + return 0; +} + +static void destroy_crypto_dma_pool(struct nitrox_device *ndev) +{ + if (!ndev->ctx_pool) + return; + + dma_pool_destroy(ndev->ctx_pool); + ndev->ctx_pool = NULL; +} + +/* + * crypto_alloc_context - Allocate crypto context from pool + * @ndev: NITROX Device + */ +void *crypto_alloc_context(struct nitrox_device *ndev) +{ + struct ctx_hdr *ctx; + void *vaddr; + dma_addr_t dma; + + vaddr = dma_pool_alloc(ndev->ctx_pool, (GFP_ATOMIC | __GFP_ZERO), &dma); + if (!vaddr) + return NULL; + + /* fill meta data */ + ctx = vaddr; + ctx->pool = ndev->ctx_pool; + ctx->dma = dma; + ctx->ctx_dma = dma + sizeof(struct ctx_hdr); + + return ((u8 *)vaddr + sizeof(struct ctx_hdr)); +} + +/** + * crypto_free_context - Free crypto context to pool + * @ctx: context to free + */ +void crypto_free_context(void *ctx) +{ + struct ctx_hdr *ctxp; + + if (!ctx) + return; + + ctxp = (struct ctx_hdr *)((u8 *)ctx - sizeof(struct ctx_hdr)); + dma_pool_free(ctxp->pool, ctxp, ctxp->dma); +} + +/** + * nitrox_common_sw_init - allocate software resources. + * @ndev: NITROX device + * + * Allocates crypto context pools and command queues etc. + * + * Return: 0 on success, or a negative error code on error. + */ +int nitrox_common_sw_init(struct nitrox_device *ndev) +{ + int err = 0; + + /* per device crypto context pool */ + err = create_crypto_dma_pool(ndev); + if (err) + return err; + + err = nitrox_init_pkt_cmdqs(ndev); + if (err) + destroy_crypto_dma_pool(ndev); + + return err; +} + +/** + * nitrox_common_sw_cleanup - free software resources. + * @ndev: NITROX device + */ +void nitrox_common_sw_cleanup(struct nitrox_device *ndev) +{ + nitrox_cleanup_pkt_cmdqs(ndev); + destroy_crypto_dma_pool(ndev); +} diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c new file mode 100644 index 000000000000..ae44a464cd2d --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c @@ -0,0 +1,640 @@ +#include <linux/aer.h> +#include <linux/delay.h> +#include <linux/debugfs.h> +#include <linux/firmware.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/pci.h> +#include <linux/pci_ids.h> + +#include "nitrox_dev.h" +#include "nitrox_common.h" +#include "nitrox_csr.h" + +#define CNN55XX_DEV_ID 0x12 +#define MAX_PF_QUEUES 64 +#define UCODE_HLEN 48 +#define SE_GROUP 0 + +#define DRIVER_VERSION "1.0" +/* SE microcode */ +#define SE_FW "cnn55xx_se.fw" + +static const char nitrox_driver_name[] = "CNN55XX"; + +static LIST_HEAD(ndevlist); +static DEFINE_MUTEX(devlist_lock); +static unsigned int num_devices; + +/** + * nitrox_pci_tbl - PCI Device ID Table + */ +static const struct pci_device_id nitrox_pci_tbl[] = { + {PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0}, + /* required last entry */ + {0, } +}; +MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl); + +static unsigned int qlen = DEFAULT_CMD_QLEN; +module_param(qlen, uint, 0644); +MODULE_PARM_DESC(qlen, "Command queue length - default 2048"); + +/** + * struct ucode - Firmware Header + * @id: microcode ID + * @version: firmware version + * @code_size: code section size + * @raz: alignment + * @code: code section + */ +struct ucode { + u8 id; + char version[VERSION_LEN - 1]; + __be32 code_size; + u8 raz[12]; + u64 code[0]; +}; + +/** + * write_to_ucd_unit - Write Firmware to NITROX UCD unit + */ +static void write_to_ucd_unit(struct nitrox_device *ndev, + struct ucode *ucode) +{ + u32 code_size = be32_to_cpu(ucode->code_size) * 2; + u64 offset, data; + int i = 0; + + /* + * UCD structure + * + * ------------- + * | BLK 7 | + * ------------- + * | BLK 6 | + * ------------- + * | ... | + * ------------- + * | BLK 0 | + * ------------- + * Total of 8 blocks, each size 32KB + */ + + /* set the block number */ + offset = UCD_UCODE_LOAD_BLOCK_NUM; + nitrox_write_csr(ndev, offset, 0); + + code_size = roundup(code_size, 8); + while (code_size) { + data = ucode->code[i]; + /* write 8 bytes at a time */ + offset = UCD_UCODE_LOAD_IDX_DATAX(i); + nitrox_write_csr(ndev, offset, data); + code_size -= 8; + i++; + } + + /* put all SE cores in group 0 */ + offset = POM_GRP_EXECMASKX(SE_GROUP); + nitrox_write_csr(ndev, offset, (~0ULL)); + + for (i = 0; i < ndev->hw.se_cores; i++) { + /* + * write block number and firware length + * bit:<2:0> block number + * bit:3 is set SE uses 32KB microcode + * bit:3 is clear SE uses 64KB microcode + */ + offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i); + nitrox_write_csr(ndev, offset, 0x8); + } + usleep_range(300, 400); +} + +static int nitrox_load_fw(struct nitrox_device *ndev, const char *fw_name) +{ + const struct firmware *fw; + struct ucode *ucode; + int ret; + + dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name); + + ret = request_firmware(&fw, fw_name, DEV(ndev)); + if (ret < 0) { + dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name); + return ret; + } + + ucode = (struct ucode *)fw->data; + /* copy the firmware version */ + memcpy(ndev->hw.fw_name, ucode->version, (VERSION_LEN - 2)); + ndev->hw.fw_name[VERSION_LEN - 1] = '\0'; + + write_to_ucd_unit(ndev, ucode); + release_firmware(fw); + + set_bit(NITROX_UCODE_LOADED, &ndev->status); + /* barrier to sync with other cpus */ + smp_mb__after_atomic(); + return 0; +} + +/** + * nitrox_add_to_devlist - add NITROX device to global device list + * @ndev: NITROX device + */ +static int nitrox_add_to_devlist(struct nitrox_device *ndev) +{ + struct nitrox_device *dev; + int ret = 0; + + INIT_LIST_HEAD(&ndev->list); + refcount_set(&ndev->refcnt, 1); + + mutex_lock(&devlist_lock); + list_for_each_entry(dev, &ndevlist, list) { + if (dev == ndev) { + ret = -EEXIST; + goto unlock; + } + } + ndev->idx = num_devices++; + list_add_tail(&ndev->list, &ndevlist); +unlock: + mutex_unlock(&devlist_lock); + return ret; +} + +/** + * nitrox_remove_from_devlist - remove NITROX device from + * global device list + * @ndev: NITROX device + */ +static void nitrox_remove_from_devlist(struct nitrox_device *ndev) +{ + mutex_lock(&devlist_lock); + list_del(&ndev->list); + num_devices--; + mutex_unlock(&devlist_lock); +} + +struct nitrox_device *nitrox_get_first_device(void) +{ + struct nitrox_device *ndev = NULL; + + mutex_lock(&devlist_lock); + list_for_each_entry(ndev, &ndevlist, list) { + if (nitrox_ready(ndev)) + break; + } + mutex_unlock(&devlist_lock); + if (!ndev) + return NULL; + + refcount_inc(&ndev->refcnt); + /* barrier to sync with other cpus */ + smp_mb__after_atomic(); + return ndev; +} + +void nitrox_put_device(struct nitrox_device *ndev) +{ + if (!ndev) + return; + + refcount_dec(&ndev->refcnt); + /* barrier to sync with other cpus */ + smp_mb__after_atomic(); +} + +static int nitrox_reset_device(struct pci_dev *pdev) +{ + int pos = 0; + + pos = pci_save_state(pdev); + if (pos) { + dev_err(&pdev->dev, "Failed to save pci state\n"); + return -ENOMEM; + } + + pos = pci_pcie_cap(pdev); + if (!pos) + return -ENOTTY; + + if (!pci_wait_for_pending_transaction(pdev)) + dev_err(&pdev->dev, "waiting for pending transaction\n"); + + pcie_capability_set_word(pdev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_BCR_FLR); + msleep(100); + pci_restore_state(pdev); + + return 0; +} + +static int nitrox_pf_sw_init(struct nitrox_device *ndev) +{ + int err; + + err = nitrox_common_sw_init(ndev); + if (err) + return err; + + err = nitrox_pf_init_isr(ndev); + if (err) + nitrox_common_sw_cleanup(ndev); + + return err; +} + +static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev) +{ + nitrox_pf_cleanup_isr(ndev); + nitrox_common_sw_cleanup(ndev); +} + +/** + * nitrox_bist_check - Check NITORX BIST registers status + * @ndev: NITROX device + */ +static int nitrox_bist_check(struct nitrox_device *ndev) +{ + u64 value = 0; + int i; + + for (i = 0; i < NR_CLUSTERS; i++) { + value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i)); + value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i)); + } + value += nitrox_read_csr(ndev, UCD_BIST_STATUS); + value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG); + value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG); + value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG); + value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG); + value += nitrox_read_csr(ndev, POM_BIST_REG); + value += nitrox_read_csr(ndev, BMI_BIST_REG); + value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT); + value += nitrox_read_csr(ndev, BMO_BIST_REG); + value += nitrox_read_csr(ndev, LBC_BIST_STATUS); + value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0)); + if (value) + return -EIO; + return 0; +} + +static void nitrox_get_hwinfo(struct nitrox_device *ndev) +{ + union emu_fuse_map emu_fuse; + u64 offset; + int i; + + for (i = 0; i < NR_CLUSTERS; i++) { + u8 dead_cores; + + offset = EMU_FUSE_MAPX(i); + emu_fuse.value = nitrox_read_csr(ndev, offset); + if (emu_fuse.s.valid) { + dead_cores = hweight32(emu_fuse.s.ae_fuse); + ndev->hw.ae_cores += AE_CORES_PER_CLUSTER - dead_cores; + dead_cores = hweight16(emu_fuse.s.se_fuse); + ndev->hw.se_cores += SE_CORES_PER_CLUSTER - dead_cores; + } + } +} + +static int nitrox_pf_hw_init(struct nitrox_device *ndev) +{ + int err; + + err = nitrox_bist_check(ndev); + if (err) { + dev_err(&ndev->pdev->dev, "BIST check failed\n"); + return err; + } + /* get cores information */ + nitrox_get_hwinfo(ndev); + + nitrox_config_nps_unit(ndev); + nitrox_config_pom_unit(ndev); + nitrox_config_efl_unit(ndev); + /* configure IO units */ + nitrox_config_bmi_unit(ndev); + nitrox_config_bmo_unit(ndev); + /* configure Local Buffer Cache */ + nitrox_config_lbc_unit(ndev); + nitrox_config_rand_unit(ndev); + + /* load firmware on SE cores */ + err = nitrox_load_fw(ndev, SE_FW); + if (err) + return err; + + nitrox_config_emu_unit(ndev); + + return 0; +} + +#if IS_ENABLED(CONFIG_DEBUG_FS) +static int registers_show(struct seq_file *s, void *v) +{ + struct nitrox_device *ndev = s->private; + u64 offset; + + /* NPS DMA stats */ + offset = NPS_STATS_PKT_DMA_RD_CNT; + seq_printf(s, "NPS_STATS_PKT_DMA_RD_CNT 0x%016llx\n", + nitrox_read_csr(ndev, offset)); + offset = NPS_STATS_PKT_DMA_WR_CNT; + seq_printf(s, "NPS_STATS_PKT_DMA_WR_CNT 0x%016llx\n", + nitrox_read_csr(ndev, offset)); + + /* BMI/BMO stats */ + offset = BMI_NPS_PKT_CNT; + seq_printf(s, "BMI_NPS_PKT_CNT 0x%016llx\n", + nitrox_read_csr(ndev, offset)); + offset = BMO_NPS_SLC_PKT_CNT; + seq_printf(s, "BMO_NPS_PKT_CNT 0x%016llx\n", + nitrox_read_csr(ndev, offset)); + + return 0; +} + +static int registers_open(struct inode *inode, struct file *file) +{ + return single_open(file, registers_show, inode->i_private); +} + +static const struct file_operations register_fops = { + .owner = THIS_MODULE, + .open = registers_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int firmware_show(struct seq_file *s, void *v) +{ + struct nitrox_device *ndev = s->private; + + seq_printf(s, "Version: %s\n", ndev->hw.fw_name); + return 0; +} + +static int firmware_open(struct inode *inode, struct file *file) +{ + return single_open(file, firmware_show, inode->i_private); +} + +static const struct file_operations firmware_fops = { + .owner = THIS_MODULE, + .open = firmware_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static int nitrox_show(struct seq_file *s, void *v) +{ + struct nitrox_device *ndev = s->private; + + seq_printf(s, "NITROX-5 [idx: %d]\n", ndev->idx); + seq_printf(s, " Revision ID: 0x%0x\n", ndev->hw.revision_id); + seq_printf(s, " Cores [AE: %u SE: %u]\n", + ndev->hw.ae_cores, ndev->hw.se_cores); + seq_printf(s, " Number of Queues: %u\n", ndev->nr_queues); + seq_printf(s, " Queue length: %u\n", ndev->qlen); + seq_printf(s, " Node: %u\n", ndev->node); + + return 0; +} + +static int nitrox_open(struct inode *inode, struct file *file) +{ + return single_open(file, nitrox_show, inode->i_private); +} + +static const struct file_operations nitrox_fops = { + .owner = THIS_MODULE, + .open = nitrox_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +static void nitrox_debugfs_exit(struct nitrox_device *ndev) +{ + debugfs_remove_recursive(ndev->debugfs_dir); + ndev->debugfs_dir = NULL; +} + +static int nitrox_debugfs_init(struct nitrox_device *ndev) +{ + struct dentry *dir, *f; + + dir = debugfs_create_dir(KBUILD_MODNAME, NULL); + if (!dir) + return -ENOMEM; + + ndev->debugfs_dir = dir; + f = debugfs_create_file("counters", 0400, dir, ndev, ®ister_fops); + if (!f) + goto err; + f = debugfs_create_file("firmware", 0400, dir, ndev, &firmware_fops); + if (!f) + goto err; + f = debugfs_create_file("nitrox", 0400, dir, ndev, &nitrox_fops); + if (!f) + goto err; + + return 0; + +err: + nitrox_debugfs_exit(ndev); + return -ENODEV; +} +#else +static int nitrox_debugfs_init(struct nitrox_device *ndev) +{ + return 0; +} + +static void nitrox_debugfs_exit(struct nitrox_device *ndev) +{ +} +#endif + +/** + * nitrox_probe - NITROX Initialization function. + * @pdev: PCI device information struct + * @id: entry in nitrox_pci_tbl + * + * Return: 0, if the driver is bound to the device, or + * a negative error if there is failure. + */ +static int nitrox_probe(struct pci_dev *pdev, + const struct pci_device_id *id) +{ + struct nitrox_device *ndev; + int err; + + dev_info_once(&pdev->dev, "%s driver version %s\n", + nitrox_driver_name, DRIVER_VERSION); + + err = pci_enable_device_mem(pdev); + if (err) + return err; + + /* do FLR */ + err = nitrox_reset_device(pdev); + if (err) { + dev_err(&pdev->dev, "FLR failed\n"); + pci_disable_device(pdev); + return err; + } + + if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) { + dev_dbg(&pdev->dev, "DMA to 64-BIT address\n"); + } else { + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (err) { + dev_err(&pdev->dev, "DMA configuration failed\n"); + pci_disable_device(pdev); + return err; + } + } + + err = pci_request_mem_regions(pdev, nitrox_driver_name); + if (err) { + pci_disable_device(pdev); + return err; + } + pci_set_master(pdev); + + ndev = kzalloc(sizeof(*ndev), GFP_KERNEL); + if (!ndev) + goto ndev_fail; + + pci_set_drvdata(pdev, ndev); + ndev->pdev = pdev; + + /* add to device list */ + nitrox_add_to_devlist(ndev); + + ndev->hw.vendor_id = pdev->vendor; + ndev->hw.device_id = pdev->device; + ndev->hw.revision_id = pdev->revision; + /* command timeout in jiffies */ + ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT); + ndev->node = dev_to_node(&pdev->dev); + if (ndev->node == NUMA_NO_NODE) + ndev->node = 0; + + ndev->bar_addr = ioremap(pci_resource_start(pdev, 0), + pci_resource_len(pdev, 0)); + if (!ndev->bar_addr) { + err = -EIO; + goto ioremap_err; + } + /* allocate command queus based on cpus, max queues are 64 */ + ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus()); + ndev->qlen = qlen; + + err = nitrox_pf_sw_init(ndev); + if (err) + goto ioremap_err; + + err = nitrox_pf_hw_init(ndev); + if (err) + goto pf_hw_fail; + + err = nitrox_debugfs_init(ndev); + if (err) + goto pf_hw_fail; + + set_bit(NITROX_READY, &ndev->status); + /* barrier to sync with other cpus */ + smp_mb__after_atomic(); + + err = nitrox_crypto_register(); + if (err) + goto crypto_fail; + + return 0; + +crypto_fail: + nitrox_debugfs_exit(ndev); + clear_bit(NITROX_READY, &ndev->status); + /* barrier to sync with other cpus */ + smp_mb__after_atomic(); +pf_hw_fail: + nitrox_pf_sw_cleanup(ndev); +ioremap_err: + nitrox_remove_from_devlist(ndev); + kfree(ndev); + pci_set_drvdata(pdev, NULL); +ndev_fail: + pci_release_mem_regions(pdev); + pci_disable_device(pdev); + return err; +} + +/** + * nitrox_remove - Unbind the driver from the device. + * @pdev: PCI device information struct + */ +static void nitrox_remove(struct pci_dev *pdev) +{ + struct nitrox_device *ndev = pci_get_drvdata(pdev); + + if (!ndev) + return; + + if (!refcount_dec_and_test(&ndev->refcnt)) { + dev_err(DEV(ndev), "Device refcnt not zero (%d)\n", + refcount_read(&ndev->refcnt)); + return; + } + + dev_info(DEV(ndev), "Removing Device %x:%x\n", + ndev->hw.vendor_id, ndev->hw.device_id); + + clear_bit(NITROX_READY, &ndev->status); + /* barrier to sync with other cpus */ + smp_mb__after_atomic(); + + nitrox_remove_from_devlist(ndev); + nitrox_crypto_unregister(); + nitrox_debugfs_exit(ndev); + nitrox_pf_sw_cleanup(ndev); + + iounmap(ndev->bar_addr); + kfree(ndev); + + pci_set_drvdata(pdev, NULL); + pci_release_mem_regions(pdev); + pci_disable_device(pdev); +} + +static void nitrox_shutdown(struct pci_dev *pdev) +{ + pci_set_drvdata(pdev, NULL); + pci_release_mem_regions(pdev); + pci_disable_device(pdev); +} + +static struct pci_driver nitrox_driver = { + .name = nitrox_driver_name, + .id_table = nitrox_pci_tbl, + .probe = nitrox_probe, + .remove = nitrox_remove, + .shutdown = nitrox_shutdown, +}; + +module_pci_driver(nitrox_driver); + +MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>"); +MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " "); +MODULE_LICENSE("GPL"); +MODULE_VERSION(DRIVER_VERSION); +MODULE_FIRMWARE(SE_FW); diff --git a/drivers/crypto/cavium/nitrox/nitrox_req.h b/drivers/crypto/cavium/nitrox/nitrox_req.h new file mode 100644 index 000000000000..74f4c20dc87d --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_req.h @@ -0,0 +1,445 @@ +#ifndef __NITROX_REQ_H +#define __NITROX_REQ_H + +#include <linux/dma-mapping.h> +#include <crypto/aes.h> + +#include "nitrox_dev.h" + +/** + * struct gphdr - General purpose Header + * @param0: first parameter. + * @param1: second parameter. + * @param2: third parameter. + * @param3: fourth parameter. + * + * Params tell the iv and enc/dec data offsets. + */ +struct gphdr { + __be16 param0; + __be16 param1; + __be16 param2; + __be16 param3; +}; + +/** + * struct se_req_ctrl - SE request information. + * @arg: Minor number of the opcode + * @ctxc: Context control. + * @unca: Uncertainity enabled. + * @info: Additional information for SE cores. + * @ctxl: Context length in bytes. + * @uddl: User defined data length + */ +union se_req_ctrl { + u64 value; + struct { + u64 raz : 22; + u64 arg : 8; + u64 ctxc : 2; + u64 unca : 1; + u64 info : 3; + u64 unc : 8; + u64 ctxl : 12; + u64 uddl : 8; + } s; +}; + +struct nitrox_sglist { + u16 len; + u16 raz0; + u32 raz1; + dma_addr_t dma; +}; + +#define MAX_IV_LEN 16 + +/** + * struct se_crypto_request - SE crypto request structure. + * @opcode: Request opcode (enc/dec) + * @flags: flags from crypto subsystem + * @ctx_handle: Crypto context handle. + * @gph: GP Header + * @ctrl: Request Information. + * @in: Input sglist + * @out: Output sglist + */ +struct se_crypto_request { + u8 opcode; + gfp_t gfp; + u32 flags; + u64 ctx_handle; + + struct gphdr gph; + union se_req_ctrl ctrl; + + u8 iv[MAX_IV_LEN]; + u16 ivsize; + + struct scatterlist *src; + struct scatterlist *dst; +}; + +/* Crypto opcodes */ +#define FLEXI_CRYPTO_ENCRYPT_HMAC 0x33 +#define ENCRYPT 0 +#define DECRYPT 1 + +/* IV from context */ +#define IV_FROM_CTX 0 +/* IV from Input data */ +#define IV_FROM_DPTR 1 + +/** + * cipher opcodes for firmware + */ +enum flexi_cipher { + CIPHER_NULL = 0, + CIPHER_3DES_CBC, + CIPHER_3DES_ECB, + CIPHER_AES_CBC, + CIPHER_AES_ECB, + CIPHER_AES_CFB, + CIPHER_AES_CTR, + CIPHER_AES_GCM, + CIPHER_AES_XTS, + CIPHER_AES_CCM, + CIPHER_AES_CBC_CTS, + CIPHER_AES_ECB_CTS, + CIPHER_INVALID +}; + +/** + * struct crypto_keys - Crypto keys + * @key: Encryption key or KEY1 for AES-XTS + * @iv: Encryption IV or Tweak for AES-XTS + */ +struct crypto_keys { + union { + u8 key[AES_MAX_KEY_SIZE]; + u8 key1[AES_MAX_KEY_SIZE]; + } u; + u8 iv[AES_BLOCK_SIZE]; +}; + +/** + * struct auth_keys - Authentication keys + * @ipad: IPAD or KEY2 for AES-XTS + * @opad: OPAD or AUTH KEY if auth_input_type = 1 + */ +struct auth_keys { + union { + u8 ipad[64]; + u8 key2[64]; + } u; + u8 opad[64]; +}; + +/** + * struct flexi_crypto_context - Crypto context + * @cipher_type: Encryption cipher type + * @aes_keylen: AES key length + * @iv_source: Encryption IV source + * @hash_type: Authentication type + * @auth_input_type: Authentication input type + * 1 - Authentication IV and KEY, microcode calculates OPAD/IPAD + * 0 - Authentication OPAD/IPAD + * @mac_len: mac length + * @crypto: Crypto keys + * @auth: Authentication keys + */ +struct flexi_crypto_context { + union { + __be64 flags; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 cipher_type : 4; + u64 reserved_59 : 1; + u64 aes_keylen : 2; + u64 iv_source : 1; + u64 hash_type : 4; + u64 reserved_49_51 : 3; + u64 auth_input_type: 1; + u64 mac_len : 8; + u64 reserved_0_39 : 40; +#else + u64 reserved_0_39 : 40; + u64 mac_len : 8; + u64 auth_input_type: 1; + u64 reserved_49_51 : 3; + u64 hash_type : 4; + u64 iv_source : 1; + u64 aes_keylen : 2; + u64 reserved_59 : 1; + u64 cipher_type : 4; +#endif + } w0; + }; + + struct crypto_keys crypto; + struct auth_keys auth; +}; + +struct nitrox_crypto_ctx { + struct nitrox_device *ndev; + union { + u64 ctx_handle; + struct flexi_crypto_context *fctx; + } u; +}; + +struct nitrox_kcrypt_request { + struct se_crypto_request creq; + struct nitrox_crypto_ctx *nctx; + struct skcipher_request *skreq; +}; + +/** + * struct pkt_instr_hdr - Packet Instruction Header + * @g: Gather used + * When [G] is set and [GSZ] != 0, the instruction is + * indirect gather instruction. + * When [G] is set and [GSZ] = 0, the instruction is + * direct gather instruction. + * @gsz: Number of pointers in the indirect gather list + * @ihi: When set hardware duplicates the 1st 8 bytes of pkt_instr_hdr + * and adds them to the packet after the pkt_instr_hdr but before any UDD + * @ssz: Not used by the input hardware. But can become slc_store_int[SSZ] + * when [IHI] is set. + * @fsz: The number of front data bytes directly included in the + * PCIe instruction. + * @tlen: The length of the input packet in bytes, include: + * - 16B pkt_hdr + * - Inline context bytes if any, + * - UDD if any, + * - packet payload bytes + */ +union pkt_instr_hdr { + u64 value; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 raz_48_63 : 16; + u64 g : 1; + u64 gsz : 7; + u64 ihi : 1; + u64 ssz : 7; + u64 raz_30_31 : 2; + u64 fsz : 6; + u64 raz_16_23 : 8; + u64 tlen : 16; +#else + u64 tlen : 16; + u64 raz_16_23 : 8; + u64 fsz : 6; + u64 raz_30_31 : 2; + u64 ssz : 7; + u64 ihi : 1; + u64 gsz : 7; + u64 g : 1; + u64 raz_48_63 : 16; +#endif + } s; +}; + +/** + * struct pkt_hdr - Packet Input Header + * @opcode: Request opcode (Major) + * @arg: Request opcode (Minor) + * @ctxc: Context control. + * @unca: When set [UNC] is the uncertainty count for an input packet. + * The hardware uses uncertainty counts to predict + * output buffer use and avoid deadlock. + * @info: Not used by input hardware. Available for use + * during SE processing. + * @destport: The expected destination port/ring/channel for the packet. + * @unc: Uncertainty count for an input packet. + * @grp: SE group that will process the input packet. + * @ctxl: Context Length in 64-bit words. + * @uddl: User-defined data (UDD) length in bytes. + * @ctxp: Context pointer. CTXP<63,2:0> must be zero in all cases. + */ +union pkt_hdr { + u64 value[2]; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 opcode : 8; + u64 arg : 8; + u64 ctxc : 2; + u64 unca : 1; + u64 raz_44 : 1; + u64 info : 3; + u64 destport : 9; + u64 unc : 8; + u64 raz_19_23 : 5; + u64 grp : 3; + u64 raz_15 : 1; + u64 ctxl : 7; + u64 uddl : 8; +#else + u64 uddl : 8; + u64 ctxl : 7; + u64 raz_15 : 1; + u64 grp : 3; + u64 raz_19_23 : 5; + u64 unc : 8; + u64 destport : 9; + u64 info : 3; + u64 raz_44 : 1; + u64 unca : 1; + u64 ctxc : 2; + u64 arg : 8; + u64 opcode : 8; +#endif + __be64 ctxp; + } s; +}; + +/** + * struct slc_store_info - Solicited Paceket Output Store Information. + * @ssz: The number of scatterlist pointers for the solicited output port + * packet. + * @rptr: The result pointer for the solicited output port packet. + * If [SSZ]=0, [RPTR] must point directly to a buffer on the remote + * host that is large enough to hold the entire output packet. + * If [SSZ]!=0, [RPTR] must point to an array of ([SSZ]+3)/4 + * sglist components at [RPTR] on the remote host. + */ +union slc_store_info { + u64 value[2]; + struct { +#if defined(__BIG_ENDIAN_BITFIELD) + u64 raz_39_63 : 25; + u64 ssz : 7; + u64 raz_0_31 : 32; +#else + u64 raz_0_31 : 32; + u64 ssz : 7; + u64 raz_39_63 : 25; +#endif + __be64 rptr; + } s; +}; + +/** + * struct nps_pkt_instr - NPS Packet Instruction of SE cores. + * @dptr0 : Input pointer points to buffer in remote host. + * @ih: Packet Instruction Header (8 bytes) + * @irh: Packet Input Header (16 bytes) + * @slc: Solicited Packet Output Store Information (16 bytes) + * @fdata: Front data + * + * 64-Byte Instruction Format + */ +struct nps_pkt_instr { + __be64 dptr0; + union pkt_instr_hdr ih; + union pkt_hdr irh; + union slc_store_info slc; + u64 fdata[2]; +}; + +/** + * struct ctx_hdr - Book keeping data about the crypto context + * @pool: Pool used to allocate crypto context + * @dma: Base DMA address of the cypto context + * @ctx_dma: Actual usable crypto context for NITROX + */ +struct ctx_hdr { + struct dma_pool *pool; + dma_addr_t dma; + dma_addr_t ctx_dma; +}; + +/* + * struct sglist_component - SG list component format + * @len0: The number of bytes at [PTR0] on the remote host. + * @len1: The number of bytes at [PTR1] on the remote host. + * @len2: The number of bytes at [PTR2] on the remote host. + * @len3: The number of bytes at [PTR3] on the remote host. + * @dma0: First pointer point to buffer in remote host. + * @dma1: Second pointer point to buffer in remote host. + * @dma2: Third pointer point to buffer in remote host. + * @dma3: Fourth pointer point to buffer in remote host. + */ +struct nitrox_sgcomp { + __be16 len[4]; + __be64 dma[4]; +}; + +/* + * strutct nitrox_sgtable - SG list information + * @map_cnt: Number of buffers mapped + * @nr_comp: Number of sglist components + * @total_bytes: Total bytes in sglist. + * @len: Total sglist components length. + * @dma: DMA address of sglist component. + * @dir: DMA direction. + * @buf: crypto request buffer. + * @sglist: SG list of input/output buffers. + * @sgcomp: sglist component for NITROX. + */ +struct nitrox_sgtable { + u8 map_bufs_cnt; + u8 nr_sgcomp; + u16 total_bytes; + u32 len; + dma_addr_t dma; + enum dma_data_direction dir; + + struct scatterlist *buf; + struct nitrox_sglist *sglist; + struct nitrox_sgcomp *sgcomp; +}; + +/* Response Header Length */ +#define ORH_HLEN 8 +/* Completion bytes Length */ +#define COMP_HLEN 8 + +struct resp_hdr { + u64 orh; + dma_addr_t orh_dma; + u64 completion; + dma_addr_t completion_dma; +}; + +typedef void (*completion_t)(struct skcipher_request *skreq, int err); + +/** + * struct nitrox_softreq - Represents the NIROX Request. + * @response: response list entry + * @backlog: Backlog list entry + * @ndev: Device used to submit the request + * @cmdq: Command queue for submission + * @resp: Response headers + * @instr: 64B instruction + * @in: SG table for input + * @out SG table for output + * @tstamp: Request submitted time in jiffies + * @callback: callback after request completion/timeout + * @cb_arg: callback argument + */ +struct nitrox_softreq { + struct list_head response; + struct list_head backlog; + + u32 flags; + gfp_t gfp; + atomic_t status; + bool inplace; + + struct nitrox_device *ndev; + struct nitrox_cmdq *cmdq; + + struct nps_pkt_instr instr; + struct resp_hdr resp; + struct nitrox_sgtable in; + struct nitrox_sgtable out; + + unsigned long tstamp; + + completion_t callback; + struct skcipher_request *skreq; +}; + +#endif /* __NITROX_REQ_H */ diff --git a/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c new file mode 100644 index 000000000000..4bb4377c5ac0 --- /dev/null +++ b/drivers/crypto/cavium/nitrox/nitrox_reqmgr.c @@ -0,0 +1,735 @@ +#include <linux/gfp.h> +#include <linux/workqueue.h> +#include <crypto/internal/skcipher.h> + +#include "nitrox_dev.h" +#include "nitrox_req.h" +#include "nitrox_csr.h" +#include "nitrox_req.h" + +/* SLC_STORE_INFO */ +#define MIN_UDD_LEN 16 +/* PKT_IN_HDR + SLC_STORE_INFO */ +#define FDATA_SIZE 32 +/* Base destination port for the solicited requests */ +#define SOLICIT_BASE_DPORT 256 +#define PENDING_SIG 0xFFFFFFFFFFFFFFFFUL + +#define REQ_NOT_POSTED 1 +#define REQ_BACKLOG 2 +#define REQ_POSTED 3 + +/** + * Response codes from SE microcode + * 0x00 - Success + * Completion with no error + * 0x43 - ERR_GC_DATA_LEN_INVALID + * Invalid Data length if Encryption Data length is + * less than 16 bytes for AES-XTS and AES-CTS. + * 0x45 - ERR_GC_CTX_LEN_INVALID + * Invalid context length: CTXL != 23 words. + * 0x4F - ERR_GC_DOCSIS_CIPHER_INVALID + * DOCSIS support is enabled with other than + * AES/DES-CBC mode encryption. + * 0x50 - ERR_GC_DOCSIS_OFFSET_INVALID + * Authentication offset is other than 0 with + * Encryption IV source = 0. + * Authentication offset is other than 8 (DES)/16 (AES) + * with Encryption IV source = 1 + * 0x51 - ERR_GC_CRC32_INVALID_SELECTION + * CRC32 is enabled for other than DOCSIS encryption. + * 0x52 - ERR_GC_AES_CCM_FLAG_INVALID + * Invalid flag options in AES-CCM IV. + */ + +/** + * dma_free_sglist - unmap and free the sg lists. + * @ndev: N5 device + * @sgtbl: SG table + */ +static void softreq_unmap_sgbufs(struct nitrox_softreq *sr) +{ + struct nitrox_device *ndev = sr->ndev; + struct device *dev = DEV(ndev); + struct nitrox_sglist *sglist; + + /* unmap in sgbuf */ + sglist = sr->in.sglist; + if (!sglist) + goto out_unmap; + + /* unmap iv */ + dma_unmap_single(dev, sglist->dma, sglist->len, DMA_BIDIRECTIONAL); + /* unmpa src sglist */ + dma_unmap_sg(dev, sr->in.buf, (sr->in.map_bufs_cnt - 1), sr->in.dir); + /* unamp gather component */ + dma_unmap_single(dev, sr->in.dma, sr->in.len, DMA_TO_DEVICE); + kfree(sr->in.sglist); + kfree(sr->in.sgcomp); + sr->in.sglist = NULL; + sr->in.buf = NULL; + sr->in.map_bufs_cnt = 0; + +out_unmap: + /* unmap out sgbuf */ + sglist = sr->out.sglist; + if (!sglist) + return; + + /* unmap orh */ + dma_unmap_single(dev, sr->resp.orh_dma, ORH_HLEN, sr->out.dir); + + /* unmap dst sglist */ + if (!sr->inplace) { + dma_unmap_sg(dev, sr->out.buf, (sr->out.map_bufs_cnt - 3), + sr->out.dir); + } + /* unmap completion */ + dma_unmap_single(dev, sr->resp.completion_dma, COMP_HLEN, sr->out.dir); + + /* unmap scatter component */ + dma_unmap_single(dev, sr->out.dma, sr->out.len, DMA_TO_DEVICE); + kfree(sr->out.sglist); + kfree(sr->out.sgcomp); + sr->out.sglist = NULL; + sr->out.buf = NULL; + sr->out.map_bufs_cnt = 0; +} + +static void softreq_destroy(struct nitrox_softreq *sr) +{ + softreq_unmap_sgbufs(sr); + kfree(sr); +} + +/** + * create_sg_component - create SG componets for N5 device. + * @sr: Request structure + * @sgtbl: SG table + * @nr_comp: total number of components required + * + * Component structure + * + * 63 48 47 32 31 16 15 0 + * -------------------------------------- + * | LEN0 | LEN1 | LEN2 | LEN3 | + * |------------------------------------- + * | PTR0 | + * -------------------------------------- + * | PTR1 | + * -------------------------------------- + * | PTR2 | + * -------------------------------------- + * | PTR3 | + * -------------------------------------- + * + * Returns 0 if success or a negative errno code on error. + */ +static int create_sg_component(struct nitrox_softreq *sr, + struct nitrox_sgtable *sgtbl, int map_nents) +{ + struct nitrox_device *ndev = sr->ndev; + struct nitrox_sgcomp *sgcomp; + struct nitrox_sglist *sglist; + dma_addr_t dma; + size_t sz_comp; + int i, j, nr_sgcomp; + + nr_sgcomp = roundup(map_nents, 4) / 4; + + /* each component holds 4 dma pointers */ + sz_comp = nr_sgcomp * sizeof(*sgcomp); + sgcomp = kzalloc(sz_comp, sr->gfp); + if (!sgcomp) + return -ENOMEM; + + sgtbl->sgcomp = sgcomp; + sgtbl->nr_sgcomp = nr_sgcomp; + + sglist = sgtbl->sglist; + /* populate device sg component */ + for (i = 0; i < nr_sgcomp; i++) { + for (j = 0; j < 4; j++) { + sgcomp->len[j] = cpu_to_be16(sglist->len); + sgcomp->dma[j] = cpu_to_be64(sglist->dma); + sglist++; + } + sgcomp++; + } + /* map the device sg component */ + dma = dma_map_single(DEV(ndev), sgtbl->sgcomp, sz_comp, DMA_TO_DEVICE); + if (dma_mapping_error(DEV(ndev), dma)) { + kfree(sgtbl->sgcomp); + sgtbl->sgcomp = NULL; + return -ENOMEM; + } + + sgtbl->dma = dma; + sgtbl->len = sz_comp; + + return 0; +} + +/** + * dma_map_inbufs - DMA map input sglist and creates sglist component + * for N5 device. + * @sr: Request structure + * @req: Crypto request structre + * + * Returns 0 if successful or a negative errno code on error. + */ +static int dma_map_inbufs(struct nitrox_softreq *sr, + struct se_crypto_request *req) +{ + struct device *dev = DEV(sr->ndev); + struct scatterlist *sg = req->src; + struct nitrox_sglist *glist; + int i, nents, ret = 0; + dma_addr_t dma; + size_t sz; + + nents = sg_nents(req->src); + + /* creater gather list IV and src entries */ + sz = roundup((1 + nents), 4) * sizeof(*glist); + glist = kzalloc(sz, sr->gfp); + if (!glist) + return -ENOMEM; + + sr->in.sglist = glist; + /* map IV */ + dma = dma_map_single(dev, &req->iv, req->ivsize, DMA_BIDIRECTIONAL); + if (dma_mapping_error(dev, dma)) { + ret = -EINVAL; + goto iv_map_err; + } + + sr->in.dir = (req->src == req->dst) ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE; + /* map src entries */ + nents = dma_map_sg(dev, req->src, nents, sr->in.dir); + if (!nents) { + ret = -EINVAL; + goto src_map_err; + } + sr->in.buf = req->src; + + /* store the mappings */ + glist->len = req->ivsize; + glist->dma = dma; + glist++; + sr->in.total_bytes += req->ivsize; + + for_each_sg(req->src, sg, nents, i) { + glist->len = sg_dma_len(sg); + glist->dma = sg_dma_address(sg); + sr->in.total_bytes += glist->len; + glist++; + } + /* roundup map count to align with entires in sg component */ + sr->in.map_bufs_cnt = (1 + nents); + + /* create NITROX gather component */ + ret = create_sg_component(sr, &sr->in, sr->in.map_bufs_cnt); + if (ret) + goto incomp_err; + + return 0; + +incomp_err: + dma_unmap_sg(dev, req->src, nents, sr->in.dir); + sr->in.map_bufs_cnt = 0; +src_map_err: + dma_unmap_single(dev, dma, req->ivsize, DMA_BIDIRECTIONAL); +iv_map_err: + kfree(sr->in.sglist); + sr->in.sglist = NULL; + return ret; +} + +static int dma_map_outbufs(struct nitrox_softreq *sr, + struct se_crypto_request *req) +{ + struct device *dev = DEV(sr->ndev); + struct nitrox_sglist *glist = sr->in.sglist; + struct nitrox_sglist *slist; + struct scatterlist *sg; + int i, nents, map_bufs_cnt, ret = 0; + size_t sz; + + nents = sg_nents(req->dst); + + /* create scatter list ORH, IV, dst entries and Completion header */ + sz = roundup((3 + nents), 4) * sizeof(*slist); + slist = kzalloc(sz, sr->gfp); + if (!slist) + return -ENOMEM; + + sr->out.sglist = slist; + sr->out.dir = DMA_BIDIRECTIONAL; + /* map ORH */ + sr->resp.orh_dma = dma_map_single(dev, &sr->resp.orh, ORH_HLEN, + sr->out.dir); + if (dma_mapping_error(dev, sr->resp.orh_dma)) { + ret = -EINVAL; + goto orh_map_err; + } + + /* map completion */ + sr->resp.completion_dma = dma_map_single(dev, &sr->resp.completion, + COMP_HLEN, sr->out.dir); + if (dma_mapping_error(dev, sr->resp.completion_dma)) { + ret = -EINVAL; + goto compl_map_err; + } + + sr->inplace = (req->src == req->dst) ? true : false; + /* out place */ + if (!sr->inplace) { + nents = dma_map_sg(dev, req->dst, nents, sr->out.dir); + if (!nents) { + ret = -EINVAL; + goto dst_map_err; + } + } + sr->out.buf = req->dst; + + /* store the mappings */ + /* orh */ + slist->len = ORH_HLEN; + slist->dma = sr->resp.orh_dma; + slist++; + + /* copy the glist mappings */ + if (sr->inplace) { + nents = sr->in.map_bufs_cnt - 1; + map_bufs_cnt = sr->in.map_bufs_cnt; + while (map_bufs_cnt--) { + slist->len = glist->len; + slist->dma = glist->dma; + slist++; + glist++; + } + } else { + /* copy iv mapping */ + slist->len = glist->len; + slist->dma = glist->dma; + slist++; + /* copy remaining maps */ + for_each_sg(req->dst, sg, nents, i) { + slist->len = sg_dma_len(sg); + slist->dma = sg_dma_address(sg); + slist++; + } + } + + /* completion */ + slist->len = COMP_HLEN; + slist->dma = sr->resp.completion_dma; + + sr->out.map_bufs_cnt = (3 + nents); + + ret = create_sg_component(sr, &sr->out, sr->out.map_bufs_cnt); + if (ret) + goto outcomp_map_err; + + return 0; + +outcomp_map_err: + if (!sr->inplace) + dma_unmap_sg(dev, req->dst, nents, sr->out.dir); + sr->out.map_bufs_cnt = 0; + sr->out.buf = NULL; +dst_map_err: + dma_unmap_single(dev, sr->resp.completion_dma, COMP_HLEN, sr->out.dir); + sr->resp.completion_dma = 0; +compl_map_err: + dma_unmap_single(dev, sr->resp.orh_dma, ORH_HLEN, sr->out.dir); + sr->resp.orh_dma = 0; +orh_map_err: + kfree(sr->out.sglist); + sr->out.sglist = NULL; + return ret; +} + +static inline int softreq_map_iobuf(struct nitrox_softreq *sr, + struct se_crypto_request *creq) +{ + int ret; + + ret = dma_map_inbufs(sr, creq); + if (ret) + return ret; + + ret = dma_map_outbufs(sr, creq); + if (ret) + softreq_unmap_sgbufs(sr); + + return ret; +} + +static inline void backlog_list_add(struct nitrox_softreq *sr, + struct nitrox_cmdq *cmdq) +{ + INIT_LIST_HEAD(&sr->backlog); + + spin_lock_bh(&cmdq->backlog_lock); + list_add_tail(&sr->backlog, &cmdq->backlog_head); + atomic_inc(&cmdq->backlog_count); + atomic_set(&sr->status, REQ_BACKLOG); + spin_unlock_bh(&cmdq->backlog_lock); +} + +static inline void response_list_add(struct nitrox_softreq *sr, + struct nitrox_cmdq *cmdq) +{ + INIT_LIST_HEAD(&sr->response); + + spin_lock_bh(&cmdq->response_lock); + list_add_tail(&sr->response, &cmdq->response_head); + spin_unlock_bh(&cmdq->response_lock); +} + +static inline void response_list_del(struct nitrox_softreq *sr, + struct nitrox_cmdq *cmdq) +{ + spin_lock_bh(&cmdq->response_lock); + list_del(&sr->response); + spin_unlock_bh(&cmdq->response_lock); +} + +static struct nitrox_softreq * +get_first_response_entry(struct nitrox_cmdq *cmdq) +{ + return list_first_entry_or_null(&cmdq->response_head, + struct nitrox_softreq, response); +} + +static inline bool cmdq_full(struct nitrox_cmdq *cmdq, int qlen) +{ + if (atomic_inc_return(&cmdq->pending_count) > qlen) { + atomic_dec(&cmdq->pending_count); + /* sync with other cpus */ + smp_mb__after_atomic(); + return true; + } + return false; +} + +/** + * post_se_instr - Post SE instruction to Packet Input ring + * @sr: Request structure + * + * Returns 0 if successful or a negative error code, + * if no space in ring. + */ +static void post_se_instr(struct nitrox_softreq *sr, + struct nitrox_cmdq *cmdq) +{ + struct nitrox_device *ndev = sr->ndev; + union nps_pkt_in_instr_baoff_dbell pkt_in_baoff_dbell; + u64 offset; + u8 *ent; + + spin_lock_bh(&cmdq->cmdq_lock); + + /* get the next write offset */ + offset = NPS_PKT_IN_INSTR_BAOFF_DBELLX(cmdq->qno); + pkt_in_baoff_dbell.value = nitrox_read_csr(ndev, offset); + /* copy the instruction */ + ent = cmdq->head + pkt_in_baoff_dbell.s.aoff; + memcpy(ent, &sr->instr, cmdq->instr_size); + /* flush the command queue updates */ + dma_wmb(); + + sr->tstamp = jiffies; + atomic_set(&sr->status, REQ_POSTED); + response_list_add(sr, cmdq); + + /* Ring doorbell with count 1 */ + writeq(1, cmdq->dbell_csr_addr); + /* orders the doorbell rings */ + mmiowb(); + + spin_unlock_bh(&cmdq->cmdq_lock); +} + +static int post_backlog_cmds(struct nitrox_cmdq *cmdq) +{ + struct nitrox_device *ndev = cmdq->ndev; + struct nitrox_softreq *sr, *tmp; + int ret = 0; + + spin_lock_bh(&cmdq->backlog_lock); + + list_for_each_entry_safe(sr, tmp, &cmdq->backlog_head, backlog) { + struct skcipher_request *skreq; + + /* submit until space available */ + if (unlikely(cmdq_full(cmdq, ndev->qlen))) { + ret = -EBUSY; + break; + } + /* delete from backlog list */ + list_del(&sr->backlog); + atomic_dec(&cmdq->backlog_count); + /* sync with other cpus */ + smp_mb__after_atomic(); + + skreq = sr->skreq; + /* post the command */ + post_se_instr(sr, cmdq); + + /* backlog requests are posted, wakeup with -EINPROGRESS */ + skcipher_request_complete(skreq, -EINPROGRESS); + } + spin_unlock_bh(&cmdq->backlog_lock); + + return ret; +} + +static int nitrox_enqueue_request(struct nitrox_softreq *sr) +{ + struct nitrox_cmdq *cmdq = sr->cmdq; + struct nitrox_device *ndev = sr->ndev; + int ret = -EBUSY; + + if (unlikely(cmdq_full(cmdq, ndev->qlen))) { + if (!(sr->flags & CRYPTO_TFM_REQ_MAY_BACKLOG)) + return -EAGAIN; + + backlog_list_add(sr, cmdq); + } else { + ret = post_backlog_cmds(cmdq); + if (ret) { + backlog_list_add(sr, cmdq); + return ret; + } + post_se_instr(sr, cmdq); + ret = -EINPROGRESS; + } + return ret; +} + +/** + * nitrox_se_request - Send request to SE core + * @ndev: NITROX device + * @req: Crypto request + * + * Returns 0 on success, or a negative error code. + */ +int nitrox_process_se_request(struct nitrox_device *ndev, + struct se_crypto_request *req, + completion_t callback, + struct skcipher_request *skreq) +{ + struct nitrox_softreq *sr; + dma_addr_t ctx_handle = 0; + int qno, ret = 0; + + if (!nitrox_ready(ndev)) + return -ENODEV; + + sr = kzalloc(sizeof(*sr), req->gfp); + if (!sr) + return -ENOMEM; + + sr->ndev = ndev; + sr->flags = req->flags; + sr->gfp = req->gfp; + sr->callback = callback; + sr->skreq = skreq; + + atomic_set(&sr->status, REQ_NOT_POSTED); + + WRITE_ONCE(sr->resp.orh, PENDING_SIG); + WRITE_ONCE(sr->resp.completion, PENDING_SIG); + + ret = softreq_map_iobuf(sr, req); + if (ret) { + kfree(sr); + return ret; + } + + /* get the context handle */ + if (req->ctx_handle) { + struct ctx_hdr *hdr; + u8 *ctx_ptr; + + ctx_ptr = (u8 *)(uintptr_t)req->ctx_handle; + hdr = (struct ctx_hdr *)(ctx_ptr - sizeof(struct ctx_hdr)); + ctx_handle = hdr->ctx_dma; + } + + /* select the queue */ + qno = smp_processor_id() % ndev->nr_queues; + + sr->cmdq = &ndev->pkt_cmdqs[qno]; + + /* + * 64-Byte Instruction Format + * + * ---------------------- + * | DPTR0 | 8 bytes + * ---------------------- + * | PKT_IN_INSTR_HDR | 8 bytes + * ---------------------- + * | PKT_IN_HDR | 16 bytes + * ---------------------- + * | SLC_INFO | 16 bytes + * ---------------------- + * | Front data | 16 bytes + * ---------------------- + */ + + /* fill the packet instruction */ + /* word 0 */ + sr->instr.dptr0 = cpu_to_be64(sr->in.dma); + + /* word 1 */ + sr->instr.ih.value = 0; + sr->instr.ih.s.g = 1; + sr->instr.ih.s.gsz = sr->in.map_bufs_cnt; + sr->instr.ih.s.ssz = sr->out.map_bufs_cnt; + sr->instr.ih.s.fsz = FDATA_SIZE + sizeof(struct gphdr); + sr->instr.ih.s.tlen = sr->instr.ih.s.fsz + sr->in.total_bytes; + sr->instr.ih.value = cpu_to_be64(sr->instr.ih.value); + + /* word 2 */ + sr->instr.irh.value[0] = 0; + sr->instr.irh.s.uddl = MIN_UDD_LEN; + /* context length in 64-bit words */ + sr->instr.irh.s.ctxl = (req->ctrl.s.ctxl / 8); + /* offset from solicit base port 256 */ + sr->instr.irh.s.destport = SOLICIT_BASE_DPORT + qno; + sr->instr.irh.s.ctxc = req->ctrl.s.ctxc; + sr->instr.irh.s.arg = req->ctrl.s.arg; + sr->instr.irh.s.opcode = req->opcode; + sr->instr.irh.value[0] = cpu_to_be64(sr->instr.irh.value[0]); + + /* word 3 */ + sr->instr.irh.s.ctxp = cpu_to_be64(ctx_handle); + + /* word 4 */ + sr->instr.slc.value[0] = 0; + sr->instr.slc.s.ssz = sr->out.map_bufs_cnt; + sr->instr.slc.value[0] = cpu_to_be64(sr->instr.slc.value[0]); + + /* word 5 */ + sr->instr.slc.s.rptr = cpu_to_be64(sr->out.dma); + + /* + * No conversion for front data, + * It goes into payload + * put GP Header in front data + */ + sr->instr.fdata[0] = *((u64 *)&req->gph); + sr->instr.fdata[1] = 0; + /* flush the soft_req changes before posting the cmd */ + wmb(); + + ret = nitrox_enqueue_request(sr); + if (ret == -EAGAIN) + goto send_fail; + + return ret; + +send_fail: + softreq_destroy(sr); + return ret; +} + +static inline int cmd_timeout(unsigned long tstamp, unsigned long timeout) +{ + return time_after_eq(jiffies, (tstamp + timeout)); +} + +void backlog_qflush_work(struct work_struct *work) +{ + struct nitrox_cmdq *cmdq; + + cmdq = container_of(work, struct nitrox_cmdq, backlog_qflush); + post_backlog_cmds(cmdq); +} + +/** + * process_request_list - process completed requests + * @ndev: N5 device + * @qno: queue to operate + * + * Returns the number of responses processed. + */ +static void process_response_list(struct nitrox_cmdq *cmdq) +{ + struct nitrox_device *ndev = cmdq->ndev; + struct nitrox_softreq *sr; + struct skcipher_request *skreq; + completion_t callback; + int req_completed = 0, err = 0, budget; + + /* check all pending requests */ + budget = atomic_read(&cmdq->pending_count); + + while (req_completed < budget) { + sr = get_first_response_entry(cmdq); + if (!sr) + break; + + if (atomic_read(&sr->status) != REQ_POSTED) + break; + + /* check orh and completion bytes updates */ + if (READ_ONCE(sr->resp.orh) == READ_ONCE(sr->resp.completion)) { + /* request not completed, check for timeout */ + if (!cmd_timeout(sr->tstamp, ndev->timeout)) + break; + dev_err_ratelimited(DEV(ndev), + "Request timeout, orh 0x%016llx\n", + READ_ONCE(sr->resp.orh)); + } + atomic_dec(&cmdq->pending_count); + /* sync with other cpus */ + smp_mb__after_atomic(); + /* remove from response list */ + response_list_del(sr, cmdq); + + callback = sr->callback; + skreq = sr->skreq; + + /* ORH error code */ + err = READ_ONCE(sr->resp.orh) & 0xff; + softreq_destroy(sr); + + if (callback) + callback(skreq, err); + + req_completed++; + } +} + +/** + * pkt_slc_resp_handler - post processing of SE responses + */ +void pkt_slc_resp_handler(unsigned long data) +{ + struct bh_data *bh = (void *)(uintptr_t)(data); + struct nitrox_cmdq *cmdq = bh->cmdq; + union nps_pkt_slc_cnts pkt_slc_cnts; + + /* read completion count */ + pkt_slc_cnts.value = readq(bh->completion_cnt_csr_addr); + /* resend the interrupt if more work to do */ + pkt_slc_cnts.s.resend = 1; + + process_response_list(cmdq); + + /* + * clear the interrupt with resend bit enabled, + * MSI-X interrupt generates if Completion count > Threshold + */ + writeq(pkt_slc_cnts.value, bh->completion_cnt_csr_addr); + /* order the writes */ + mmiowb(); + + if (atomic_read(&cmdq->backlog_count)) + schedule_work(&cmdq->backlog_qflush); +} |