diff options
Diffstat (limited to 'arch')
153 files changed, 7519 insertions, 3376 deletions
diff --git a/arch/arm/boot/dts/imx51.dtsi b/arch/arm/boot/dts/imx51.dtsi index 922adefdd291..53cbaa3d4f90 100644 --- a/arch/arm/boot/dts/imx51.dtsi +++ b/arch/arm/boot/dts/imx51.dtsi @@ -127,7 +127,7 @@ }; gpio1: gpio@73f84000 { - compatible = "fsl,imx51-gpio", "fsl,imx31-gpio"; + compatible = "fsl,imx51-gpio", "fsl,imx35-gpio"; reg = <0x73f84000 0x4000>; interrupts = <50 51>; gpio-controller; @@ -137,7 +137,7 @@ }; gpio2: gpio@73f88000 { - compatible = "fsl,imx51-gpio", "fsl,imx31-gpio"; + compatible = "fsl,imx51-gpio", "fsl,imx35-gpio"; reg = <0x73f88000 0x4000>; interrupts = <52 53>; gpio-controller; @@ -147,7 +147,7 @@ }; gpio3: gpio@73f8c000 { - compatible = "fsl,imx51-gpio", "fsl,imx31-gpio"; + compatible = "fsl,imx51-gpio", "fsl,imx35-gpio"; reg = <0x73f8c000 0x4000>; interrupts = <54 55>; gpio-controller; @@ -157,7 +157,7 @@ }; gpio4: gpio@73f90000 { - compatible = "fsl,imx51-gpio", "fsl,imx31-gpio"; + compatible = "fsl,imx51-gpio", "fsl,imx35-gpio"; reg = <0x73f90000 0x4000>; interrupts = <56 57>; gpio-controller; diff --git a/arch/arm/boot/dts/imx53.dtsi b/arch/arm/boot/dts/imx53.dtsi index 4e735edc78ed..fc79cdc4b4e6 100644 --- a/arch/arm/boot/dts/imx53.dtsi +++ b/arch/arm/boot/dts/imx53.dtsi @@ -129,7 +129,7 @@ }; gpio1: gpio@53f84000 { - compatible = "fsl,imx53-gpio", "fsl,imx31-gpio"; + compatible = "fsl,imx53-gpio", "fsl,imx35-gpio"; reg = <0x53f84000 0x4000>; interrupts = <50 51>; gpio-controller; @@ -139,7 +139,7 @@ }; gpio2: gpio@53f88000 { - compatible = "fsl,imx53-gpio", "fsl,imx31-gpio"; + compatible = "fsl,imx53-gpio", "fsl,imx35-gpio"; reg = <0x53f88000 0x4000>; interrupts = <52 53>; gpio-controller; @@ -149,7 +149,7 @@ }; gpio3: gpio@53f8c000 { - compatible = "fsl,imx53-gpio", "fsl,imx31-gpio"; + compatible = "fsl,imx53-gpio", "fsl,imx35-gpio"; reg = <0x53f8c000 0x4000>; interrupts = <54 55>; gpio-controller; @@ -159,7 +159,7 @@ }; gpio4: gpio@53f90000 { - compatible = "fsl,imx53-gpio", "fsl,imx31-gpio"; + compatible = "fsl,imx53-gpio", "fsl,imx35-gpio"; reg = <0x53f90000 0x4000>; interrupts = <56 57>; gpio-controller; @@ -197,7 +197,7 @@ }; gpio5: gpio@53fdc000 { - compatible = "fsl,imx53-gpio", "fsl,imx31-gpio"; + compatible = "fsl,imx53-gpio", "fsl,imx35-gpio"; reg = <0x53fdc000 0x4000>; interrupts = <103 104>; gpio-controller; @@ -207,7 +207,7 @@ }; gpio6: gpio@53fe0000 { - compatible = "fsl,imx53-gpio", "fsl,imx31-gpio"; + compatible = "fsl,imx53-gpio", "fsl,imx35-gpio"; reg = <0x53fe0000 0x4000>; interrupts = <105 106>; gpio-controller; @@ -217,7 +217,7 @@ }; gpio7: gpio@53fe4000 { - compatible = "fsl,imx53-gpio", "fsl,imx31-gpio"; + compatible = "fsl,imx53-gpio", "fsl,imx35-gpio"; reg = <0x53fe4000 0x4000>; interrupts = <107 108>; gpio-controller; diff --git a/arch/arm/boot/dts/imx6q.dtsi b/arch/arm/boot/dts/imx6q.dtsi index c25d49584814..3d3c64b014e6 100644 --- a/arch/arm/boot/dts/imx6q.dtsi +++ b/arch/arm/boot/dts/imx6q.dtsi @@ -277,7 +277,7 @@ }; gpio1: gpio@0209c000 { - compatible = "fsl,imx6q-gpio", "fsl,imx31-gpio"; + compatible = "fsl,imx6q-gpio", "fsl,imx35-gpio"; reg = <0x0209c000 0x4000>; interrupts = <0 66 0x04 0 67 0x04>; gpio-controller; @@ -287,7 +287,7 @@ }; gpio2: gpio@020a0000 { - compatible = "fsl,imx6q-gpio", "fsl,imx31-gpio"; + compatible = "fsl,imx6q-gpio", "fsl,imx35-gpio"; reg = <0x020a0000 0x4000>; interrupts = <0 68 0x04 0 69 0x04>; gpio-controller; @@ -297,7 +297,7 @@ }; gpio3: gpio@020a4000 { - compatible = "fsl,imx6q-gpio", "fsl,imx31-gpio"; + compatible = "fsl,imx6q-gpio", "fsl,imx35-gpio"; reg = <0x020a4000 0x4000>; interrupts = <0 70 0x04 0 71 0x04>; gpio-controller; @@ -307,7 +307,7 @@ }; gpio4: gpio@020a8000 { - compatible = "fsl,imx6q-gpio", "fsl,imx31-gpio"; + compatible = "fsl,imx6q-gpio", "fsl,imx35-gpio"; reg = <0x020a8000 0x4000>; interrupts = <0 72 0x04 0 73 0x04>; gpio-controller; @@ -317,7 +317,7 @@ }; gpio5: gpio@020ac000 { - compatible = "fsl,imx6q-gpio", "fsl,imx31-gpio"; + compatible = "fsl,imx6q-gpio", "fsl,imx35-gpio"; reg = <0x020ac000 0x4000>; interrupts = <0 74 0x04 0 75 0x04>; gpio-controller; @@ -327,7 +327,7 @@ }; gpio6: gpio@020b0000 { - compatible = "fsl,imx6q-gpio", "fsl,imx31-gpio"; + compatible = "fsl,imx6q-gpio", "fsl,imx35-gpio"; reg = <0x020b0000 0x4000>; interrupts = <0 76 0x04 0 77 0x04>; gpio-controller; @@ -337,7 +337,7 @@ }; gpio7: gpio@020b4000 { - compatible = "fsl,imx6q-gpio", "fsl,imx31-gpio"; + compatible = "fsl,imx6q-gpio", "fsl,imx35-gpio"; reg = <0x020b4000 0x4000>; interrupts = <0 78 0x04 0 79 0x04>; gpio-controller; diff --git a/arch/arm/mach-at91/at91sam9g45.c b/arch/arm/mach-at91/at91sam9g45.c index 977127368a7d..ef6cedd52e3c 100644 --- a/arch/arm/mach-at91/at91sam9g45.c +++ b/arch/arm/mach-at91/at91sam9g45.c @@ -183,6 +183,13 @@ static struct clk adc_op_clk = { .rate_hz = 13200000, }; +/* AES/TDES/SHA clock - Only for sam9m11/sam9g56 */ +static struct clk aestdessha_clk = { + .name = "aestdessha_clk", + .pmc_mask = 1 << AT91SAM9G45_ID_AESTDESSHA, + .type = CLK_TYPE_PERIPHERAL, +}; + static struct clk *periph_clocks[] __initdata = { &pioA_clk, &pioB_clk, @@ -212,6 +219,7 @@ static struct clk *periph_clocks[] __initdata = { &udphs_clk, &mmc1_clk, &adc_op_clk, + &aestdessha_clk, // irq0 }; @@ -232,6 +240,9 @@ static struct clk_lookup periph_clocks_lookups[] = { CLKDEV_CON_DEV_ID("pclk", "ssc.0", &ssc0_clk), CLKDEV_CON_DEV_ID("pclk", "ssc.1", &ssc1_clk), CLKDEV_CON_DEV_ID(NULL, "atmel-trng", &trng_clk), + CLKDEV_CON_DEV_ID(NULL, "atmel_sha", &aestdessha_clk), + CLKDEV_CON_DEV_ID(NULL, "atmel_tdes", &aestdessha_clk), + CLKDEV_CON_DEV_ID(NULL, "atmel_aes", &aestdessha_clk), /* more usart lookup table for DT entries */ CLKDEV_CON_DEV_ID("usart", "ffffee00.serial", &mck), CLKDEV_CON_DEV_ID("usart", "fff8c000.serial", &usart0_clk), @@ -388,7 +399,7 @@ static unsigned int at91sam9g45_default_irq_priority[NR_AIC_IRQS] __initdata = { 3, /* Ethernet */ 0, /* Image Sensor Interface */ 2, /* USB Device High speed port */ - 0, + 0, /* AESTDESSHA Crypto HW Accelerators */ 0, /* Multimedia Card Interface 1 */ 0, 0, /* Advanced Interrupt Controller (IRQ0) */ diff --git a/arch/arm/mach-at91/at91sam9g45_devices.c b/arch/arm/mach-at91/at91sam9g45_devices.c index 40fb79df2de0..06073996a382 100644 --- a/arch/arm/mach-at91/at91sam9g45_devices.c +++ b/arch/arm/mach-at91/at91sam9g45_devices.c @@ -18,6 +18,7 @@ #include <linux/platform_device.h> #include <linux/i2c-gpio.h> #include <linux/atmel-mci.h> +#include <linux/platform_data/atmel-aes.h> #include <linux/platform_data/at91_adc.h> @@ -1830,6 +1831,130 @@ void __init at91_register_uart(unsigned id, unsigned portnr, unsigned pins) {} void __init at91_add_device_serial(void) {} #endif +/* -------------------------------------------------------------------- + * SHA1/SHA256 + * -------------------------------------------------------------------- */ + +#if defined(CONFIG_CRYPTO_DEV_ATMEL_SHA) || defined(CONFIG_CRYPTO_DEV_ATMEL_SHA_MODULE) +static struct resource sha_resources[] = { + { + .start = AT91SAM9G45_BASE_SHA, + .end = AT91SAM9G45_BASE_SHA + SZ_16K - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = AT91SAM9G45_ID_AESTDESSHA, + .end = AT91SAM9G45_ID_AESTDESSHA, + .flags = IORESOURCE_IRQ, + }, +}; + +static struct platform_device at91sam9g45_sha_device = { + .name = "atmel_sha", + .id = -1, + .resource = sha_resources, + .num_resources = ARRAY_SIZE(sha_resources), +}; + +static void __init at91_add_device_sha(void) +{ + platform_device_register(&at91sam9g45_sha_device); +} +#else +static void __init at91_add_device_sha(void) {} +#endif + +/* -------------------------------------------------------------------- + * DES/TDES + * -------------------------------------------------------------------- */ + +#if defined(CONFIG_CRYPTO_DEV_ATMEL_TDES) || defined(CONFIG_CRYPTO_DEV_ATMEL_TDES_MODULE) +static struct resource tdes_resources[] = { + [0] = { + .start = AT91SAM9G45_BASE_TDES, + .end = AT91SAM9G45_BASE_TDES + SZ_16K - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = AT91SAM9G45_ID_AESTDESSHA, + .end = AT91SAM9G45_ID_AESTDESSHA, + .flags = IORESOURCE_IRQ, + }, +}; + +static struct platform_device at91sam9g45_tdes_device = { + .name = "atmel_tdes", + .id = -1, + .resource = tdes_resources, + .num_resources = ARRAY_SIZE(tdes_resources), +}; + +static void __init at91_add_device_tdes(void) +{ + platform_device_register(&at91sam9g45_tdes_device); +} +#else +static void __init at91_add_device_tdes(void) {} +#endif + +/* -------------------------------------------------------------------- + * AES + * -------------------------------------------------------------------- */ + +#if defined(CONFIG_CRYPTO_DEV_ATMEL_AES) || defined(CONFIG_CRYPTO_DEV_ATMEL_AES_MODULE) +static struct aes_platform_data aes_data; +static u64 aes_dmamask = DMA_BIT_MASK(32); + +static struct resource aes_resources[] = { + [0] = { + .start = AT91SAM9G45_BASE_AES, + .end = AT91SAM9G45_BASE_AES + SZ_16K - 1, + .flags = IORESOURCE_MEM, + }, + [1] = { + .start = AT91SAM9G45_ID_AESTDESSHA, + .end = AT91SAM9G45_ID_AESTDESSHA, + .flags = IORESOURCE_IRQ, + }, +}; + +static struct platform_device at91sam9g45_aes_device = { + .name = "atmel_aes", + .id = -1, + .dev = { + .dma_mask = &aes_dmamask, + .coherent_dma_mask = DMA_BIT_MASK(32), + .platform_data = &aes_data, + }, + .resource = aes_resources, + .num_resources = ARRAY_SIZE(aes_resources), +}; + +static void __init at91_add_device_aes(void) +{ + struct at_dma_slave *atslave; + struct aes_dma_data *alt_atslave; + + alt_atslave = kzalloc(sizeof(struct aes_dma_data), GFP_KERNEL); + + /* DMA TX slave channel configuration */ + atslave = &alt_atslave->txdata; + atslave->dma_dev = &at_hdmac_device.dev; + atslave->cfg = ATC_FIFOCFG_ENOUGHSPACE | ATC_SRC_H2SEL_HW | + ATC_SRC_PER(AT_DMA_ID_AES_RX); + + /* DMA RX slave channel configuration */ + atslave = &alt_atslave->rxdata; + atslave->dma_dev = &at_hdmac_device.dev; + atslave->cfg = ATC_FIFOCFG_ENOUGHSPACE | ATC_DST_H2SEL_HW | + ATC_DST_PER(AT_DMA_ID_AES_TX); + + aes_data.dma_slave = alt_atslave; + platform_device_register(&at91sam9g45_aes_device); +} +#else +static void __init at91_add_device_aes(void) {} +#endif /* -------------------------------------------------------------------- */ /* @@ -1847,6 +1972,9 @@ static int __init at91_add_standard_devices(void) at91_add_device_trng(); at91_add_device_watchdog(); at91_add_device_tc(); + at91_add_device_sha(); + at91_add_device_tdes(); + at91_add_device_aes(); return 0; } diff --git a/arch/arm/mach-at91/include/mach/at91sam9g45.h b/arch/arm/mach-at91/include/mach/at91sam9g45.h index 3a4da24d5911..8eba1021f533 100644 --- a/arch/arm/mach-at91/include/mach/at91sam9g45.h +++ b/arch/arm/mach-at91/include/mach/at91sam9g45.h @@ -136,6 +136,8 @@ #define AT_DMA_ID_SSC1_RX 8 #define AT_DMA_ID_AC97_TX 9 #define AT_DMA_ID_AC97_RX 10 +#define AT_DMA_ID_AES_TX 11 +#define AT_DMA_ID_AES_RX 12 #define AT_DMA_ID_MCI1 13 #endif diff --git a/arch/arm/mach-imx/mm-imx25.c b/arch/arm/mach-imx/mm-imx25.c index 388928fdb11a..f3f5c6542ab4 100644 --- a/arch/arm/mach-imx/mm-imx25.c +++ b/arch/arm/mach-imx/mm-imx25.c @@ -89,11 +89,11 @@ static const struct resource imx25_audmux_res[] __initconst = { void __init imx25_soc_init(void) { - /* i.mx25 has the i.mx31 type gpio */ - mxc_register_gpio("imx31-gpio", 0, MX25_GPIO1_BASE_ADDR, SZ_16K, MX25_INT_GPIO1, 0); - mxc_register_gpio("imx31-gpio", 1, MX25_GPIO2_BASE_ADDR, SZ_16K, MX25_INT_GPIO2, 0); - mxc_register_gpio("imx31-gpio", 2, MX25_GPIO3_BASE_ADDR, SZ_16K, MX25_INT_GPIO3, 0); - mxc_register_gpio("imx31-gpio", 3, MX25_GPIO4_BASE_ADDR, SZ_16K, MX25_INT_GPIO4, 0); + /* i.mx25 has the i.mx35 type gpio */ + mxc_register_gpio("imx35-gpio", 0, MX25_GPIO1_BASE_ADDR, SZ_16K, MX25_INT_GPIO1, 0); + mxc_register_gpio("imx35-gpio", 1, MX25_GPIO2_BASE_ADDR, SZ_16K, MX25_INT_GPIO2, 0); + mxc_register_gpio("imx35-gpio", 2, MX25_GPIO3_BASE_ADDR, SZ_16K, MX25_INT_GPIO3, 0); + mxc_register_gpio("imx35-gpio", 3, MX25_GPIO4_BASE_ADDR, SZ_16K, MX25_INT_GPIO4, 0); pinctrl_provide_dummies(); /* i.mx25 has the i.mx35 type sdma */ diff --git a/arch/arm/mach-imx/mm-imx3.c b/arch/arm/mach-imx/mm-imx3.c index fe96105109b3..9d2c843bde02 100644 --- a/arch/arm/mach-imx/mm-imx3.c +++ b/arch/arm/mach-imx/mm-imx3.c @@ -272,10 +272,9 @@ void __init imx35_soc_init(void) imx3_init_l2x0(); - /* i.mx35 has the i.mx31 type gpio */ - mxc_register_gpio("imx31-gpio", 0, MX35_GPIO1_BASE_ADDR, SZ_16K, MX35_INT_GPIO1, 0); - mxc_register_gpio("imx31-gpio", 1, MX35_GPIO2_BASE_ADDR, SZ_16K, MX35_INT_GPIO2, 0); - mxc_register_gpio("imx31-gpio", 2, MX35_GPIO3_BASE_ADDR, SZ_16K, MX35_INT_GPIO3, 0); + mxc_register_gpio("imx35-gpio", 0, MX35_GPIO1_BASE_ADDR, SZ_16K, MX35_INT_GPIO1, 0); + mxc_register_gpio("imx35-gpio", 1, MX35_GPIO2_BASE_ADDR, SZ_16K, MX35_INT_GPIO2, 0); + mxc_register_gpio("imx35-gpio", 2, MX35_GPIO3_BASE_ADDR, SZ_16K, MX35_INT_GPIO3, 0); pinctrl_provide_dummies(); if (to_version == 1) { diff --git a/arch/arm/mach-imx/mm-imx5.c b/arch/arm/mach-imx/mm-imx5.c index f19d604e1b2a..52d8f534be10 100644 --- a/arch/arm/mach-imx/mm-imx5.c +++ b/arch/arm/mach-imx/mm-imx5.c @@ -161,13 +161,13 @@ static const struct resource imx53_audmux_res[] __initconst = { void __init imx50_soc_init(void) { - /* i.mx50 has the i.mx31 type gpio */ - mxc_register_gpio("imx31-gpio", 0, MX50_GPIO1_BASE_ADDR, SZ_16K, MX50_INT_GPIO1_LOW, MX50_INT_GPIO1_HIGH); - mxc_register_gpio("imx31-gpio", 1, MX50_GPIO2_BASE_ADDR, SZ_16K, MX50_INT_GPIO2_LOW, MX50_INT_GPIO2_HIGH); - mxc_register_gpio("imx31-gpio", 2, MX50_GPIO3_BASE_ADDR, SZ_16K, MX50_INT_GPIO3_LOW, MX50_INT_GPIO3_HIGH); - mxc_register_gpio("imx31-gpio", 3, MX50_GPIO4_BASE_ADDR, SZ_16K, MX50_INT_GPIO4_LOW, MX50_INT_GPIO4_HIGH); - mxc_register_gpio("imx31-gpio", 4, MX50_GPIO5_BASE_ADDR, SZ_16K, MX50_INT_GPIO5_LOW, MX50_INT_GPIO5_HIGH); - mxc_register_gpio("imx31-gpio", 5, MX50_GPIO6_BASE_ADDR, SZ_16K, MX50_INT_GPIO6_LOW, MX50_INT_GPIO6_HIGH); + /* i.mx50 has the i.mx35 type gpio */ + mxc_register_gpio("imx35-gpio", 0, MX50_GPIO1_BASE_ADDR, SZ_16K, MX50_INT_GPIO1_LOW, MX50_INT_GPIO1_HIGH); + mxc_register_gpio("imx35-gpio", 1, MX50_GPIO2_BASE_ADDR, SZ_16K, MX50_INT_GPIO2_LOW, MX50_INT_GPIO2_HIGH); + mxc_register_gpio("imx35-gpio", 2, MX50_GPIO3_BASE_ADDR, SZ_16K, MX50_INT_GPIO3_LOW, MX50_INT_GPIO3_HIGH); + mxc_register_gpio("imx35-gpio", 3, MX50_GPIO4_BASE_ADDR, SZ_16K, MX50_INT_GPIO4_LOW, MX50_INT_GPIO4_HIGH); + mxc_register_gpio("imx35-gpio", 4, MX50_GPIO5_BASE_ADDR, SZ_16K, MX50_INT_GPIO5_LOW, MX50_INT_GPIO5_HIGH); + mxc_register_gpio("imx35-gpio", 5, MX50_GPIO6_BASE_ADDR, SZ_16K, MX50_INT_GPIO6_LOW, MX50_INT_GPIO6_HIGH); /* i.mx50 has the i.mx31 type audmux */ platform_device_register_simple("imx31-audmux", 0, imx50_audmux_res, @@ -176,11 +176,11 @@ void __init imx50_soc_init(void) void __init imx51_soc_init(void) { - /* i.mx51 has the i.mx31 type gpio */ - mxc_register_gpio("imx31-gpio", 0, MX51_GPIO1_BASE_ADDR, SZ_16K, MX51_INT_GPIO1_LOW, MX51_INT_GPIO1_HIGH); - mxc_register_gpio("imx31-gpio", 1, MX51_GPIO2_BASE_ADDR, SZ_16K, MX51_INT_GPIO2_LOW, MX51_INT_GPIO2_HIGH); - mxc_register_gpio("imx31-gpio", 2, MX51_GPIO3_BASE_ADDR, SZ_16K, MX51_INT_GPIO3_LOW, MX51_INT_GPIO3_HIGH); - mxc_register_gpio("imx31-gpio", 3, MX51_GPIO4_BASE_ADDR, SZ_16K, MX51_INT_GPIO4_LOW, MX51_INT_GPIO4_HIGH); + /* i.mx51 has the i.mx35 type gpio */ + mxc_register_gpio("imx35-gpio", 0, MX51_GPIO1_BASE_ADDR, SZ_16K, MX51_INT_GPIO1_LOW, MX51_INT_GPIO1_HIGH); + mxc_register_gpio("imx35-gpio", 1, MX51_GPIO2_BASE_ADDR, SZ_16K, MX51_INT_GPIO2_LOW, MX51_INT_GPIO2_HIGH); + mxc_register_gpio("imx35-gpio", 2, MX51_GPIO3_BASE_ADDR, SZ_16K, MX51_INT_GPIO3_LOW, MX51_INT_GPIO3_HIGH); + mxc_register_gpio("imx35-gpio", 3, MX51_GPIO4_BASE_ADDR, SZ_16K, MX51_INT_GPIO4_LOW, MX51_INT_GPIO4_HIGH); pinctrl_provide_dummies(); @@ -198,14 +198,14 @@ void __init imx51_soc_init(void) void __init imx53_soc_init(void) { - /* i.mx53 has the i.mx31 type gpio */ - mxc_register_gpio("imx31-gpio", 0, MX53_GPIO1_BASE_ADDR, SZ_16K, MX53_INT_GPIO1_LOW, MX53_INT_GPIO1_HIGH); - mxc_register_gpio("imx31-gpio", 1, MX53_GPIO2_BASE_ADDR, SZ_16K, MX53_INT_GPIO2_LOW, MX53_INT_GPIO2_HIGH); - mxc_register_gpio("imx31-gpio", 2, MX53_GPIO3_BASE_ADDR, SZ_16K, MX53_INT_GPIO3_LOW, MX53_INT_GPIO3_HIGH); - mxc_register_gpio("imx31-gpio", 3, MX53_GPIO4_BASE_ADDR, SZ_16K, MX53_INT_GPIO4_LOW, MX53_INT_GPIO4_HIGH); - mxc_register_gpio("imx31-gpio", 4, MX53_GPIO5_BASE_ADDR, SZ_16K, MX53_INT_GPIO5_LOW, MX53_INT_GPIO5_HIGH); - mxc_register_gpio("imx31-gpio", 5, MX53_GPIO6_BASE_ADDR, SZ_16K, MX53_INT_GPIO6_LOW, MX53_INT_GPIO6_HIGH); - mxc_register_gpio("imx31-gpio", 6, MX53_GPIO7_BASE_ADDR, SZ_16K, MX53_INT_GPIO7_LOW, MX53_INT_GPIO7_HIGH); + /* i.mx53 has the i.mx35 type gpio */ + mxc_register_gpio("imx35-gpio", 0, MX53_GPIO1_BASE_ADDR, SZ_16K, MX53_INT_GPIO1_LOW, MX53_INT_GPIO1_HIGH); + mxc_register_gpio("imx35-gpio", 1, MX53_GPIO2_BASE_ADDR, SZ_16K, MX53_INT_GPIO2_LOW, MX53_INT_GPIO2_HIGH); + mxc_register_gpio("imx35-gpio", 2, MX53_GPIO3_BASE_ADDR, SZ_16K, MX53_INT_GPIO3_LOW, MX53_INT_GPIO3_HIGH); + mxc_register_gpio("imx35-gpio", 3, MX53_GPIO4_BASE_ADDR, SZ_16K, MX53_INT_GPIO4_LOW, MX53_INT_GPIO4_HIGH); + mxc_register_gpio("imx35-gpio", 4, MX53_GPIO5_BASE_ADDR, SZ_16K, MX53_INT_GPIO5_LOW, MX53_INT_GPIO5_HIGH); + mxc_register_gpio("imx35-gpio", 5, MX53_GPIO6_BASE_ADDR, SZ_16K, MX53_INT_GPIO6_LOW, MX53_INT_GPIO6_HIGH); + mxc_register_gpio("imx35-gpio", 6, MX53_GPIO7_BASE_ADDR, SZ_16K, MX53_INT_GPIO7_LOW, MX53_INT_GPIO7_HIGH); pinctrl_provide_dummies(); /* i.mx53 has the i.mx35 type sdma */ diff --git a/arch/arm/mach-omap2/clock3xxx_data.c b/arch/arm/mach-omap2/clock3xxx_data.c index 91b3d5c60bfe..83bed9ad3017 100644 --- a/arch/arm/mach-omap2/clock3xxx_data.c +++ b/arch/arm/mach-omap2/clock3xxx_data.c @@ -3376,15 +3376,15 @@ static struct omap_clk omap3xxx_clks[] = { CLK(NULL, "usbhost_48m_fck", &usbhost_48m_fck, CK_3430ES2PLUS | CK_AM35XX | CK_36XX), CLK(NULL, "usbhost_ick", &usbhost_ick, CK_3430ES2PLUS | CK_AM35XX | CK_36XX), CLK("usbhs_omap", "usbhost_ick", &usbhost_ick, CK_3430ES2PLUS | CK_AM35XX | CK_36XX), - CLK("usbhs_omap", "utmi_p1_gfclk", &dummy_ck, CK_3XXX), - CLK("usbhs_omap", "utmi_p2_gfclk", &dummy_ck, CK_3XXX), - CLK("usbhs_omap", "xclk60mhsp1_ck", &dummy_ck, CK_3XXX), - CLK("usbhs_omap", "xclk60mhsp2_ck", &dummy_ck, CK_3XXX), - CLK("usbhs_omap", "usb_host_hs_utmi_p1_clk", &dummy_ck, CK_3XXX), - CLK("usbhs_omap", "usb_host_hs_utmi_p2_clk", &dummy_ck, CK_3XXX), + CLK(NULL, "utmi_p1_gfclk", &dummy_ck, CK_3XXX), + CLK(NULL, "utmi_p2_gfclk", &dummy_ck, CK_3XXX), + CLK(NULL, "xclk60mhsp1_ck", &dummy_ck, CK_3XXX), + CLK(NULL, "xclk60mhsp2_ck", &dummy_ck, CK_3XXX), + CLK(NULL, "usb_host_hs_utmi_p1_clk", &dummy_ck, CK_3XXX), + CLK(NULL, "usb_host_hs_utmi_p2_clk", &dummy_ck, CK_3XXX), CLK("usbhs_omap", "usb_tll_hs_usb_ch0_clk", &dummy_ck, CK_3XXX), CLK("usbhs_omap", "usb_tll_hs_usb_ch1_clk", &dummy_ck, CK_3XXX), - CLK("usbhs_omap", "init_60m_fclk", &dummy_ck, CK_3XXX), + CLK(NULL, "init_60m_fclk", &dummy_ck, CK_3XXX), CLK(NULL, "usim_fck", &usim_fck, CK_3430ES2PLUS | CK_36XX), CLK(NULL, "gpt1_fck", &gpt1_fck, CK_3XXX), CLK(NULL, "wkup_32k_fck", &wkup_32k_fck, CK_3XXX), diff --git a/arch/arm/plat-spear/include/plat/keyboard.h b/arch/arm/plat-spear/include/plat/keyboard.h index 0562f134621d..9248e3a7e333 100644 --- a/arch/arm/plat-spear/include/plat/keyboard.h +++ b/arch/arm/plat-spear/include/plat/keyboard.h @@ -149,6 +149,7 @@ int _name[] = { \ * keymap: pointer to keymap data (table and size) * rep: enables key autorepeat * mode: choose keyboard support(9x9, 6x6, 2x2) + * suspended_rate: rate at which keyboard would operate in suspended mode * * This structure is supposed to be used by platform code to supply * keymaps to drivers that implement keyboards. @@ -157,6 +158,7 @@ struct kbd_platform_data { const struct matrix_keymap_data *keymap; bool rep; unsigned int mode; + unsigned int suspended_rate; }; #endif /* __PLAT_KEYBOARD_H */ diff --git a/arch/m68k/Kconfig.bus b/arch/m68k/Kconfig.bus index 3adb499584fb..ffc0601a2a19 100644 --- a/arch/m68k/Kconfig.bus +++ b/arch/m68k/Kconfig.bus @@ -48,6 +48,13 @@ config ISA config GENERIC_ISA_DMA def_bool ISA +config PCI + bool "PCI support" + depends on M54xx + help + Enable the PCI bus. Support for the PCI bus hardware built into the + ColdFire 547x and 548x processors. + source "drivers/pci/Kconfig" source "drivers/zorro/Kconfig" diff --git a/arch/m68k/Kconfig.cpu b/arch/m68k/Kconfig.cpu index 2b53254ad994..43a9f8f1b8eb 100644 --- a/arch/m68k/Kconfig.cpu +++ b/arch/m68k/Kconfig.cpu @@ -23,7 +23,7 @@ config M68KCLASSIC config COLDFIRE bool "Coldfire CPU family support" select GENERIC_GPIO - select ARCH_REQUIRE_GPIOLIB + select ARCH_WANT_OPTIONAL_GPIOLIB select ARCH_HAVE_CUSTOM_GPIO_H select CPU_HAS_NO_BITFIELDS select CPU_HAS_NO_MULDIV64 @@ -167,6 +167,14 @@ config M5249 help Motorola ColdFire 5249 processor support. +config M525x + bool "MCF525x" + depends on !MMU + select COLDFIRE_SW_A7 + select HAVE_MBAR + help + Freescale (Motorola) Coldfire 5251/5253 processor support. + config M527x bool @@ -253,6 +261,14 @@ config M548x help Freescale ColdFire 5480/5481/5482/5483/5484/5485 processor support. +config M5441x + bool "MCF5441x" + depends on !MMU + select GENERIC_CLOCKEVENTS + select HAVE_CACHE_CB + help + Freescale Coldfire 54410/54415/54416/54417/54418 processor support. + endif # COLDFIRE diff --git a/arch/m68k/Makefile b/arch/m68k/Makefile index b7f2e2d5cd2e..7636751f2f87 100644 --- a/arch/m68k/Makefile +++ b/arch/m68k/Makefile @@ -41,6 +41,7 @@ cpuflags-$(CONFIG_M68030) := cpuflags-$(CONFIG_M68020) := cpuflags-$(CONFIG_M68360) := -m68332 cpuflags-$(CONFIG_M68000) := -m68000 +cpuflags-$(CONFIG_M5441x) := $(call cc-option,-mcpu=54455,-mcfv4e) cpuflags-$(CONFIG_M54xx) := $(call cc-option,-mcpu=5475,-m5200) cpuflags-$(CONFIG_M5407) := $(call cc-option,-mcpu=5407,-m5200) cpuflags-$(CONFIG_M532x) := $(call cc-option,-mcpu=532x,-m5307) @@ -50,6 +51,7 @@ cpuflags-$(CONFIG_M5275) := $(call cc-option,-mcpu=5275,-m5307) cpuflags-$(CONFIG_M5272) := $(call cc-option,-mcpu=5272,-m5307) cpuflags-$(CONFIG_M5271) := $(call cc-option,-mcpu=5271,-m5307) cpuflags-$(CONFIG_M523x) := $(call cc-option,-mcpu=523x,-m5307) +cpuflags-$(CONFIG_M525x) := $(call cc-option,-mcpu=5253,-m5200) cpuflags-$(CONFIG_M5249) := $(call cc-option,-mcpu=5249,-m5200) cpuflags-$(CONFIG_M520x) := $(call cc-option,-mcpu=5208,-m5200) cpuflags-$(CONFIG_M5206e) := $(call cc-option,-mcpu=5206e,-m5200) diff --git a/arch/m68k/include/asm/cacheflush_mm.h b/arch/m68k/include/asm/cacheflush_mm.h index 8104bd874649..fa2c3d681d84 100644 --- a/arch/m68k/include/asm/cacheflush_mm.h +++ b/arch/m68k/include/asm/cacheflush_mm.h @@ -16,7 +16,48 @@ #define DCACHE_MAX_ADDR 0 #define DCACHE_SETMASK 0 #endif +#ifndef CACHE_MODE +#define CACHE_MODE 0 +#define CACR_ICINVA 0 +#define CACR_DCINVA 0 +#define CACR_BCINVA 0 +#endif + +/* + * ColdFire architecture has no way to clear individual cache lines, so we + * are stuck invalidating all the cache entries when we want a clear operation. + */ +static inline void clear_cf_icache(unsigned long start, unsigned long end) +{ + __asm__ __volatile__ ( + "movec %0,%%cacr\n\t" + "nop" + : + : "r" (CACHE_MODE | CACR_ICINVA | CACR_BCINVA)); +} + +static inline void clear_cf_dcache(unsigned long start, unsigned long end) +{ + __asm__ __volatile__ ( + "movec %0,%%cacr\n\t" + "nop" + : + : "r" (CACHE_MODE | CACR_DCINVA)); +} +static inline void clear_cf_bcache(unsigned long start, unsigned long end) +{ + __asm__ __volatile__ ( + "movec %0,%%cacr\n\t" + "nop" + : + : "r" (CACHE_MODE | CACR_ICINVA | CACR_BCINVA | CACR_DCINVA)); +} + +/* + * Use the ColdFire cpushl instruction to push (and invalidate) cache lines. + * The start and end addresses are cache line numbers not memory addresses. + */ static inline void flush_cf_icache(unsigned long start, unsigned long end) { unsigned long set; diff --git a/arch/m68k/include/asm/dma.h b/arch/m68k/include/asm/dma.h index 6fbdfe895104..0ff3fc6a6d9a 100644 --- a/arch/m68k/include/asm/dma.h +++ b/arch/m68k/include/asm/dma.h @@ -33,7 +33,9 @@ * Set number of channels of DMA on ColdFire for different implementations. */ #if defined(CONFIG_M5249) || defined(CONFIG_M5307) || defined(CONFIG_M5407) || \ - defined(CONFIG_M523x) || defined(CONFIG_M527x) || defined(CONFIG_M528x) + defined(CONFIG_M523x) || defined(CONFIG_M527x) || \ + defined(CONFIG_M528x) || defined(CONFIG_M525x) + #define MAX_M68K_DMA_CHANNELS 4 #elif defined(CONFIG_M5272) #define MAX_M68K_DMA_CHANNELS 1 @@ -486,6 +488,10 @@ static __inline__ int get_dma_residue(unsigned int dmanr) extern int request_dma(unsigned int dmanr, const char * device_id); /* reserve a DMA channel */ extern void free_dma(unsigned int dmanr); /* release it again */ +#ifdef CONFIG_PCI +extern int isa_dma_bridge_buggy; +#else #define isa_dma_bridge_buggy (0) +#endif #endif /* _M68K_DMA_H */ diff --git a/arch/m68k/include/asm/gpio.h b/arch/m68k/include/asm/gpio.h index 00d0071de4c3..4395ffc51fdb 100644 --- a/arch/m68k/include/asm/gpio.h +++ b/arch/m68k/include/asm/gpio.h @@ -17,170 +17,9 @@ #define coldfire_gpio_h #include <linux/io.h> -#include <asm-generic/gpio.h> #include <asm/coldfire.h> #include <asm/mcfsim.h> - -/* - * The Freescale Coldfire family is quite varied in how they implement GPIO. - * Some parts have 8 bit ports, some have 16bit and some have 32bit; some have - * only one port, others have multiple ports; some have a single data latch - * for both input and output, others have a separate pin data register to read - * input; some require a read-modify-write access to change an output, others - * have set and clear registers for some of the outputs; Some have all the - * GPIOs in a single control area, others have some GPIOs implemented in - * different modules. - * - * This implementation attempts accommodate the differences while presenting - * a generic interface that will optimize to as few instructions as possible. - */ -#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \ - defined(CONFIG_M520x) || defined(CONFIG_M523x) || \ - defined(CONFIG_M527x) || defined(CONFIG_M528x) || \ - defined(CONFIG_M532x) || defined(CONFIG_M54xx) - -/* These parts have GPIO organized by 8 bit ports */ - -#define MCFGPIO_PORTTYPE u8 -#define MCFGPIO_PORTSIZE 8 -#define mcfgpio_read(port) __raw_readb(port) -#define mcfgpio_write(data, port) __raw_writeb(data, port) - -#elif defined(CONFIG_M5307) || defined(CONFIG_M5407) || defined(CONFIG_M5272) - -/* These parts have GPIO organized by 16 bit ports */ - -#define MCFGPIO_PORTTYPE u16 -#define MCFGPIO_PORTSIZE 16 -#define mcfgpio_read(port) __raw_readw(port) -#define mcfgpio_write(data, port) __raw_writew(data, port) - -#elif defined(CONFIG_M5249) - -/* These parts have GPIO organized by 32 bit ports */ - -#define MCFGPIO_PORTTYPE u32 -#define MCFGPIO_PORTSIZE 32 -#define mcfgpio_read(port) __raw_readl(port) -#define mcfgpio_write(data, port) __raw_writel(data, port) - -#endif - -#define mcfgpio_bit(gpio) (1 << ((gpio) % MCFGPIO_PORTSIZE)) -#define mcfgpio_port(gpio) ((gpio) / MCFGPIO_PORTSIZE) - -#if defined(CONFIG_M520x) || defined(CONFIG_M523x) || \ - defined(CONFIG_M527x) || defined(CONFIG_M528x) || defined(CONFIG_M532x) -/* - * These parts have an 'Edge' Port module (external interrupt/GPIO) which uses - * read-modify-write to change an output and a GPIO module which has separate - * set/clr registers to directly change outputs with a single write access. - */ -#if defined(CONFIG_M528x) -/* - * The 528x also has GPIOs in other modules (GPT, QADC) which use - * read-modify-write as well as those controlled by the EPORT and GPIO modules. - */ -#define MCFGPIO_SCR_START 40 -#else -#define MCFGPIO_SCR_START 8 -#endif - -#define MCFGPIO_SETR_PORT(gpio) (MCFGPIO_SETR + \ - mcfgpio_port(gpio - MCFGPIO_SCR_START)) - -#define MCFGPIO_CLRR_PORT(gpio) (MCFGPIO_CLRR + \ - mcfgpio_port(gpio - MCFGPIO_SCR_START)) -#else - -#define MCFGPIO_SCR_START MCFGPIO_PIN_MAX -/* with MCFGPIO_SCR == MCFGPIO_PIN_MAX, these will be optimized away */ -#define MCFGPIO_SETR_PORT(gpio) 0 -#define MCFGPIO_CLRR_PORT(gpio) 0 - -#endif -/* - * Coldfire specific helper functions - */ - -/* return the port pin data register for a gpio */ -static inline u32 __mcf_gpio_ppdr(unsigned gpio) -{ -#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \ - defined(CONFIG_M5307) || defined(CONFIG_M5407) - return MCFSIM_PADAT; -#elif defined(CONFIG_M5272) - if (gpio < 16) - return MCFSIM_PADAT; - else if (gpio < 32) - return MCFSIM_PBDAT; - else - return MCFSIM_PCDAT; -#elif defined(CONFIG_M5249) - if (gpio < 32) - return MCFSIM2_GPIOREAD; - else - return MCFSIM2_GPIO1READ; -#elif defined(CONFIG_M520x) || defined(CONFIG_M523x) || \ - defined(CONFIG_M527x) || defined(CONFIG_M528x) || defined(CONFIG_M532x) - if (gpio < 8) - return MCFEPORT_EPPDR; -#if defined(CONFIG_M528x) - else if (gpio < 16) - return MCFGPTA_GPTPORT; - else if (gpio < 24) - return MCFGPTB_GPTPORT; - else if (gpio < 32) - return MCFQADC_PORTQA; - else if (gpio < 40) - return MCFQADC_PORTQB; -#endif - else - return MCFGPIO_PPDR + mcfgpio_port(gpio - MCFGPIO_SCR_START); -#else - return 0; -#endif -} - -/* return the port output data register for a gpio */ -static inline u32 __mcf_gpio_podr(unsigned gpio) -{ -#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \ - defined(CONFIG_M5307) || defined(CONFIG_M5407) - return MCFSIM_PADAT; -#elif defined(CONFIG_M5272) - if (gpio < 16) - return MCFSIM_PADAT; - else if (gpio < 32) - return MCFSIM_PBDAT; - else - return MCFSIM_PCDAT; -#elif defined(CONFIG_M5249) - if (gpio < 32) - return MCFSIM2_GPIOWRITE; - else - return MCFSIM2_GPIO1WRITE; -#elif defined(CONFIG_M520x) || defined(CONFIG_M523x) || \ - defined(CONFIG_M527x) || defined(CONFIG_M528x) || defined(CONFIG_M532x) - if (gpio < 8) - return MCFEPORT_EPDR; -#if defined(CONFIG_M528x) - else if (gpio < 16) - return MCFGPTA_GPTPORT; - else if (gpio < 24) - return MCFGPTB_GPTPORT; - else if (gpio < 32) - return MCFQADC_PORTQA; - else if (gpio < 40) - return MCFQADC_PORTQB; -#endif - else - return MCFGPIO_PODR + mcfgpio_port(gpio - MCFGPIO_SCR_START); -#else - return 0; -#endif -} - +#include <asm/mcfgpio.h> /* * The Generic GPIO functions * @@ -191,7 +30,7 @@ static inline u32 __mcf_gpio_podr(unsigned gpio) static inline int gpio_get_value(unsigned gpio) { if (__builtin_constant_p(gpio) && gpio < MCFGPIO_PIN_MAX) - return mcfgpio_read(__mcf_gpio_ppdr(gpio)) & mcfgpio_bit(gpio); + return mcfgpio_read(__mcfgpio_ppdr(gpio)) & mcfgpio_bit(gpio); else return __gpio_get_value(gpio); } @@ -204,12 +43,12 @@ static inline void gpio_set_value(unsigned gpio, int value) MCFGPIO_PORTTYPE data; local_irq_save(flags); - data = mcfgpio_read(__mcf_gpio_podr(gpio)); + data = mcfgpio_read(__mcfgpio_podr(gpio)); if (value) data |= mcfgpio_bit(gpio); else data &= ~mcfgpio_bit(gpio); - mcfgpio_write(data, __mcf_gpio_podr(gpio)); + mcfgpio_write(data, __mcfgpio_podr(gpio)); local_irq_restore(flags); } else { if (value) @@ -225,8 +64,14 @@ static inline void gpio_set_value(unsigned gpio, int value) static inline int gpio_to_irq(unsigned gpio) { - return (gpio < MCFGPIO_IRQ_MAX) ? gpio + MCFGPIO_IRQ_VECBASE - : __gpio_to_irq(gpio); +#if defined(MCFGPIO_IRQ_MIN) + if ((gpio >= MCFGPIO_IRQ_MIN) && (gpio < MCFGPIO_IRQ_MAX)) +#else + if (gpio < MCFGPIO_IRQ_MAX) +#endif + return gpio + MCFGPIO_IRQ_VECBASE; + else + return __gpio_to_irq(gpio); } static inline int irq_to_gpio(unsigned irq) diff --git a/arch/m68k/include/asm/io_mm.h b/arch/m68k/include/asm/io_mm.h index fa4324bcf566..a6686d26fe17 100644 --- a/arch/m68k/include/asm/io_mm.h +++ b/arch/m68k/include/asm/io_mm.h @@ -65,7 +65,53 @@ -#ifdef CONFIG_ISA +#if defined(CONFIG_PCI) && defined(CONFIG_COLDFIRE) + +#define HAVE_ARCH_PIO_SIZE +#define PIO_OFFSET 0 +#define PIO_MASK 0xffff +#define PIO_RESERVED 0x10000 + +u8 mcf_pci_inb(u32 addr); +u16 mcf_pci_inw(u32 addr); +u32 mcf_pci_inl(u32 addr); +void mcf_pci_insb(u32 addr, u8 *buf, u32 len); +void mcf_pci_insw(u32 addr, u16 *buf, u32 len); +void mcf_pci_insl(u32 addr, u32 *buf, u32 len); + +void mcf_pci_outb(u8 v, u32 addr); +void mcf_pci_outw(u16 v, u32 addr); +void mcf_pci_outl(u32 v, u32 addr); +void mcf_pci_outsb(u32 addr, const u8 *buf, u32 len); +void mcf_pci_outsw(u32 addr, const u16 *buf, u32 len); +void mcf_pci_outsl(u32 addr, const u32 *buf, u32 len); + +#define inb mcf_pci_inb +#define inb_p mcf_pci_inb +#define inw mcf_pci_inw +#define inw_p mcf_pci_inw +#define inl mcf_pci_inl +#define inl_p mcf_pci_inl +#define insb mcf_pci_insb +#define insw mcf_pci_insw +#define insl mcf_pci_insl + +#define outb mcf_pci_outb +#define outb_p mcf_pci_outb +#define outw mcf_pci_outw +#define outw_p mcf_pci_outw +#define outl mcf_pci_outl +#define outl_p mcf_pci_outl +#define outsb mcf_pci_outsb +#define outsw mcf_pci_outsw +#define outsl mcf_pci_outsl + +#define readb(addr) in_8(addr) +#define writeb(v, addr) out_8((addr), (v)) +#define readw(addr) in_le16(addr) +#define writew(v, addr) out_le16((addr), (v)) + +#elif defined(CONFIG_ISA) #if MULTI_ISA == 0 #undef MULTI_ISA @@ -340,4 +386,6 @@ static inline void memcpy_toio(volatile void __iomem *dst, const void *src, int */ #define xlate_dev_kmem_ptr(p) p +#define ioport_map(port, nr) ((void __iomem *)(port)) + #endif /* _IO_H */ diff --git a/arch/m68k/include/asm/m520xsim.h b/arch/m68k/include/asm/m520xsim.h index 17f2aab9cf97..db3f8ee4a6c6 100644 --- a/arch/m68k/include/asm/m520xsim.h +++ b/arch/m68k/include/asm/m520xsim.h @@ -42,6 +42,9 @@ #define MCFINTC1_SIMR (0) #define MCFINTC1_CIMR (0) #define MCFINTC1_ICR0 (0) +#define MCFINTC2_SIMR (0) +#define MCFINTC2_CIMR (0) +#define MCFINTC2_ICR0 (0) #define MCFINT_VECBASE 64 #define MCFINT_UART0 26 /* Interrupt number for UART0 */ @@ -62,6 +65,7 @@ #define MCF_IRQ_FECENTC0 (MCFINT_VECBASE + MCFINT_FECENTC0) #define MCF_IRQ_QSPI (MCFINT_VECBASE + MCFINT_QSPI) +#define MCF_IRQ_PIT1 (MCFINT_VECBASE + MCFINT_PIT1) /* * SDRAM configuration registers. @@ -186,5 +190,15 @@ #define MCF_RCR_SWRESET 0x80 /* Software reset bit */ #define MCF_RCR_FRCSTOUT 0x40 /* Force external reset */ +/* + * Power Management. + */ +#define MCFPM_WCR 0xfc040013 +#define MCFPM_PPMSR0 0xfc04002c +#define MCFPM_PPMCR0 0xfc04002d +#define MCFPM_PPMHR0 0xfc040030 +#define MCFPM_PPMLR0 0xfc040034 +#define MCFPM_LPCR 0xfc0a0007 + /****************************************************************************/ #endif /* m520xsim_h */ diff --git a/arch/m68k/include/asm/m523xsim.h b/arch/m68k/include/asm/m523xsim.h index 075062d4eecd..91d3abc3f2a5 100644 --- a/arch/m68k/include/asm/m523xsim.h +++ b/arch/m68k/include/asm/m523xsim.h @@ -52,6 +52,7 @@ #define MCF_IRQ_FECENTC0 (MCFINT_VECBASE + MCFINT_FECENTC0) #define MCF_IRQ_QSPI (MCFINT_VECBASE + MCFINT_QSPI) +#define MCF_IRQ_PIT1 (MCFINT_VECBASE + MCFINT_PIT1) /* * SDRAM configuration registers. diff --git a/arch/m68k/include/asm/m525xsim.h b/arch/m68k/include/asm/m525xsim.h new file mode 100644 index 000000000000..6da24f653902 --- /dev/null +++ b/arch/m68k/include/asm/m525xsim.h @@ -0,0 +1,194 @@ +/****************************************************************************/ + +/* + * m525xsim.h -- ColdFire 525x System Integration Module support. + * + * (C) Copyright 2012, Steven king <sfking@fdwdc.com> + * (C) Copyright 2002, Greg Ungerer (gerg@snapgear.com) + */ + +/****************************************************************************/ +#ifndef m525xsim_h +#define m525xsim_h +/****************************************************************************/ + +#define CPU_NAME "COLDFIRE(m525x)" +#define CPU_INSTR_PER_JIFFY 3 +#define MCF_BUSCLK (MCF_CLK / 2) + +#include <asm/m52xxacr.h> + +/* + * The 525x has a second MBAR region, define its address. + */ +#define MCF_MBAR2 0x80000000 + +/* + * Define the 525x SIM register set addresses. + */ +#define MCFSIM_RSR 0x00 /* Reset Status reg (r/w) */ +#define MCFSIM_SYPCR 0x01 /* System Protection reg (r/w)*/ +#define MCFSIM_SWIVR 0x02 /* SW Watchdog intr reg (r/w) */ +#define MCFSIM_SWSR 0x03 /* SW Watchdog service (r/w) */ +#define MCFSIM_MPARK 0x0C /* BUS Master Control Reg*/ +#define MCFSIM_IPR 0x40 /* Interrupt Pend reg (r/w) */ +#define MCFSIM_IMR 0x44 /* Interrupt Mask reg (r/w) */ +#define MCFSIM_ICR0 0x4c /* Intr Ctrl reg 0 (r/w) */ +#define MCFSIM_ICR1 0x4d /* Intr Ctrl reg 1 (r/w) */ +#define MCFSIM_ICR2 0x4e /* Intr Ctrl reg 2 (r/w) */ +#define MCFSIM_ICR3 0x4f /* Intr Ctrl reg 3 (r/w) */ +#define MCFSIM_ICR4 0x50 /* Intr Ctrl reg 4 (r/w) */ +#define MCFSIM_ICR5 0x51 /* Intr Ctrl reg 5 (r/w) */ +#define MCFSIM_ICR6 0x52 /* Intr Ctrl reg 6 (r/w) */ +#define MCFSIM_ICR7 0x53 /* Intr Ctrl reg 7 (r/w) */ +#define MCFSIM_ICR8 0x54 /* Intr Ctrl reg 8 (r/w) */ +#define MCFSIM_ICR9 0x55 /* Intr Ctrl reg 9 (r/w) */ +#define MCFSIM_ICR10 0x56 /* Intr Ctrl reg 10 (r/w) */ +#define MCFSIM_ICR11 0x57 /* Intr Ctrl reg 11 (r/w) */ + +#define MCFSIM_CSAR0 0x80 /* CS 0 Address 0 reg (r/w) */ +#define MCFSIM_CSMR0 0x84 /* CS 0 Mask 0 reg (r/w) */ +#define MCFSIM_CSCR0 0x8a /* CS 0 Control reg (r/w) */ +#define MCFSIM_CSAR1 0x8c /* CS 1 Address reg (r/w) */ +#define MCFSIM_CSMR1 0x90 /* CS 1 Mask reg (r/w) */ +#define MCFSIM_CSCR1 0x96 /* CS 1 Control reg (r/w) */ +#define MCFSIM_CSAR2 0x98 /* CS 2 Address reg (r/w) */ +#define MCFSIM_CSMR2 0x9c /* CS 2 Mask reg (r/w) */ +#define MCFSIM_CSCR2 0xa2 /* CS 2 Control reg (r/w) */ +#define MCFSIM_CSAR3 0xa4 /* CS 3 Address reg (r/w) */ +#define MCFSIM_CSMR3 0xa8 /* CS 3 Mask reg (r/w) */ +#define MCFSIM_CSCR3 0xae /* CS 3 Control reg (r/w) */ +#define MCFSIM_CSAR4 0xb0 /* CS 4 Address reg (r/w) */ +#define MCFSIM_CSMR4 0xb4 /* CS 4 Mask reg (r/w) */ +#define MCFSIM_CSCR4 0xba /* CS 4 Control reg (r/w) */ + +#define MCFSIM_DCR (MCF_MBAR + 0x100) /* DRAM Control */ +#define MCFSIM_DACR0 (MCF_MBAR + 0x108) /* DRAM 0 Addr/Ctrl */ +#define MCFSIM_DMR0 (MCF_MBAR + 0x10c) /* DRAM 0 Mask */ + +/* + * Secondary Interrupt Controller (in MBAR2) +*/ +#define MCFINTC2_INTBASE (MCF_MBAR2 + 0x168) /* Base Vector Reg */ +#define MCFINTC2_INTPRI1 (MCF_MBAR2 + 0x140) /* 0-7 priority */ +#define MCFINTC2_INTPRI2 (MCF_MBAR2 + 0x144) /* 8-15 priority */ +#define MCFINTC2_INTPRI3 (MCF_MBAR2 + 0x148) /* 16-23 priority */ +#define MCFINTC2_INTPRI4 (MCF_MBAR2 + 0x14c) /* 24-31 priority */ +#define MCFINTC2_INTPRI5 (MCF_MBAR2 + 0x150) /* 32-39 priority */ +#define MCFINTC2_INTPRI6 (MCF_MBAR2 + 0x154) /* 40-47 priority */ +#define MCFINTC2_INTPRI7 (MCF_MBAR2 + 0x158) /* 48-55 priority */ +#define MCFINTC2_INTPRI8 (MCF_MBAR2 + 0x15c) /* 56-63 priority */ + +#define MCFINTC2_INTPRI_REG(i) (MCFINTC2_INTPRI1 + \ + ((((i) - MCFINTC2_VECBASE) / 8) * 4)) +#define MCFINTC2_INTPRI_BITS(b, i) ((b) << (((i) % 8) * 4)) + +/* + * Timer module. + */ +#define MCFTIMER_BASE1 (MCF_MBAR + 0x140) /* Base of TIMER1 */ +#define MCFTIMER_BASE2 (MCF_MBAR + 0x180) /* Base of TIMER2 */ + +/* + * UART module. + */ +#define MCFUART_BASE0 (MCF_MBAR + 0x1c0) /* Base address UART0 */ +#define MCFUART_BASE1 (MCF_MBAR + 0x200) /* Base address UART1 */ + +/* + * QSPI module. + */ +#define MCFQSPI_BASE (MCF_MBAR + 0x300) /* Base address QSPI */ +#define MCFQSPI_SIZE 0x40 /* Register set size */ + + +#define MCFQSPI_CS0 15 +#define MCFQSPI_CS1 16 +#define MCFQSPI_CS2 24 +#define MCFQSPI_CS3 28 + +/* + * I2C module. + */ +#define MCFI2C_BASE0 (MCF_MBAR + 0x280) /* Base addreess I2C0 */ +#define MCFI2C_SIZE0 0x20 /* Register set size */ + +#define MCFI2C_BASE1 (MCF_MBAR2 + 0x440) /* Base addreess I2C1 */ +#define MCFI2C_SIZE1 0x20 /* Register set size */ +/* + * DMA unit base addresses. + */ +#define MCFDMA_BASE0 (MCF_MBAR + 0x300) /* Base address DMA 0 */ +#define MCFDMA_BASE1 (MCF_MBAR + 0x340) /* Base address DMA 1 */ +#define MCFDMA_BASE2 (MCF_MBAR + 0x380) /* Base address DMA 2 */ +#define MCFDMA_BASE3 (MCF_MBAR + 0x3C0) /* Base address DMA 3 */ + +/* + * Some symbol defines for the above... + */ +#define MCFSIM_SWDICR MCFSIM_ICR0 /* Watchdog timer ICR */ +#define MCFSIM_TIMER1ICR MCFSIM_ICR1 /* Timer 1 ICR */ +#define MCFSIM_TIMER2ICR MCFSIM_ICR2 /* Timer 2 ICR */ +#define MCFSIM_I2CICR MCFSIM_ICR3 /* I2C ICR */ +#define MCFSIM_UART1ICR MCFSIM_ICR4 /* UART 1 ICR */ +#define MCFSIM_UART2ICR MCFSIM_ICR5 /* UART 2 ICR */ +#define MCFSIM_DMA0ICR MCFSIM_ICR6 /* DMA 0 ICR */ +#define MCFSIM_DMA1ICR MCFSIM_ICR7 /* DMA 1 ICR */ +#define MCFSIM_DMA2ICR MCFSIM_ICR8 /* DMA 2 ICR */ +#define MCFSIM_DMA3ICR MCFSIM_ICR9 /* DMA 3 ICR */ +#define MCFSIM_QSPIICR MCFSIM_ICR10 /* QSPI ICR */ + +/* + * Define system peripheral IRQ usage. + */ +#define MCF_IRQ_QSPI 28 /* QSPI, Level 4 */ +#define MCF_IRQ_I2C0 29 +#define MCF_IRQ_TIMER 30 /* Timer0, Level 6 */ +#define MCF_IRQ_PROFILER 31 /* Timer1, Level 7 */ + +#define MCF_IRQ_UART0 73 /* UART0 */ +#define MCF_IRQ_UART1 74 /* UART1 */ + +/* + * Define the base interrupt for the second interrupt controller. + * We set it to 128, out of the way of the base interrupts, and plenty + * of room for its 64 interrupts. + */ +#define MCFINTC2_VECBASE 128 + +#define MCF_IRQ_GPIO0 (MCFINTC2_VECBASE + 32) +#define MCF_IRQ_GPIO1 (MCFINTC2_VECBASE + 33) +#define MCF_IRQ_GPIO2 (MCFINTC2_VECBASE + 34) +#define MCF_IRQ_GPIO3 (MCFINTC2_VECBASE + 35) +#define MCF_IRQ_GPIO4 (MCFINTC2_VECBASE + 36) +#define MCF_IRQ_GPIO5 (MCFINTC2_VECBASE + 37) +#define MCF_IRQ_GPIO6 (MCFINTC2_VECBASE + 38) + +#define MCF_IRQ_USBWUP (MCFINTC2_VECBASE + 40) +#define MCF_IRQ_I2C1 (MCFINTC2_VECBASE + 62) + +/* + * General purpose IO registers (in MBAR2). + */ +#define MCFSIM2_GPIOREAD (MCF_MBAR2 + 0x000) /* GPIO read values */ +#define MCFSIM2_GPIOWRITE (MCF_MBAR2 + 0x004) /* GPIO write values */ +#define MCFSIM2_GPIOENABLE (MCF_MBAR2 + 0x008) /* GPIO enabled */ +#define MCFSIM2_GPIOFUNC (MCF_MBAR2 + 0x00C) /* GPIO function */ +#define MCFSIM2_GPIO1READ (MCF_MBAR2 + 0x0B0) /* GPIO1 read values */ +#define MCFSIM2_GPIO1WRITE (MCF_MBAR2 + 0x0B4) /* GPIO1 write values */ +#define MCFSIM2_GPIO1ENABLE (MCF_MBAR2 + 0x0B8) /* GPIO1 enabled */ +#define MCFSIM2_GPIO1FUNC (MCF_MBAR2 + 0x0BC) /* GPIO1 function */ + +#define MCFSIM2_GPIOINTSTAT (MCF_MBAR2 + 0xc0) /* GPIO intr status */ +#define MCFSIM2_GPIOINTCLEAR (MCF_MBAR2 + 0xc0) /* GPIO intr clear */ +#define MCFSIM2_GPIOINTENABLE (MCF_MBAR2 + 0xc4) /* GPIO intr enable */ + +/* + * Generic GPIO support + */ +#define MCFGPIO_PIN_MAX 64 +#define MCFGPIO_IRQ_MAX 7 +#define MCFGPIO_IRQ_VECBASE MCF_IRQ_GPIO0 + +/****************************************************************************/ +#endif /* m525xsim_h */ diff --git a/arch/m68k/include/asm/m527xsim.h b/arch/m68k/include/asm/m527xsim.h index 83db8106f50a..71aa5104d3d6 100644 --- a/arch/m68k/include/asm/m527xsim.h +++ b/arch/m68k/include/asm/m527xsim.h @@ -60,6 +60,7 @@ #define MCF_IRQ_FECENTC1 (MCFINT2_VECBASE + MCFINT2_FECENTC1) #define MCF_IRQ_QSPI (MCFINT_VECBASE + MCFINT_QSPI) +#define MCF_IRQ_PIT1 (MCFINT_VECBASE + MCFINT_PIT1) /* * SDRAM configuration registers. diff --git a/arch/m68k/include/asm/m528xsim.h b/arch/m68k/include/asm/m528xsim.h index 497c31c803ff..4acb3c0a642e 100644 --- a/arch/m68k/include/asm/m528xsim.h +++ b/arch/m68k/include/asm/m528xsim.h @@ -52,7 +52,7 @@ #define MCF_IRQ_FECENTC0 (MCFINT_VECBASE + MCFINT_FECENTC0) #define MCF_IRQ_QSPI (MCFINT_VECBASE + MCFINT_QSPI) - +#define MCF_IRQ_PIT1 (MCFINT_VECBASE + MCFINT_PIT1) /* * SDRAM configuration registers. */ diff --git a/arch/m68k/include/asm/m532xsim.h b/arch/m68k/include/asm/m532xsim.h index 29b66e21413a..5ca7b298c6eb 100644 --- a/arch/m68k/include/asm/m532xsim.h +++ b/arch/m68k/include/asm/m532xsim.h @@ -82,6 +82,9 @@ #define MCFINTC1_SIMR 0xFC04C01C #define MCFINTC1_CIMR 0xFC04C01D #define MCFINTC1_ICR0 0xFC04C040 +#define MCFINTC2_SIMR (0) +#define MCFINTC2_CIMR (0) +#define MCFINTC2_ICR0 (0) #define MCFSIM_ICR_TIMER1 (0xFC048040+32) #define MCFSIM_ICR_TIMER2 (0xFC048040+33) @@ -135,6 +138,20 @@ #define MCF_RCR_SWRESET 0x80 /* Software reset bit */ #define MCF_RCR_FRCSTOUT 0x40 /* Force external reset */ + +/* + * Power Management + */ +#define MCFPM_WCR 0xfc040013 +#define MCFPM_PPMSR0 0xfc04002c +#define MCFPM_PPMCR0 0xfc04002d +#define MCFPM_PPMSR1 0xfc04002e +#define MCFPM_PPMCR1 0xfc04002f +#define MCFPM_PPMHR0 0xfc040030 +#define MCFPM_PPMLR0 0xfc040034 +#define MCFPM_PPMHR1 0xfc040038 +#define MCFPM_LPCR 0xec090007 + /********************************************************************* * * Inter-IC (I2C) Module diff --git a/arch/m68k/include/asm/m5441xsim.h b/arch/m68k/include/asm/m5441xsim.h new file mode 100644 index 000000000000..cc798ab9524b --- /dev/null +++ b/arch/m68k/include/asm/m5441xsim.h @@ -0,0 +1,276 @@ +/* + * m5441xsim.h -- Coldfire 5441x register definitions + * + * (C) Copyright 2012, Steven King <sfking@fdwdc.com> +*/ + +#ifndef m5441xsim_h +#define m5441xsim_h + +#define CPU_NAME "COLDFIRE(m5441x)" +#define CPU_INSTR_PER_JIFFY 2 +#define MCF_BUSCLK (MCF_CLK / 2) + +#include <asm/m54xxacr.h> + +/* + * Reset Controller Module. + */ + +#define MCF_RCR 0xec090000 +#define MCF_RSR 0xec090001 + +#define MCF_RCR_SWRESET 0x80 /* Software reset bit */ +#define MCF_RCR_FRCSTOUT 0x40 /* Force external reset */ + +/* + * Interrupt Controller Modules. + */ +/* the 5441x have 3 interrupt controllers, each control 64 interrupts */ +#define MCFINT_VECBASE 64 +#define MCFINT0_VECBASE MCFINT_VECBASE +#define MCFINT1_VECBASE (MCFINT0_VECBASE + 64) +#define MCFINT2_VECBASE (MCFINT1_VECBASE + 64) + +/* interrupt controller 0 */ +#define MCFINTC0_SIMR 0xfc04801c +#define MCFINTC0_CIMR 0xfc04801d +#define MCFINTC0_ICR0 0xfc048040 +/* interrupt controller 1 */ +#define MCFINTC1_SIMR 0xfc04c01c +#define MCFINTC1_CIMR 0xfc04c01d +#define MCFINTC1_ICR0 0xfc04c040 +/* interrupt controller 2 */ +#define MCFINTC2_SIMR 0xfc05001c +#define MCFINTC2_CIMR 0xfc05001d +#define MCFINTC2_ICR0 0xfc050040 + +/* on interrupt controller 0 */ +#define MCFINT0_EPORT0 1 +#define MCFINT0_UART0 26 +#define MCFINT0_UART1 27 +#define MCFINT0_UART2 28 +#define MCFINT0_UART3 29 +#define MCFINT0_I2C0 30 +#define MCFINT0_DSPI0 31 + +#define MCFINT0_TIMER0 32 +#define MCFINT0_TIMER1 33 +#define MCFINT0_TIMER2 34 +#define MCFINT0_TIMER3 35 + +#define MCFINT0_FECRX0 36 +#define MCFINT0_FECTX0 40 +#define MCFINT0_FECENTC0 42 + +#define MCFINT0_FECRX1 49 +#define MCFINT0_FECTX1 53 +#define MCFINT0_FECENTC1 55 + +/* on interrupt controller 1 */ +#define MCFINT1_UART4 48 +#define MCFINT1_UART5 49 +#define MCFINT1_UART6 50 +#define MCFINT1_UART7 51 +#define MCFINT1_UART8 52 +#define MCFINT1_UART9 53 +#define MCFINT1_DSPI1 54 +#define MCFINT1_DSPI2 55 +#define MCFINT1_DSPI3 56 +#define MCFINT1_I2C1 57 +#define MCFINT1_I2C2 58 +#define MCFINT1_I2C3 59 +#define MCFINT1_I2C4 60 +#define MCFINT1_I2C5 61 + +/* on interrupt controller 2 */ +#define MCFINT2_PIT0 13 +#define MCFINT2_PIT1 14 +#define MCFINT2_PIT2 15 +#define MCFINT2_PIT3 16 +#define MCFINT2_RTC 26 + +/* + * PIT timer module. + */ +#define MCFPIT_BASE0 0xFC080000 /* Base address of TIMER0 */ +#define MCFPIT_BASE1 0xFC084000 /* Base address of TIMER1 */ +#define MCFPIT_BASE2 0xFC088000 /* Base address of TIMER2 */ +#define MCFPIT_BASE3 0xFC08C000 /* Base address of TIMER3 */ + + +#define MCF_IRQ_PIT1 (MCFINT2_VECBASE + MCFINT2_PIT1) + +/* + * Power Management + */ +#define MCFPM_WCR 0xfc040013 +#define MCFPM_PPMSR0 0xfc04002c +#define MCFPM_PPMCR0 0xfc04002d +#define MCFPM_PPMSR1 0xfc04002e +#define MCFPM_PPMCR1 0xfc04002f +#define MCFPM_PPMHR0 0xfc040030 +#define MCFPM_PPMLR0 0xfc040034 +#define MCFPM_PPMHR1 0xfc040038 +#define MCFPM_PPMLR1 0xfc04003c +#define MCFPM_LPCR 0xec090007 +/* + * UART module. + */ +#define MCFUART_BASE0 0xfc060000 /* Base address of UART0 */ +#define MCFUART_BASE1 0xfc064000 /* Base address of UART1 */ +#define MCFUART_BASE2 0xfc068000 /* Base address of UART2 */ +#define MCFUART_BASE3 0xfc06c000 /* Base address of UART3 */ +#define MCFUART_BASE4 0xec060000 /* Base address of UART4 */ +#define MCFUART_BASE5 0xec064000 /* Base address of UART5 */ +#define MCFUART_BASE6 0xec068000 /* Base address of UART6 */ +#define MCFUART_BASE7 0xec06c000 /* Base address of UART7 */ +#define MCFUART_BASE8 0xec070000 /* Base address of UART8 */ +#define MCFUART_BASE9 0xec074000 /* Base address of UART9 */ + +#define MCF_IRQ_UART0 (MCFINT0_VECBASE + MCFINT0_UART0) +#define MCF_IRQ_UART1 (MCFINT0_VECBASE + MCFINT0_UART1) +#define MCF_IRQ_UART2 (MCFINT0_VECBASE + MCFINT0_UART2) +#define MCF_IRQ_UART3 (MCFINT0_VECBASE + MCFINT0_UART3) +#define MCF_IRQ_UART4 (MCFINT1_VECBASE + MCFINT1_UART4) +#define MCF_IRQ_UART5 (MCFINT1_VECBASE + MCFINT1_UART5) +#define MCF_IRQ_UART6 (MCFINT1_VECBASE + MCFINT1_UART6) +#define MCF_IRQ_UART7 (MCFINT1_VECBASE + MCFINT1_UART7) +#define MCF_IRQ_UART8 (MCFINT1_VECBASE + MCFINT1_UART8) +#define MCF_IRQ_UART9 (MCFINT1_VECBASE + MCFINT1_UART9) +/* + * FEC modules. + */ +#define MCFFEC_BASE0 0xfc0d4000 +#define MCFFEC_SIZE0 0x800 +#define MCF_IRQ_FECRX0 (MCFINT0_VECBASE + MCFINT0_FECRX0) +#define MCF_IRQ_FECTX0 (MCFINT0_VECBASE + MCFINT0_FECTX0) +#define MCF_IRQ_FECENTC0 (MCFINT0_VECBASE + MCFINT0_FECENTC0) + +#define MCFFEC_BASE1 0xfc0d8000 +#define MCFFEC_SIZE1 0x800 +#define MCF_IRQ_FECRX1 (MCFINT0_VECBASE + MCFINT0_FECRX1) +#define MCF_IRQ_FECTX1 (MCFINT0_VECBASE + MCFINT0_FECTX1) +#define MCF_IRQ_FECENTC1 (MCFINT0_VECBASE + MCFINT0_FECENTC1) +/* + * I2C modules. + */ +#define MCFI2C_BASE0 0xfc058000 +#define MCFI2C_SIZE0 0x20 +#define MCFI2C_BASE1 0xfc038000 +#define MCFI2C_SIZE1 0x20 +#define MCFI2C_BASE2 0xec010000 +#define MCFI2C_SIZE2 0x20 +#define MCFI2C_BASE3 0xec014000 +#define MCFI2C_SIZE3 0x20 +#define MCFI2C_BASE4 0xec018000 +#define MCFI2C_SIZE4 0x20 +#define MCFI2C_BASE5 0xec01c000 +#define MCFI2C_SIZE5 0x20 + +#define MCF_IRQ_I2C0 (MCFINT0_VECBASE + MCFINT0_I2C0) +#define MCF_IRQ_I2C1 (MCFINT1_VECBASE + MCFINT1_I2C1) +#define MCF_IRQ_I2C2 (MCFINT1_VECBASE + MCFINT1_I2C2) +#define MCF_IRQ_I2C3 (MCFINT1_VECBASE + MCFINT1_I2C3) +#define MCF_IRQ_I2C4 (MCFINT1_VECBASE + MCFINT1_I2C4) +#define MCF_IRQ_I2C5 (MCFINT1_VECBASE + MCFINT1_I2C5) +/* + * EPORT Module. + */ +#define MCFEPORT_EPPAR 0xfc090000 +#define MCFEPORT_EPIER 0xfc090003 +#define MCFEPORT_EPFR 0xfc090006 +/* + * RTC Module. + */ +#define MCFRTC_BASE 0xfc0a8000 +#define MCFRTC_SIZE (0xfc0a8840 - 0xfc0a8000) +#define MCF_IRQ_RTC (MCFINT2_VECBASE + MCFINT2_RTC) + +/* + * GPIO Module. + */ +#define MCFGPIO_PODR_A 0xec094000 +#define MCFGPIO_PODR_B 0xec094001 +#define MCFGPIO_PODR_C 0xec094002 +#define MCFGPIO_PODR_D 0xec094003 +#define MCFGPIO_PODR_E 0xec094004 +#define MCFGPIO_PODR_F 0xec094005 +#define MCFGPIO_PODR_G 0xec094006 +#define MCFGPIO_PODR_H 0xec094007 +#define MCFGPIO_PODR_I 0xec094008 +#define MCFGPIO_PODR_J 0xec094009 +#define MCFGPIO_PODR_K 0xec09400a + +#define MCFGPIO_PDDR_A 0xec09400c +#define MCFGPIO_PDDR_B 0xec09400d +#define MCFGPIO_PDDR_C 0xec09400e +#define MCFGPIO_PDDR_D 0xec09400f +#define MCFGPIO_PDDR_E 0xec094010 +#define MCFGPIO_PDDR_F 0xec094011 +#define MCFGPIO_PDDR_G 0xec094012 +#define MCFGPIO_PDDR_H 0xec094013 +#define MCFGPIO_PDDR_I 0xec094014 +#define MCFGPIO_PDDR_J 0xec094015 +#define MCFGPIO_PDDR_K 0xec094016 + +#define MCFGPIO_PPDSDR_A 0xec094018 +#define MCFGPIO_PPDSDR_B 0xec094019 +#define MCFGPIO_PPDSDR_C 0xec09401a +#define MCFGPIO_PPDSDR_D 0xec09401b +#define MCFGPIO_PPDSDR_E 0xec09401c +#define MCFGPIO_PPDSDR_F 0xec09401d +#define MCFGPIO_PPDSDR_G 0xec09401e +#define MCFGPIO_PPDSDR_H 0xec09401f +#define MCFGPIO_PPDSDR_I 0xec094020 +#define MCFGPIO_PPDSDR_J 0xec094021 +#define MCFGPIO_PPDSDR_K 0xec094022 + +#define MCFGPIO_PCLRR_A 0xec094024 +#define MCFGPIO_PCLRR_B 0xec094025 +#define MCFGPIO_PCLRR_C 0xec094026 +#define MCFGPIO_PCLRR_D 0xec094027 +#define MCFGPIO_PCLRR_E 0xec094028 +#define MCFGPIO_PCLRR_F 0xec094029 +#define MCFGPIO_PCLRR_G 0xec09402a +#define MCFGPIO_PCLRR_H 0xec09402b +#define MCFGPIO_PCLRR_I 0xec09402c +#define MCFGPIO_PCLRR_J 0xec09402d +#define MCFGPIO_PCLRR_K 0xec09402e + +#define MCFGPIO_PAR_FBCTL 0xec094048 +#define MCFGPIO_PAR_BE 0xec094049 +#define MCFGPIO_PAR_CS 0xec09404a +#define MCFGPIO_PAR_CANI2C 0xec09404b +#define MCFGPIO_PAR_IRQ0H 0xec09404c +#define MCFGPIO_PAR_IRQ0L 0xec09404d +#define MCFGPIO_PAR_DSPIOWH 0xec09404e +#define MCFGPIO_PAR_DSPIOWL 0xec09404f +#define MCFGPIO_PAR_TIMER 0xec094050 +#define MCFGPIO_PAR_UART2 0xec094051 +#define MCFGPIO_PAR_UART1 0xec094052 +#define MCFGPIO_PAR_UART0 0xec094053 +#define MCFGPIO_PAR_SDHCH 0xec094054 +#define MCFGPIO_PAR_SDHCL 0xec094055 +#define MCFGPIO_PAR_SIMP0H 0xec094056 +#define MCFGPIO_PAR_SIMP0L 0xec094057 +#define MCFGPIO_PAR_SSI0H 0xec094058 +#define MCFGPIO_PAR_SSI0L 0xec094059 +#define MCFGPIO_PAR_DEBUGH1 0xec09405a +#define MCFGPIO_PAR_DEBUGH0 0xec09405b +#define MCFGPIO_PAR_DEBUGl 0xec09405c +#define MCFGPIO_PAR_FEC 0xec09405e + +/* generalization for generic gpio support */ +#define MCFGPIO_PODR MCFGPIO_PODR_A +#define MCFGPIO_PDDR MCFGPIO_PDDR_A +#define MCFGPIO_PPDR MCFGPIO_PPDSDR_A +#define MCFGPIO_SETR MCFGPIO_PPDSDR_A +#define MCFGPIO_CLRR MCFGPIO_PCLRR_A + +#define MCFGPIO_IRQ_MIN 17 +#define MCFGPIO_IRQ_MAX 24 +#define MCFGPIO_IRQ_VECBASE (MCFINT_VECBASE - MCFGPIO_IRQ_MIN) +#define MCFGPIO_PIN_MAX 87 + +#endif /* m5441xsim_h */ diff --git a/arch/m68k/include/asm/m54xxacr.h b/arch/m68k/include/asm/m54xxacr.h index 47906aafbf67..192bbfeabf70 100644 --- a/arch/m68k/include/asm/m54xxacr.h +++ b/arch/m68k/include/asm/m54xxacr.h @@ -55,6 +55,10 @@ #define ICACHE_SIZE 0x8000 /* instruction - 32k */ #define DCACHE_SIZE 0x8000 /* data - 32k */ +#elif defined(CONFIG_M5441x) + +#define ICACHE_SIZE 0x2000 /* instruction - 8k */ +#define DCACHE_SIZE 0x2000 /* data - 8k */ #endif #define CACHE_LINE_SIZE 0x0010 /* 16 bytes */ diff --git a/arch/m68k/include/asm/m54xxpci.h b/arch/m68k/include/asm/m54xxpci.h new file mode 100644 index 000000000000..6fbf54f72f2e --- /dev/null +++ b/arch/m68k/include/asm/m54xxpci.h @@ -0,0 +1,138 @@ +/****************************************************************************/ + +/* + * m54xxpci.h -- ColdFire 547x and 548x PCI bus support + * + * (C) Copyright 2011, Greg Ungerer <gerg@uclinux.org> + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file COPYING in the main directory of this archive + * for more details. + */ + +/****************************************************************************/ +#ifndef M54XXPCI_H +#define M54XXPCI_H +/****************************************************************************/ + +/* + * The core set of PCI support registers are mapped into the MBAR region. + */ +#define PCIIDR (CONFIG_MBAR + 0xb00) /* PCI device/vendor ID */ +#define PCISCR (CONFIG_MBAR + 0xb04) /* PCI status/command */ +#define PCICCRIR (CONFIG_MBAR + 0xb08) /* PCI class/revision */ +#define PCICR1 (CONFIG_MBAR + 0xb0c) /* PCI configuration 1 */ +#define PCIBAR0 (CONFIG_MBAR + 0xb10) /* PCI base address 0 */ +#define PCIBAR1 (CONFIG_MBAR + 0xb14) /* PCI base address 1 */ +#define PCICCPR (CONFIG_MBAR + 0xb28) /* PCI cardbus CIS pointer */ +#define PCISID (CONFIG_MBAR + 0xb2c) /* PCI subsystem IDs */ +#define PCIERBAR (CONFIG_MBAR + 0xb30) /* PCI expansion ROM */ +#define PCICPR (CONFIG_MBAR + 0xb34) /* PCI capabilities pointer */ +#define PCICR2 (CONFIG_MBAR + 0xb3c) /* PCI configuration 2 */ + +#define PCIGSCR (CONFIG_MBAR + 0xb60) /* Global status/control */ +#define PCITBATR0 (CONFIG_MBAR + 0xb64) /* Target base translation 0 */ +#define PCITBATR1 (CONFIG_MBAR + 0xb68) /* Target base translation 1 */ +#define PCITCR (CONFIG_MBAR + 0xb6c) /* Target control */ +#define PCIIW0BTAR (CONFIG_MBAR + 0xb70) /* Initiator window 0 */ +#define PCIIW1BTAR (CONFIG_MBAR + 0xb74) /* Initiator window 1 */ +#define PCIIW2BTAR (CONFIG_MBAR + 0xb78) /* Initiator window 2 */ +#define PCIIWCR (CONFIG_MBAR + 0xb80) /* Initiator window config */ +#define PCIICR (CONFIG_MBAR + 0xb84) /* Initiator control */ +#define PCIISR (CONFIG_MBAR + 0xb88) /* Initiator status */ +#define PCICAR (CONFIG_MBAR + 0xbf8) /* Configuration address */ + +#define PCITPSR (CONFIG_MBAR + 0x8400) /* TX packet size */ +#define PCITSAR (CONFIG_MBAR + 0x8404) /* TX start address */ +#define PCITTCR (CONFIG_MBAR + 0x8408) /* TX transaction control */ +#define PCITER (CONFIG_MBAR + 0x840c) /* TX enables */ +#define PCITNAR (CONFIG_MBAR + 0x8410) /* TX next address */ +#define PCITLWR (CONFIG_MBAR + 0x8414) /* TX last word */ +#define PCITDCR (CONFIG_MBAR + 0x8418) /* TX done counts */ +#define PCITSR (CONFIG_MBAR + 0x841c) /* TX status */ +#define PCITFDR (CONFIG_MBAR + 0x8440) /* TX FIFO data */ +#define PCITFSR (CONFIG_MBAR + 0x8444) /* TX FIFO status */ +#define PCITFCR (CONFIG_MBAR + 0x8448) /* TX FIFO control */ +#define PCITFAR (CONFIG_MBAR + 0x844c) /* TX FIFO alarm */ +#define PCITFRPR (CONFIG_MBAR + 0x8450) /* TX FIFO read pointer */ +#define PCITFWPR (CONFIG_MBAR + 0x8454) /* TX FIFO write pointer */ + +#define PCIRPSR (CONFIG_MBAR + 0x8480) /* RX packet size */ +#define PCIRSAR (CONFIG_MBAR + 0x8484) /* RX start address */ +#define PCIRTCR (CONFIG_MBAR + 0x8488) /* RX transaction control */ +#define PCIRER (CONFIG_MBAR + 0x848c) /* RX enables */ +#define PCIRNAR (CONFIG_MBAR + 0x8490) /* RX next address */ +#define PCIRDCR (CONFIG_MBAR + 0x8498) /* RX done counts */ +#define PCIRSR (CONFIG_MBAR + 0x849c) /* RX status */ +#define PCIRFDR (CONFIG_MBAR + 0x84c0) /* RX FIFO data */ +#define PCIRFSR (CONFIG_MBAR + 0x84c4) /* RX FIFO status */ +#define PCIRFCR (CONFIG_MBAR + 0x84c8) /* RX FIFO control */ +#define PCIRFAR (CONFIG_MBAR + 0x84cc) /* RX FIFO alarm */ +#define PCIRFRPR (CONFIG_MBAR + 0x84d0) /* RX FIFO read pointer */ +#define PCIRFWPR (CONFIG_MBAR + 0x84d4) /* RX FIFO write pointer */ + +#define PACR (CONFIG_MBAR + 0xc00) /* PCI arbiter control */ +#define PASR (COFNIG_MBAR + 0xc04) /* PCI arbiter status */ + +/* + * Definitions for the Global status and control register. + */ +#define PCIGSCR_PE 0x20000000 /* Parity error detected */ +#define PCIGSCR_SE 0x10000000 /* System error detected */ +#define PCIGSCR_XCLKBIN 0x07000000 /* XLB2CLKIN mask */ +#define PCIGSCR_PEE 0x00002000 /* Parity error intr enable */ +#define PCIGSCR_SEE 0x00001000 /* System error intr enable */ +#define PCIGSCR_RESET 0x00000001 /* Reset bit */ + +/* + * Bit definitions for the PCICAR configuration address register. + */ +#define PCICAR_E 0x80000000 /* Enable config space */ +#define PCICAR_BUSN 16 /* Move bus bits */ +#define PCICAR_DEVFNN 8 /* Move devfn bits */ +#define PCICAR_DWORDN 0 /* Move dword bits */ + +/* + * The initiator windows hold the memory and IO mapping information. + * This macro creates the register values from the desired addresses. + */ +#define WXBTAR(hostaddr, pciaddr, size) \ + (((hostaddr) & 0xff000000) | \ + ((((size) - 1) & 0xff000000) >> 8) | \ + (((pciaddr) & 0xff000000) >> 16)) + +#define PCIIWCR_W0_MEM 0x00000000 /* Window 0 is memory */ +#define PCIIWCR_W0_IO 0x08000000 /* Window 0 is IO */ +#define PCIIWCR_W0_MRD 0x00000000 /* Window 0 memory read */ +#define PCIIWCR_W0_MRDL 0x02000000 /* Window 0 memory read line */ +#define PCIIWCR_W0_MRDM 0x04000000 /* Window 0 memory read mult */ +#define PCIIWCR_W0_E 0x01000000 /* Window 0 enable */ + +#define PCIIWCR_W1_MEM 0x00000000 /* Window 0 is memory */ +#define PCIIWCR_W1_IO 0x00080000 /* Window 0 is IO */ +#define PCIIWCR_W1_MRD 0x00000000 /* Window 0 memory read */ +#define PCIIWCR_W1_MRDL 0x00020000 /* Window 0 memory read line */ +#define PCIIWCR_W1_MRDM 0x00040000 /* Window 0 memory read mult */ +#define PCIIWCR_W1_E 0x00010000 /* Window 0 enable */ + +/* + * Bit definitions for the PCIBATR registers. + */ +#define PCITBATR0_E 0x00000001 /* Enable window 0 */ +#define PCITBATR1_E 0x00000001 /* Enable window 1 */ + +/* + * PCI arbiter support definitions and macros. + */ +#define PACR_INTMPRI 0x00000001 +#define PACR_EXTMPRI(x) (((x) & 0x1f) << 1) +#define PACR_INTMINTE 0x00010000 +#define PACR_EXTMINTE(x) (((x) & 0x1f) << 17) +#define PACR_PKMD 0x40000000 +#define PACR_DS 0x80000000 + +#define PCICR1_CL(x) ((x) & 0xf) /* Cacheline size field */ +#define PCICR1_LT(x) (((x) & 0xff) << 8) /* Latency timer field */ + +/****************************************************************************/ +#endif /* M54XXPCI_H */ diff --git a/arch/m68k/include/asm/m54xxsim.h b/arch/m68k/include/asm/m54xxsim.h index ae56b8848a9d..d3c5e0dbdadf 100644 --- a/arch/m68k/include/asm/m54xxsim.h +++ b/arch/m68k/include/asm/m54xxsim.h @@ -81,4 +81,7 @@ #define MCF_PAR_PSC_RTS_RTS (0x30) #define MCF_PAR_PSC_CANRX (0x40) +#define MCF_PAR_PCIBG (CONFIG_MBAR + 0xa48) /* PCI bus grant */ +#define MCF_PAR_PCIBR (CONFIG_MBAR + 0xa4a) /* PCI */ + #endif /* m54xxsim_h */ diff --git a/arch/m68k/include/asm/mcfclk.h b/arch/m68k/include/asm/mcfclk.h new file mode 100644 index 000000000000..b676a02bb392 --- /dev/null +++ b/arch/m68k/include/asm/mcfclk.h @@ -0,0 +1,43 @@ +/* + * mcfclk.h -- coldfire specific clock structure + */ + + +#ifndef mcfclk_h +#define mcfclk_h + +struct clk; + +#ifdef MCFPM_PPMCR0 +struct clk_ops { + void (*enable)(struct clk *); + void (*disable)(struct clk *); +}; + +struct clk { + const char *name; + struct clk_ops *clk_ops; + unsigned long rate; + unsigned long enabled; + u8 slot; +}; + +extern struct clk *mcf_clks[]; +extern struct clk_ops clk_ops0; +#ifdef MCFPM_PPMCR1 +extern struct clk_ops clk_ops1; +#endif /* MCFPM_PPMCR1 */ + +#define DEFINE_CLK(clk_bank, clk_name, clk_slot, clk_rate) \ +static struct clk __clk_##clk_bank##_##clk_slot = { \ + .name = clk_name, \ + .clk_ops = &clk_ops##clk_bank, \ + .rate = clk_rate, \ + .slot = clk_slot, \ +} + +void __clk_init_enabled(struct clk *); +void __clk_init_disabled(struct clk *); +#endif /* MCFPM_PPMCR0 */ + +#endif /* mcfclk_h */ diff --git a/arch/m68k/include/asm/mcfgpio.h b/arch/m68k/include/asm/mcfgpio.h index fe468eaa51e0..fa1059f50dfc 100644 --- a/arch/m68k/include/asm/mcfgpio.h +++ b/arch/m68k/include/asm/mcfgpio.h @@ -16,82 +16,289 @@ #ifndef mcfgpio_h #define mcfgpio_h -#include <linux/io.h> +#ifdef CONFIG_GPIOLIB #include <asm-generic/gpio.h> +#else + +int __mcfgpio_get_value(unsigned gpio); +void __mcfgpio_set_value(unsigned gpio, int value); +int __mcfgpio_direction_input(unsigned gpio); +int __mcfgpio_direction_output(unsigned gpio, int value); +int __mcfgpio_request(unsigned gpio); +void __mcfgpio_free(unsigned gpio); + +/* our alternate 'gpiolib' functions */ +static inline int __gpio_get_value(unsigned gpio) +{ + if (gpio < MCFGPIO_PIN_MAX) + return __mcfgpio_get_value(gpio); + else + return -EINVAL; +} + +static inline void __gpio_set_value(unsigned gpio, int value) +{ + if (gpio < MCFGPIO_PIN_MAX) + __mcfgpio_set_value(gpio, value); +} + +static inline int __gpio_cansleep(unsigned gpio) +{ + if (gpio < MCFGPIO_PIN_MAX) + return 0; + else + return -EINVAL; +} + +static inline int __gpio_to_irq(unsigned gpio) +{ + return -EINVAL; +} + +static inline int gpio_direction_input(unsigned gpio) +{ + if (gpio < MCFGPIO_PIN_MAX) + return __mcfgpio_direction_input(gpio); + else + return -EINVAL; +} + +static inline int gpio_direction_output(unsigned gpio, int value) +{ + if (gpio < MCFGPIO_PIN_MAX) + return __mcfgpio_direction_output(gpio, value); + else + return -EINVAL; +} + +static inline int gpio_request(unsigned gpio, const char *label) +{ + if (gpio < MCFGPIO_PIN_MAX) + return __mcfgpio_request(gpio); + else + return -EINVAL; +} + +static inline void gpio_free(unsigned gpio) +{ + if (gpio < MCFGPIO_PIN_MAX) + __mcfgpio_free(gpio); +} + +#endif /* CONFIG_GPIOLIB */ -struct mcf_gpio_chip { - struct gpio_chip gpio_chip; - void __iomem *pddr; - void __iomem *podr; - void __iomem *ppdr; - void __iomem *setr; - void __iomem *clrr; - const u8 *gpio_to_pinmux; -}; - -extern struct mcf_gpio_chip mcf_gpio_chips[]; -extern unsigned int mcf_gpio_chips_size; - -int mcf_gpio_direction_input(struct gpio_chip *, unsigned); -int mcf_gpio_get_value(struct gpio_chip *, unsigned); -int mcf_gpio_direction_output(struct gpio_chip *, unsigned, int); -void mcf_gpio_set_value(struct gpio_chip *, unsigned, int); -void mcf_gpio_set_value_fast(struct gpio_chip *, unsigned, int); -int mcf_gpio_request(struct gpio_chip *, unsigned); -void mcf_gpio_free(struct gpio_chip *, unsigned); /* - * Define macros to ease the pain of setting up the GPIO tables. There - * are two cases we need to deal with here, they cover all currently - * available ColdFire GPIO hardware. There are of course minor differences - * in the layout and number of bits in each ColdFire part, but the macros - * take all that in. + * The Freescale Coldfire family is quite varied in how they implement GPIO. + * Some parts have 8 bit ports, some have 16bit and some have 32bit; some have + * only one port, others have multiple ports; some have a single data latch + * for both input and output, others have a separate pin data register to read + * input; some require a read-modify-write access to change an output, others + * have set and clear registers for some of the outputs; Some have all the + * GPIOs in a single control area, others have some GPIOs implemented in + * different modules. * - * Firstly is the conventional GPIO registers where we toggle individual - * bits in a register, preserving the other bits in the register. For - * lack of a better term I have called this the slow method. + * This implementation attempts accommodate the differences while presenting + * a generic interface that will optimize to as few instructions as possible. + */ +#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \ + defined(CONFIG_M520x) || defined(CONFIG_M523x) || \ + defined(CONFIG_M527x) || defined(CONFIG_M528x) || \ + defined(CONFIG_M532x) || defined(CONFIG_M54xx) || \ + defined(CONFIG_M5441x) + +/* These parts have GPIO organized by 8 bit ports */ + +#define MCFGPIO_PORTTYPE u8 +#define MCFGPIO_PORTSIZE 8 +#define mcfgpio_read(port) __raw_readb(port) +#define mcfgpio_write(data, port) __raw_writeb(data, port) + +#elif defined(CONFIG_M5307) || defined(CONFIG_M5407) || defined(CONFIG_M5272) + +/* These parts have GPIO organized by 16 bit ports */ + +#define MCFGPIO_PORTTYPE u16 +#define MCFGPIO_PORTSIZE 16 +#define mcfgpio_read(port) __raw_readw(port) +#define mcfgpio_write(data, port) __raw_writew(data, port) + +#elif defined(CONFIG_M5249) || defined(CONFIG_M525x) + +/* These parts have GPIO organized by 32 bit ports */ + +#define MCFGPIO_PORTTYPE u32 +#define MCFGPIO_PORTSIZE 32 +#define mcfgpio_read(port) __raw_readl(port) +#define mcfgpio_write(data, port) __raw_writel(data, port) + +#endif + +#define mcfgpio_bit(gpio) (1 << ((gpio) % MCFGPIO_PORTSIZE)) +#define mcfgpio_port(gpio) ((gpio) / MCFGPIO_PORTSIZE) + +#if defined(CONFIG_M520x) || defined(CONFIG_M523x) || \ + defined(CONFIG_M527x) || defined(CONFIG_M528x) || \ + defined(CONFIG_M532x) || defined(CONFIG_M5441x) +/* + * These parts have an 'Edge' Port module (external interrupt/GPIO) which uses + * read-modify-write to change an output and a GPIO module which has separate + * set/clr registers to directly change outputs with a single write access. + */ +#if defined(CONFIG_M528x) +/* + * The 528x also has GPIOs in other modules (GPT, QADC) which use + * read-modify-write as well as those controlled by the EPORT and GPIO modules. */ -#define MCFGPS(mlabel, mbase, mngpio, mpddr, mpodr, mppdr) \ - { \ - .gpio_chip = { \ - .label = #mlabel, \ - .request = mcf_gpio_request, \ - .free = mcf_gpio_free, \ - .direction_input = mcf_gpio_direction_input, \ - .direction_output = mcf_gpio_direction_output,\ - .get = mcf_gpio_get_value, \ - .set = mcf_gpio_set_value, \ - .base = mbase, \ - .ngpio = mngpio, \ - }, \ - .pddr = (void __iomem *) mpddr, \ - .podr = (void __iomem *) mpodr, \ - .ppdr = (void __iomem *) mppdr, \ - } +#define MCFGPIO_SCR_START 40 +#elif defined(CONFIGM5441x) +/* The m5441x EPORT doesn't have its own GPIO port, uses PORT C */ +#define MCFGPIO_SCR_START 0 +#else +#define MCFGPIO_SCR_START 8 +#endif +#define MCFGPIO_SETR_PORT(gpio) (MCFGPIO_SETR + \ + mcfgpio_port(gpio - MCFGPIO_SCR_START)) + +#define MCFGPIO_CLRR_PORT(gpio) (MCFGPIO_CLRR + \ + mcfgpio_port(gpio - MCFGPIO_SCR_START)) +#else + +#define MCFGPIO_SCR_START MCFGPIO_PIN_MAX +/* with MCFGPIO_SCR == MCFGPIO_PIN_MAX, these will be optimized away */ +#define MCFGPIO_SETR_PORT(gpio) 0 +#define MCFGPIO_CLRR_PORT(gpio) 0 + +#endif /* - * Secondly is the faster case, where we have set and clear registers - * that allow us to set or clear a bit with a single write, not having - * to worry about preserving other bits. + * Coldfire specific helper functions */ -#define MCFGPF(mlabel, mbase, mngpio) \ - { \ - .gpio_chip = { \ - .label = #mlabel, \ - .request = mcf_gpio_request, \ - .free = mcf_gpio_free, \ - .direction_input = mcf_gpio_direction_input, \ - .direction_output = mcf_gpio_direction_output,\ - .get = mcf_gpio_get_value, \ - .set = mcf_gpio_set_value_fast, \ - .base = mbase, \ - .ngpio = mngpio, \ - }, \ - .pddr = (void __iomem *) MCFGPIO_PDDR_##mlabel, \ - .podr = (void __iomem *) MCFGPIO_PODR_##mlabel, \ - .ppdr = (void __iomem *) MCFGPIO_PPDSDR_##mlabel, \ - .setr = (void __iomem *) MCFGPIO_PPDSDR_##mlabel, \ - .clrr = (void __iomem *) MCFGPIO_PCLRR_##mlabel, \ - } +/* return the port pin data register for a gpio */ +static inline u32 __mcfgpio_ppdr(unsigned gpio) +{ +#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \ + defined(CONFIG_M5307) || defined(CONFIG_M5407) + return MCFSIM_PADAT; +#elif defined(CONFIG_M5272) + if (gpio < 16) + return MCFSIM_PADAT; + else if (gpio < 32) + return MCFSIM_PBDAT; + else + return MCFSIM_PCDAT; +#elif defined(CONFIG_M5249) || defined(CONFIG_M525x) + if (gpio < 32) + return MCFSIM2_GPIOREAD; + else + return MCFSIM2_GPIO1READ; +#elif defined(CONFIG_M520x) || defined(CONFIG_M523x) || \ + defined(CONFIG_M527x) || defined(CONFIG_M528x) || \ + defined(CONFIG_M532x) || defined(CONFIG_M5441x) +#if !defined(CONFIG_M5441x) + if (gpio < 8) + return MCFEPORT_EPPDR; +#if defined(CONFIG_M528x) + else if (gpio < 16) + return MCFGPTA_GPTPORT; + else if (gpio < 24) + return MCFGPTB_GPTPORT; + else if (gpio < 32) + return MCFQADC_PORTQA; + else if (gpio < 40) + return MCFQADC_PORTQB; +#endif /* defined(CONFIG_M528x) */ + else +#endif /* !defined(CONFIG_M5441x) */ + return MCFGPIO_PPDR + mcfgpio_port(gpio - MCFGPIO_SCR_START); +#else + return 0; #endif +} + +/* return the port output data register for a gpio */ +static inline u32 __mcfgpio_podr(unsigned gpio) +{ +#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \ + defined(CONFIG_M5307) || defined(CONFIG_M5407) + return MCFSIM_PADAT; +#elif defined(CONFIG_M5272) + if (gpio < 16) + return MCFSIM_PADAT; + else if (gpio < 32) + return MCFSIM_PBDAT; + else + return MCFSIM_PCDAT; +#elif defined(CONFIG_M5249) || defined(CONFIG_M525x) + if (gpio < 32) + return MCFSIM2_GPIOWRITE; + else + return MCFSIM2_GPIO1WRITE; +#elif defined(CONFIG_M520x) || defined(CONFIG_M523x) || \ + defined(CONFIG_M527x) || defined(CONFIG_M528x) || \ + defined(CONFIG_M532x) || defined(CONFIG_M5441x) +#if !defined(CONFIG_M5441x) + if (gpio < 8) + return MCFEPORT_EPDR; +#if defined(CONFIG_M528x) + else if (gpio < 16) + return MCFGPTA_GPTPORT; + else if (gpio < 24) + return MCFGPTB_GPTPORT; + else if (gpio < 32) + return MCFQADC_PORTQA; + else if (gpio < 40) + return MCFQADC_PORTQB; +#endif /* defined(CONFIG_M528x) */ + else +#endif /* !defined(CONFIG_M5441x) */ + return MCFGPIO_PODR + mcfgpio_port(gpio - MCFGPIO_SCR_START); +#else + return 0; +#endif +} + +/* return the port direction data register for a gpio */ +static inline u32 __mcfgpio_pddr(unsigned gpio) +{ +#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \ + defined(CONFIG_M5307) || defined(CONFIG_M5407) + return MCFSIM_PADDR; +#elif defined(CONFIG_M5272) + if (gpio < 16) + return MCFSIM_PADDR; + else if (gpio < 32) + return MCFSIM_PBDDR; + else + return MCFSIM_PCDDR; +#elif defined(CONFIG_M5249) || defined(CONFIG_M525x) + if (gpio < 32) + return MCFSIM2_GPIOENABLE; + else + return MCFSIM2_GPIO1ENABLE; +#elif defined(CONFIG_M520x) || defined(CONFIG_M523x) || \ + defined(CONFIG_M527x) || defined(CONFIG_M528x) || \ + defined(CONFIG_M532x) || defined(CONFIG_M5441x) +#if !defined(CONFIG_M5441x) + if (gpio < 8) + return MCFEPORT_EPDDR; +#if defined(CONFIG_M528x) + else if (gpio < 16) + return MCFGPTA_GPTDDR; + else if (gpio < 24) + return MCFGPTB_GPTDDR; + else if (gpio < 32) + return MCFQADC_DDRQA; + else if (gpio < 40) + return MCFQADC_DDRQB; +#endif /* defined(CONFIG_M528x) */ + else +#endif /* !defined(CONFIG_M5441x) */ + return MCFGPIO_PDDR + mcfgpio_port(gpio - MCFGPIO_SCR_START); +#else + return 0; +#endif +} + +#endif /* mcfgpio_h */ diff --git a/arch/m68k/include/asm/mcfsim.h b/arch/m68k/include/asm/mcfsim.h index ebd0304054ad..7a83e619e73b 100644 --- a/arch/m68k/include/asm/mcfsim.h +++ b/arch/m68k/include/asm/mcfsim.h @@ -27,6 +27,9 @@ #elif defined(CONFIG_M5249) #include <asm/m5249sim.h> #include <asm/mcfintc.h> +#elif defined(CONFIG_M525x) +#include <asm/m525xsim.h> +#include <asm/mcfintc.h> #elif defined(CONFIG_M527x) #include <asm/m527xsim.h> #elif defined(CONFIG_M5272) @@ -43,6 +46,8 @@ #include <asm/mcfintc.h> #elif defined(CONFIG_M54xx) #include <asm/m54xxsim.h> +#elif defined(CONFIG_M5441x) +#include <asm/m5441xsim.h> #endif /****************************************************************************/ diff --git a/arch/m68k/include/asm/mcftimer.h b/arch/m68k/include/asm/mcftimer.h index 351c27237874..da2fa43c2e45 100644 --- a/arch/m68k/include/asm/mcftimer.h +++ b/arch/m68k/include/asm/mcftimer.h @@ -19,7 +19,7 @@ #define MCFTIMER_TRR 0x04 /* Timer Reference (r/w) */ #define MCFTIMER_TCR 0x08 /* Timer Capture reg (r/w) */ #define MCFTIMER_TCN 0x0C /* Timer Counter reg (r/w) */ -#if defined(CONFIG_M532x) +#if defined(CONFIG_M532x) || defined(CONFIG_M5441x) #define MCFTIMER_TER 0x03 /* Timer Event reg (r/w) */ #else #define MCFTIMER_TER 0x11 /* Timer Event reg (r/w) */ diff --git a/arch/m68k/include/asm/mcfuart.h b/arch/m68k/include/asm/mcfuart.h index 2d3bc774b3c5..b40c20f66647 100644 --- a/arch/m68k/include/asm/mcfuart.h +++ b/arch/m68k/include/asm/mcfuart.h @@ -43,8 +43,8 @@ struct mcf_platform_uart { #define MCFUART_UFPD 0x30 /* Frac Prec. Divider (r/w) */ #endif #if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \ - defined(CONFIG_M5249) || defined(CONFIG_M5307) || \ - defined(CONFIG_M5407) + defined(CONFIG_M5249) || defined(CONFIG_M525x) || \ + defined(CONFIG_M5307) || defined(CONFIG_M5407) #define MCFUART_UIVR 0x30 /* Interrupt Vector (r/w) */ #endif #define MCFUART_UIPR 0x34 /* Input Port (r) */ diff --git a/arch/m68k/include/asm/pci.h b/arch/m68k/include/asm/pci.h index 4ad0aea48ab4..848c3dfaad50 100644 --- a/arch/m68k/include/asm/pci.h +++ b/arch/m68k/include/asm/pci.h @@ -2,6 +2,7 @@ #define _ASM_M68K_PCI_H #include <asm-generic/pci-dma-compat.h> +#include <asm-generic/pci.h> /* The PCI address space does equal the physical memory * address space. The networking and block device layers use @@ -9,4 +10,9 @@ */ #define PCI_DMA_BUS_IS_PHYS (1) +#define pcibios_assign_all_busses() 1 + +#define PCIBIOS_MIN_IO 0x00000100 +#define PCIBIOS_MIN_MEM 0x02000000 + #endif /* _ASM_M68K_PCI_H */ diff --git a/arch/m68k/include/asm/pinmux.h b/arch/m68k/include/asm/pinmux.h deleted file mode 100644 index 119ee686dbd1..000000000000 --- a/arch/m68k/include/asm/pinmux.h +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Coldfire generic GPIO pinmux support. - * - * (C) Copyright 2009, Steven King <sfking@fdwdc.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - */ - -#ifndef pinmux_h -#define pinmux_h - -#define MCFPINMUX_NONE -1 - -extern int mcf_pinmux_request(unsigned, unsigned); -extern void mcf_pinmux_release(unsigned, unsigned); - -static inline int mcf_pinmux_is_valid(unsigned pinmux) -{ - return pinmux != MCFPINMUX_NONE; -} - -#endif - diff --git a/arch/m68k/kernel/Makefile b/arch/m68k/kernel/Makefile index 5c7070e21eb7..068ad49210d6 100644 --- a/arch/m68k/kernel/Makefile +++ b/arch/m68k/kernel/Makefile @@ -18,6 +18,7 @@ obj-y += setup.o signal.o sys_m68k.o syscalltable.o time.o traps.o obj-$(CONFIG_MMU_MOTOROLA) += ints.o vectors.o obj-$(CONFIG_MMU_SUN3) += ints.o vectors.o +obj-$(CONFIG_PCI) += pcibios.o ifndef CONFIG_MMU_SUN3 obj-y += dma.o diff --git a/arch/m68k/kernel/dma.c b/arch/m68k/kernel/dma.c index f6daf6e15d2e..e546a5534dd4 100644 --- a/arch/m68k/kernel/dma.c +++ b/arch/m68k/kernel/dma.c @@ -16,7 +16,7 @@ #include <asm/pgalloc.h> -#ifdef CONFIG_MMU +#if defined(CONFIG_MMU) && !defined(CONFIG_COLDFIRE) void *dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *handle, gfp_t flag) @@ -96,7 +96,7 @@ void dma_free_coherent(struct device *dev, size_t size, free_pages((unsigned long)vaddr, get_order(size)); } -#endif /* CONFIG_MMU */ +#endif /* CONFIG_MMU && !CONFIG_COLDFIRE */ EXPORT_SYMBOL(dma_alloc_coherent); EXPORT_SYMBOL(dma_free_coherent); @@ -105,6 +105,7 @@ void dma_sync_single_for_device(struct device *dev, dma_addr_t handle, size_t size, enum dma_data_direction dir) { switch (dir) { + case DMA_BIDIRECTIONAL: case DMA_TO_DEVICE: cache_push(handle, size); break; diff --git a/arch/m68k/kernel/entry.S b/arch/m68k/kernel/entry.S index b8daf64e347d..165ee9f9d5c9 100644 --- a/arch/m68k/kernel/entry.S +++ b/arch/m68k/kernel/entry.S @@ -1,5 +1,451 @@ -#if defined(CONFIG_MMU) && !defined(CONFIG_COLDFIRE) -#include "entry_mm.S" +/* -*- mode: asm -*- + * + * linux/arch/m68k/kernel/entry.S + * + * Copyright (C) 1991, 1992 Linus Torvalds + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file README.legal in the main directory of this archive + * for more details. + * + * Linux/m68k support by Hamish Macdonald + * + * 68060 fixes by Jesper Skov + * + */ + +/* + * entry.S contains the system-call and fault low-level handling routines. + * This also contains the timer-interrupt handler, as well as all interrupts + * and faults that can result in a task-switch. + * + * NOTE: This code handles signal-recognition, which happens every time + * after a timer-interrupt and after each system call. + * + */ + +/* + * 12/03/96 Jes: Currently we only support m68k single-cpu systems, so + * all pointers that used to be 'current' are now entry + * number 0 in the 'current_set' list. + * + * 6/05/00 RZ: addedd writeback completion after return from sighandler + * for 68040 + */ + +#include <linux/linkage.h> +#include <asm/errno.h> +#include <asm/setup.h> +#include <asm/segment.h> +#include <asm/traps.h> +#include <asm/unistd.h> +#include <asm/asm-offsets.h> +#include <asm/entry.h> + +.globl system_call, buserr, trap, resume +.globl sys_call_table +.globl sys_fork, sys_clone, sys_vfork +.globl ret_from_interrupt, bad_interrupt +.globl auto_irqhandler_fixup +.globl user_irqvec_fixup + +.text +ENTRY(sys_fork) + SAVE_SWITCH_STACK + pea %sp@(SWITCH_STACK_SIZE) + jbsr m68k_fork + addql #4,%sp + RESTORE_SWITCH_STACK + rts + +ENTRY(sys_clone) + SAVE_SWITCH_STACK + pea %sp@(SWITCH_STACK_SIZE) + jbsr m68k_clone + addql #4,%sp + RESTORE_SWITCH_STACK + rts + +ENTRY(sys_vfork) + SAVE_SWITCH_STACK + pea %sp@(SWITCH_STACK_SIZE) + jbsr m68k_vfork + addql #4,%sp + RESTORE_SWITCH_STACK + rts + +ENTRY(sys_sigreturn) + SAVE_SWITCH_STACK + jbsr do_sigreturn + RESTORE_SWITCH_STACK + rts + +ENTRY(sys_rt_sigreturn) + SAVE_SWITCH_STACK + jbsr do_rt_sigreturn + RESTORE_SWITCH_STACK + rts + +ENTRY(buserr) + SAVE_ALL_INT + GET_CURRENT(%d0) + movel %sp,%sp@- | stack frame pointer argument + jbsr buserr_c + addql #4,%sp + jra ret_from_exception + +ENTRY(trap) + SAVE_ALL_INT + GET_CURRENT(%d0) + movel %sp,%sp@- | stack frame pointer argument + jbsr trap_c + addql #4,%sp + jra ret_from_exception + + | After a fork we jump here directly from resume, + | so that %d1 contains the previous task + | schedule_tail now used regardless of CONFIG_SMP +ENTRY(ret_from_fork) + movel %d1,%sp@- + jsr schedule_tail + addql #4,%sp + jra ret_from_exception + +#if defined(CONFIG_COLDFIRE) || !defined(CONFIG_MMU) + +#ifdef TRAP_DBG_INTERRUPT + +.globl dbginterrupt +ENTRY(dbginterrupt) + SAVE_ALL_INT + GET_CURRENT(%d0) + movel %sp,%sp@- /* stack frame pointer argument */ + jsr dbginterrupt_c + addql #4,%sp + jra ret_from_exception +#endif + +ENTRY(reschedule) + /* save top of frame */ + pea %sp@ + jbsr set_esp0 + addql #4,%sp + pea ret_from_exception + jmp schedule + +ENTRY(ret_from_user_signal) + moveq #__NR_sigreturn,%d0 + trap #0 + +ENTRY(ret_from_user_rt_signal) + movel #__NR_rt_sigreturn,%d0 + trap #0 + #else -#include "entry_no.S" + +do_trace_entry: + movel #-ENOSYS,%sp@(PT_OFF_D0)| needed for strace + subql #4,%sp + SAVE_SWITCH_STACK + jbsr syscall_trace + RESTORE_SWITCH_STACK + addql #4,%sp + movel %sp@(PT_OFF_ORIG_D0),%d0 + cmpl #NR_syscalls,%d0 + jcs syscall +badsys: + movel #-ENOSYS,%sp@(PT_OFF_D0) + jra ret_from_syscall + +do_trace_exit: + subql #4,%sp + SAVE_SWITCH_STACK + jbsr syscall_trace + RESTORE_SWITCH_STACK + addql #4,%sp + jra .Lret_from_exception + +ENTRY(ret_from_signal) + movel %curptr@(TASK_STACK),%a1 + tstb %a1@(TINFO_FLAGS+2) + jge 1f + jbsr syscall_trace +1: RESTORE_SWITCH_STACK + addql #4,%sp +/* on 68040 complete pending writebacks if any */ +#ifdef CONFIG_M68040 + bfextu %sp@(PT_OFF_FORMATVEC){#0,#4},%d0 + subql #7,%d0 | bus error frame ? + jbne 1f + movel %sp,%sp@- + jbsr berr_040cleanup + addql #4,%sp +1: +#endif + jra .Lret_from_exception + +ENTRY(system_call) + SAVE_ALL_SYS + + GET_CURRENT(%d1) + movel %d1,%a1 + + | save top of frame + movel %sp,%curptr@(TASK_THREAD+THREAD_ESP0) + + | syscall trace? + tstb %a1@(TINFO_FLAGS+2) + jmi do_trace_entry + cmpl #NR_syscalls,%d0 + jcc badsys +syscall: + jbsr @(sys_call_table,%d0:l:4)@(0) + movel %d0,%sp@(PT_OFF_D0) | save the return value +ret_from_syscall: + |oriw #0x0700,%sr + movel %curptr@(TASK_STACK),%a1 + movew %a1@(TINFO_FLAGS+2),%d0 + jne syscall_exit_work +1: RESTORE_ALL + +syscall_exit_work: + btst #5,%sp@(PT_OFF_SR) | check if returning to kernel + bnes 1b | if so, skip resched, signals + lslw #1,%d0 + jcs do_trace_exit + jmi do_delayed_trace + lslw #8,%d0 + jne do_signal_return + pea resume_userspace + jra schedule + + +ENTRY(ret_from_exception) +.Lret_from_exception: + btst #5,%sp@(PT_OFF_SR) | check if returning to kernel + bnes 1f | if so, skip resched, signals + | only allow interrupts when we are really the last one on the + | kernel stack, otherwise stack overflow can occur during + | heavy interrupt load + andw #ALLOWINT,%sr + +resume_userspace: + movel %curptr@(TASK_STACK),%a1 + moveb %a1@(TINFO_FLAGS+3),%d0 + jne exit_work +1: RESTORE_ALL + +exit_work: + | save top of frame + movel %sp,%curptr@(TASK_THREAD+THREAD_ESP0) + lslb #1,%d0 + jne do_signal_return + pea resume_userspace + jra schedule + + +do_signal_return: + |andw #ALLOWINT,%sr + subql #4,%sp | dummy return address + SAVE_SWITCH_STACK + pea %sp@(SWITCH_STACK_SIZE) + bsrl do_notify_resume + addql #4,%sp + RESTORE_SWITCH_STACK + addql #4,%sp + jbra resume_userspace + +do_delayed_trace: + bclr #7,%sp@(PT_OFF_SR) | clear trace bit in SR + pea 1 | send SIGTRAP + movel %curptr,%sp@- + pea LSIGTRAP + jbsr send_sig + addql #8,%sp + addql #4,%sp + jbra resume_userspace + + +/* This is the main interrupt handler for autovector interrupts */ + +ENTRY(auto_inthandler) + SAVE_ALL_INT + GET_CURRENT(%d0) + movel %d0,%a1 + addqb #1,%a1@(TINFO_PREEMPT+1) + | put exception # in d0 + bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0 + subw #VEC_SPUR,%d0 + + movel %sp,%sp@- + movel %d0,%sp@- | put vector # on stack +auto_irqhandler_fixup = . + 2 + jsr do_IRQ | process the IRQ + addql #8,%sp | pop parameters off stack + +ret_from_interrupt: + movel %curptr@(TASK_STACK),%a1 + subqb #1,%a1@(TINFO_PREEMPT+1) + jeq ret_from_last_interrupt +2: RESTORE_ALL + + ALIGN +ret_from_last_interrupt: + moveq #(~ALLOWINT>>8)&0xff,%d0 + andb %sp@(PT_OFF_SR),%d0 + jne 2b + + /* check if we need to do software interrupts */ + tstl irq_stat+CPUSTAT_SOFTIRQ_PENDING + jeq .Lret_from_exception + pea ret_from_exception + jra do_softirq + +/* Handler for user defined interrupt vectors */ + +ENTRY(user_inthandler) + SAVE_ALL_INT + GET_CURRENT(%d0) + movel %d0,%a1 + addqb #1,%a1@(TINFO_PREEMPT+1) + | put exception # in d0 + bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0 +user_irqvec_fixup = . + 2 + subw #VEC_USER,%d0 + + movel %sp,%sp@- + movel %d0,%sp@- | put vector # on stack + jsr do_IRQ | process the IRQ + addql #8,%sp | pop parameters off stack + + movel %curptr@(TASK_STACK),%a1 + subqb #1,%a1@(TINFO_PREEMPT+1) + jeq ret_from_last_interrupt + RESTORE_ALL + +/* Handler for uninitialized and spurious interrupts */ + +ENTRY(bad_inthandler) + SAVE_ALL_INT + GET_CURRENT(%d0) + movel %d0,%a1 + addqb #1,%a1@(TINFO_PREEMPT+1) + + movel %sp,%sp@- + jsr handle_badint + addql #4,%sp + + movel %curptr@(TASK_STACK),%a1 + subqb #1,%a1@(TINFO_PREEMPT+1) + jeq ret_from_last_interrupt + RESTORE_ALL + + +resume: + /* + * Beware - when entering resume, prev (the current task) is + * in a0, next (the new task) is in a1,so don't change these + * registers until their contents are no longer needed. + */ + + /* save sr */ + movew %sr,%a0@(TASK_THREAD+THREAD_SR) + + /* save fs (sfc,%dfc) (may be pointing to kernel memory) */ + movec %sfc,%d0 + movew %d0,%a0@(TASK_THREAD+THREAD_FS) + + /* save usp */ + /* it is better to use a movel here instead of a movew 8*) */ + movec %usp,%d0 + movel %d0,%a0@(TASK_THREAD+THREAD_USP) + + /* save non-scratch registers on stack */ + SAVE_SWITCH_STACK + + /* save current kernel stack pointer */ + movel %sp,%a0@(TASK_THREAD+THREAD_KSP) + + /* save floating point context */ +#ifndef CONFIG_M68KFPU_EMU_ONLY +#ifdef CONFIG_M68KFPU_EMU + tstl m68k_fputype + jeq 3f +#endif + fsave %a0@(TASK_THREAD+THREAD_FPSTATE) + +#if defined(CONFIG_M68060) +#if !defined(CPU_M68060_ONLY) + btst #3,m68k_cputype+3 + beqs 1f +#endif + /* The 060 FPU keeps status in bits 15-8 of the first longword */ + tstb %a0@(TASK_THREAD+THREAD_FPSTATE+2) + jeq 3f +#if !defined(CPU_M68060_ONLY) + jra 2f +#endif +#endif /* CONFIG_M68060 */ +#if !defined(CPU_M68060_ONLY) +1: tstb %a0@(TASK_THREAD+THREAD_FPSTATE) + jeq 3f +#endif +2: fmovemx %fp0-%fp7,%a0@(TASK_THREAD+THREAD_FPREG) + fmoveml %fpcr/%fpsr/%fpiar,%a0@(TASK_THREAD+THREAD_FPCNTL) +3: +#endif /* CONFIG_M68KFPU_EMU_ONLY */ + /* Return previous task in %d1 */ + movel %curptr,%d1 + + /* switch to new task (a1 contains new task) */ + movel %a1,%curptr + + /* restore floating point context */ +#ifndef CONFIG_M68KFPU_EMU_ONLY +#ifdef CONFIG_M68KFPU_EMU + tstl m68k_fputype + jeq 4f +#endif +#if defined(CONFIG_M68060) +#if !defined(CPU_M68060_ONLY) + btst #3,m68k_cputype+3 + beqs 1f +#endif + /* The 060 FPU keeps status in bits 15-8 of the first longword */ + tstb %a1@(TASK_THREAD+THREAD_FPSTATE+2) + jeq 3f +#if !defined(CPU_M68060_ONLY) + jra 2f +#endif +#endif /* CONFIG_M68060 */ +#if !defined(CPU_M68060_ONLY) +1: tstb %a1@(TASK_THREAD+THREAD_FPSTATE) + jeq 3f #endif +2: fmovemx %a1@(TASK_THREAD+THREAD_FPREG),%fp0-%fp7 + fmoveml %a1@(TASK_THREAD+THREAD_FPCNTL),%fpcr/%fpsr/%fpiar +3: frestore %a1@(TASK_THREAD+THREAD_FPSTATE) +4: +#endif /* CONFIG_M68KFPU_EMU_ONLY */ + + /* restore the kernel stack pointer */ + movel %a1@(TASK_THREAD+THREAD_KSP),%sp + + /* restore non-scratch registers */ + RESTORE_SWITCH_STACK + + /* restore user stack pointer */ + movel %a1@(TASK_THREAD+THREAD_USP),%a0 + movel %a0,%usp + + /* restore fs (sfc,%dfc) */ + movew %a1@(TASK_THREAD+THREAD_FS),%a0 + movec %a0,%sfc + movec %a0,%dfc + + /* restore status register */ + movew %a1@(TASK_THREAD+THREAD_SR),%sr + + rts + +#endif /* CONFIG_MMU && !CONFIG_COLDFIRE */ diff --git a/arch/m68k/kernel/entry_mm.S b/arch/m68k/kernel/entry_mm.S deleted file mode 100644 index f29e73ca9dbb..000000000000 --- a/arch/m68k/kernel/entry_mm.S +++ /dev/null @@ -1,419 +0,0 @@ -/* -*- mode: asm -*- - * - * linux/arch/m68k/kernel/entry.S - * - * Copyright (C) 1991, 1992 Linus Torvalds - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file README.legal in the main directory of this archive - * for more details. - * - * Linux/m68k support by Hamish Macdonald - * - * 68060 fixes by Jesper Skov - * - */ - -/* - * entry.S contains the system-call and fault low-level handling routines. - * This also contains the timer-interrupt handler, as well as all interrupts - * and faults that can result in a task-switch. - * - * NOTE: This code handles signal-recognition, which happens every time - * after a timer-interrupt and after each system call. - * - */ - -/* - * 12/03/96 Jes: Currently we only support m68k single-cpu systems, so - * all pointers that used to be 'current' are now entry - * number 0 in the 'current_set' list. - * - * 6/05/00 RZ: addedd writeback completion after return from sighandler - * for 68040 - */ - -#include <linux/linkage.h> -#include <asm/entry.h> -#include <asm/errno.h> -#include <asm/setup.h> -#include <asm/segment.h> -#include <asm/traps.h> -#include <asm/unistd.h> - -#include <asm/asm-offsets.h> - -.globl system_call, buserr, trap, resume -.globl sys_call_table -.globl sys_fork, sys_clone, sys_vfork -.globl ret_from_interrupt, bad_interrupt -.globl auto_irqhandler_fixup -.globl user_irqvec_fixup - -.text -ENTRY(buserr) - SAVE_ALL_INT - GET_CURRENT(%d0) - movel %sp,%sp@- | stack frame pointer argument - bsrl buserr_c - addql #4,%sp - jra .Lret_from_exception - -ENTRY(trap) - SAVE_ALL_INT - GET_CURRENT(%d0) - movel %sp,%sp@- | stack frame pointer argument - bsrl trap_c - addql #4,%sp - jra .Lret_from_exception - - | After a fork we jump here directly from resume, - | so that %d1 contains the previous task - | schedule_tail now used regardless of CONFIG_SMP -ENTRY(ret_from_fork) - movel %d1,%sp@- - jsr schedule_tail - addql #4,%sp - jra .Lret_from_exception - -do_trace_entry: - movel #-ENOSYS,%sp@(PT_OFF_D0)| needed for strace - subql #4,%sp - SAVE_SWITCH_STACK - jbsr syscall_trace - RESTORE_SWITCH_STACK - addql #4,%sp - movel %sp@(PT_OFF_ORIG_D0),%d0 - cmpl #NR_syscalls,%d0 - jcs syscall -badsys: - movel #-ENOSYS,%sp@(PT_OFF_D0) - jra ret_from_syscall - -do_trace_exit: - subql #4,%sp - SAVE_SWITCH_STACK - jbsr syscall_trace - RESTORE_SWITCH_STACK - addql #4,%sp - jra .Lret_from_exception - -ENTRY(ret_from_signal) - movel %curptr@(TASK_STACK),%a1 - tstb %a1@(TINFO_FLAGS+2) - jge 1f - jbsr syscall_trace -1: RESTORE_SWITCH_STACK - addql #4,%sp -/* on 68040 complete pending writebacks if any */ -#ifdef CONFIG_M68040 - bfextu %sp@(PT_OFF_FORMATVEC){#0,#4},%d0 - subql #7,%d0 | bus error frame ? - jbne 1f - movel %sp,%sp@- - jbsr berr_040cleanup - addql #4,%sp -1: -#endif - jra .Lret_from_exception - -ENTRY(system_call) - SAVE_ALL_SYS - - GET_CURRENT(%d1) - movel %d1,%a1 - - | save top of frame - movel %sp,%curptr@(TASK_THREAD+THREAD_ESP0) - - | syscall trace? - tstb %a1@(TINFO_FLAGS+2) - jmi do_trace_entry - cmpl #NR_syscalls,%d0 - jcc badsys -syscall: - jbsr @(sys_call_table,%d0:l:4)@(0) - movel %d0,%sp@(PT_OFF_D0) | save the return value -ret_from_syscall: - |oriw #0x0700,%sr - movel %curptr@(TASK_STACK),%a1 - movew %a1@(TINFO_FLAGS+2),%d0 - jne syscall_exit_work -1: RESTORE_ALL - -syscall_exit_work: - btst #5,%sp@(PT_OFF_SR) | check if returning to kernel - bnes 1b | if so, skip resched, signals - lslw #1,%d0 - jcs do_trace_exit - jmi do_delayed_trace - lslw #8,%d0 - jne do_signal_return - pea resume_userspace - jra schedule - - -ENTRY(ret_from_exception) -.Lret_from_exception: - btst #5,%sp@(PT_OFF_SR) | check if returning to kernel - bnes 1f | if so, skip resched, signals - | only allow interrupts when we are really the last one on the - | kernel stack, otherwise stack overflow can occur during - | heavy interrupt load - andw #ALLOWINT,%sr - -resume_userspace: - movel %curptr@(TASK_STACK),%a1 - moveb %a1@(TINFO_FLAGS+3),%d0 - jne exit_work -1: RESTORE_ALL - -exit_work: - | save top of frame - movel %sp,%curptr@(TASK_THREAD+THREAD_ESP0) - lslb #1,%d0 - jne do_signal_return - pea resume_userspace - jra schedule - - -do_signal_return: - |andw #ALLOWINT,%sr - subql #4,%sp | dummy return address - SAVE_SWITCH_STACK - pea %sp@(SWITCH_STACK_SIZE) - bsrl do_notify_resume - addql #4,%sp - RESTORE_SWITCH_STACK - addql #4,%sp - jbra resume_userspace - -do_delayed_trace: - bclr #7,%sp@(PT_OFF_SR) | clear trace bit in SR - pea 1 | send SIGTRAP - movel %curptr,%sp@- - pea LSIGTRAP - jbsr send_sig - addql #8,%sp - addql #4,%sp - jbra resume_userspace - - -/* This is the main interrupt handler for autovector interrupts */ - -ENTRY(auto_inthandler) - SAVE_ALL_INT - GET_CURRENT(%d0) - movel %d0,%a1 - addqb #1,%a1@(TINFO_PREEMPT+1) - | put exception # in d0 - bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0 - subw #VEC_SPUR,%d0 - - movel %sp,%sp@- - movel %d0,%sp@- | put vector # on stack -auto_irqhandler_fixup = . + 2 - jsr do_IRQ | process the IRQ - addql #8,%sp | pop parameters off stack - -ret_from_interrupt: - movel %curptr@(TASK_STACK),%a1 - subqb #1,%a1@(TINFO_PREEMPT+1) - jeq ret_from_last_interrupt -2: RESTORE_ALL - - ALIGN -ret_from_last_interrupt: - moveq #(~ALLOWINT>>8)&0xff,%d0 - andb %sp@(PT_OFF_SR),%d0 - jne 2b - - /* check if we need to do software interrupts */ - tstl irq_stat+CPUSTAT_SOFTIRQ_PENDING - jeq .Lret_from_exception - pea ret_from_exception - jra do_softirq - -/* Handler for user defined interrupt vectors */ - -ENTRY(user_inthandler) - SAVE_ALL_INT - GET_CURRENT(%d0) - movel %d0,%a1 - addqb #1,%a1@(TINFO_PREEMPT+1) - | put exception # in d0 - bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0 -user_irqvec_fixup = . + 2 - subw #VEC_USER,%d0 - - movel %sp,%sp@- - movel %d0,%sp@- | put vector # on stack - jsr do_IRQ | process the IRQ - addql #8,%sp | pop parameters off stack - - movel %curptr@(TASK_STACK),%a1 - subqb #1,%a1@(TINFO_PREEMPT+1) - jeq ret_from_last_interrupt - RESTORE_ALL - -/* Handler for uninitialized and spurious interrupts */ - -ENTRY(bad_inthandler) - SAVE_ALL_INT - GET_CURRENT(%d0) - movel %d0,%a1 - addqb #1,%a1@(TINFO_PREEMPT+1) - - movel %sp,%sp@- - jsr handle_badint - addql #4,%sp - - movel %curptr@(TASK_STACK),%a1 - subqb #1,%a1@(TINFO_PREEMPT+1) - jeq ret_from_last_interrupt - RESTORE_ALL - - -ENTRY(sys_fork) - SAVE_SWITCH_STACK - pea %sp@(SWITCH_STACK_SIZE) - jbsr m68k_fork - addql #4,%sp - RESTORE_SWITCH_STACK - rts - -ENTRY(sys_clone) - SAVE_SWITCH_STACK - pea %sp@(SWITCH_STACK_SIZE) - jbsr m68k_clone - addql #4,%sp - RESTORE_SWITCH_STACK - rts - -ENTRY(sys_vfork) - SAVE_SWITCH_STACK - pea %sp@(SWITCH_STACK_SIZE) - jbsr m68k_vfork - addql #4,%sp - RESTORE_SWITCH_STACK - rts - -ENTRY(sys_sigreturn) - SAVE_SWITCH_STACK - jbsr do_sigreturn - RESTORE_SWITCH_STACK - rts - -ENTRY(sys_rt_sigreturn) - SAVE_SWITCH_STACK - jbsr do_rt_sigreturn - RESTORE_SWITCH_STACK - rts - -resume: - /* - * Beware - when entering resume, prev (the current task) is - * in a0, next (the new task) is in a1,so don't change these - * registers until their contents are no longer needed. - */ - - /* save sr */ - movew %sr,%a0@(TASK_THREAD+THREAD_SR) - - /* save fs (sfc,%dfc) (may be pointing to kernel memory) */ - movec %sfc,%d0 - movew %d0,%a0@(TASK_THREAD+THREAD_FS) - - /* save usp */ - /* it is better to use a movel here instead of a movew 8*) */ - movec %usp,%d0 - movel %d0,%a0@(TASK_THREAD+THREAD_USP) - - /* save non-scratch registers on stack */ - SAVE_SWITCH_STACK - - /* save current kernel stack pointer */ - movel %sp,%a0@(TASK_THREAD+THREAD_KSP) - - /* save floating point context */ -#ifndef CONFIG_M68KFPU_EMU_ONLY -#ifdef CONFIG_M68KFPU_EMU - tstl m68k_fputype - jeq 3f -#endif - fsave %a0@(TASK_THREAD+THREAD_FPSTATE) - -#if defined(CONFIG_M68060) -#if !defined(CPU_M68060_ONLY) - btst #3,m68k_cputype+3 - beqs 1f -#endif - /* The 060 FPU keeps status in bits 15-8 of the first longword */ - tstb %a0@(TASK_THREAD+THREAD_FPSTATE+2) - jeq 3f -#if !defined(CPU_M68060_ONLY) - jra 2f -#endif -#endif /* CONFIG_M68060 */ -#if !defined(CPU_M68060_ONLY) -1: tstb %a0@(TASK_THREAD+THREAD_FPSTATE) - jeq 3f -#endif -2: fmovemx %fp0-%fp7,%a0@(TASK_THREAD+THREAD_FPREG) - fmoveml %fpcr/%fpsr/%fpiar,%a0@(TASK_THREAD+THREAD_FPCNTL) -3: -#endif /* CONFIG_M68KFPU_EMU_ONLY */ - /* Return previous task in %d1 */ - movel %curptr,%d1 - - /* switch to new task (a1 contains new task) */ - movel %a1,%curptr - - /* restore floating point context */ -#ifndef CONFIG_M68KFPU_EMU_ONLY -#ifdef CONFIG_M68KFPU_EMU - tstl m68k_fputype - jeq 4f -#endif -#if defined(CONFIG_M68060) -#if !defined(CPU_M68060_ONLY) - btst #3,m68k_cputype+3 - beqs 1f -#endif - /* The 060 FPU keeps status in bits 15-8 of the first longword */ - tstb %a1@(TASK_THREAD+THREAD_FPSTATE+2) - jeq 3f -#if !defined(CPU_M68060_ONLY) - jra 2f -#endif -#endif /* CONFIG_M68060 */ -#if !defined(CPU_M68060_ONLY) -1: tstb %a1@(TASK_THREAD+THREAD_FPSTATE) - jeq 3f -#endif -2: fmovemx %a1@(TASK_THREAD+THREAD_FPREG),%fp0-%fp7 - fmoveml %a1@(TASK_THREAD+THREAD_FPCNTL),%fpcr/%fpsr/%fpiar -3: frestore %a1@(TASK_THREAD+THREAD_FPSTATE) -4: -#endif /* CONFIG_M68KFPU_EMU_ONLY */ - - /* restore the kernel stack pointer */ - movel %a1@(TASK_THREAD+THREAD_KSP),%sp - - /* restore non-scratch registers */ - RESTORE_SWITCH_STACK - - /* restore user stack pointer */ - movel %a1@(TASK_THREAD+THREAD_USP),%a0 - movel %a0,%usp - - /* restore fs (sfc,%dfc) */ - movew %a1@(TASK_THREAD+THREAD_FS),%a0 - movec %a0,%sfc - movec %a0,%dfc - - /* restore status register */ - movew %a1@(TASK_THREAD+THREAD_SR),%sr - - rts - diff --git a/arch/m68k/kernel/entry_no.S b/arch/m68k/kernel/entry_no.S deleted file mode 100644 index d80cba45589f..000000000000 --- a/arch/m68k/kernel/entry_no.S +++ /dev/null @@ -1,130 +0,0 @@ -/* - * linux/arch/m68knommu/kernel/entry.S - * - * Copyright (C) 1999-2002, Greg Ungerer (gerg@snapgear.com) - * Copyright (C) 1998 D. Jeff Dionne <jeff@lineo.ca>, - * Kenneth Albanowski <kjahds@kjahds.com>, - * Copyright (C) 2000 Lineo Inc. (www.lineo.com) - * - * Based on: - * - * linux/arch/m68k/kernel/entry.S - * - * Copyright (C) 1991, 1992 Linus Torvalds - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file README.legal in the main directory of this archive - * for more details. - * - * Linux/m68k support by Hamish Macdonald - * - * 68060 fixes by Jesper Skov - * ColdFire support by Greg Ungerer (gerg@snapgear.com) - * 5307 fixes by David W. Miller - * linux 2.4 support David McCullough <davidm@snapgear.com> - */ - -#include <linux/linkage.h> -#include <asm/errno.h> -#include <asm/setup.h> -#include <asm/segment.h> -#include <asm/asm-offsets.h> -#include <asm/entry.h> -#include <asm/unistd.h> - -.text - -.globl buserr -.globl trap -.globl ret_from_exception -.globl ret_from_signal -.globl sys_fork -.globl sys_clone -.globl sys_vfork - -ENTRY(buserr) - SAVE_ALL_INT - GET_CURRENT(%d0) - movel %sp,%sp@- /* stack frame pointer argument */ - jsr buserr_c - addql #4,%sp - jra ret_from_exception - -ENTRY(trap) - SAVE_ALL_INT - GET_CURRENT(%d0) - movel %sp,%sp@- /* stack frame pointer argument */ - jsr trap_c - addql #4,%sp - jra ret_from_exception - -#ifdef TRAP_DBG_INTERRUPT - -.globl dbginterrupt -ENTRY(dbginterrupt) - SAVE_ALL_INT - GET_CURRENT(%d0) - movel %sp,%sp@- /* stack frame pointer argument */ - jsr dbginterrupt_c - addql #4,%sp - jra ret_from_exception -#endif - -ENTRY(reschedule) - /* save top of frame */ - pea %sp@ - jbsr set_esp0 - addql #4,%sp - pea ret_from_exception - jmp schedule - -ENTRY(ret_from_fork) - movel %d1,%sp@- - jsr schedule_tail - addql #4,%sp - jra ret_from_exception - -ENTRY(sys_fork) - SAVE_SWITCH_STACK - pea %sp@(SWITCH_STACK_SIZE) - jbsr m68k_fork - addql #4,%sp - RESTORE_SWITCH_STACK - rts - -ENTRY(sys_vfork) - SAVE_SWITCH_STACK - pea %sp@(SWITCH_STACK_SIZE) - jbsr m68k_vfork - addql #4,%sp - RESTORE_SWITCH_STACK - rts - -ENTRY(sys_clone) - SAVE_SWITCH_STACK - pea %sp@(SWITCH_STACK_SIZE) - jbsr m68k_clone - addql #4,%sp - RESTORE_SWITCH_STACK - rts - -ENTRY(sys_sigreturn) - SAVE_SWITCH_STACK - jbsr do_sigreturn - RESTORE_SWITCH_STACK - rts - -ENTRY(sys_rt_sigreturn) - SAVE_SWITCH_STACK - jbsr do_rt_sigreturn - RESTORE_SWITCH_STACK - rts - -ENTRY(ret_from_user_signal) - moveq #__NR_sigreturn,%d0 - trap #0 - -ENTRY(ret_from_user_rt_signal) - movel #__NR_rt_sigreturn,%d0 - trap #0 - diff --git a/arch/m68k/kernel/pcibios.c b/arch/m68k/kernel/pcibios.c new file mode 100644 index 000000000000..b2988aa1840b --- /dev/null +++ b/arch/m68k/kernel/pcibios.c @@ -0,0 +1,109 @@ +/* + * pci.c -- basic PCI support code + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + * + * (C) Copyright 2011, Greg Ungerer <gerg@uclinux.org> + */ + +#include <linux/kernel.h> +#include <linux/types.h> +#include <linux/mm.h> +#include <linux/init.h> +#include <linux/pci.h> + +/* + * From arch/i386/kernel/pci-i386.c: + * + * We need to avoid collisions with `mirrored' VGA ports + * and other strange ISA hardware, so we always want the + * addresses to be allocated in the 0x000-0x0ff region + * modulo 0x400. + * + * Why? Because some silly external IO cards only decode + * the low 10 bits of the IO address. The 0x00-0xff region + * is reserved for motherboard devices that decode all 16 + * bits, so it's ok to allocate at, say, 0x2800-0x28ff, + * but we want to try to avoid allocating at 0x2900-0x2bff + * which might be mirrored at 0x0100-0x03ff.. + */ +resource_size_t pcibios_align_resource(void *data, const struct resource *res, + resource_size_t size, resource_size_t align) +{ + resource_size_t start = res->start; + + if ((res->flags & IORESOURCE_IO) && (start & 0x300)) + start = (start + 0x3ff) & ~0x3ff; + + start = (start + align - 1) & ~(align - 1); + + return start; +} + +/* + * This is taken from the ARM code for this. + */ +int pcibios_enable_device(struct pci_dev *dev, int mask) +{ + struct resource *r; + u16 cmd, newcmd; + int idx; + + pci_read_config_word(dev, PCI_COMMAND, &cmd); + newcmd = cmd; + + for (idx = 0; idx < 6; idx++) { + /* Only set up the requested stuff */ + if (!(mask & (1 << idx))) + continue; + + r = dev->resource + idx; + if (!r->start && r->end) { + pr_err(KERN_ERR "PCI: Device %s not available because of resource collisions\n", + pci_name(dev)); + return -EINVAL; + } + if (r->flags & IORESOURCE_IO) + newcmd |= PCI_COMMAND_IO; + if (r->flags & IORESOURCE_MEM) + newcmd |= PCI_COMMAND_MEMORY; + } + + /* + * Bridges (eg, cardbus bridges) need to be fully enabled + */ + if ((dev->class >> 16) == PCI_BASE_CLASS_BRIDGE) + newcmd |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY; + + + if (newcmd != cmd) { + pr_info("PCI: enabling device %s (0x%04x -> 0x%04x)\n", + pci_name(dev), cmd, newcmd); + pci_write_config_word(dev, PCI_COMMAND, newcmd); + } + return 0; +} + +void pcibios_update_irq(struct pci_dev *dev, int irq) +{ + pci_write_config_byte(dev, PCI_INTERRUPT_LINE, irq); +} + +void __devinit pcibios_fixup_bus(struct pci_bus *bus) +{ + struct pci_dev *dev; + + list_for_each_entry(dev, &bus->devices, bus_list) { + pci_write_config_byte(dev, PCI_CACHE_LINE_SIZE, 8); + pci_write_config_byte(dev, PCI_LATENCY_TIMER, 32); + } +} + +char __devinit *pcibios_setup(char *str) +{ + return str; +} + diff --git a/arch/m68k/mm/memory.c b/arch/m68k/mm/memory.c index 250b8b786f4f..51bc9d258ede 100644 --- a/arch/m68k/mm/memory.c +++ b/arch/m68k/mm/memory.c @@ -203,7 +203,7 @@ static inline void pushcl040(unsigned long paddr) void cache_clear (unsigned long paddr, int len) { if (CPU_IS_COLDFIRE) { - flush_cf_bcache(0, DCACHE_MAX_ADDR); + clear_cf_bcache(0, DCACHE_MAX_ADDR); } else if (CPU_IS_040_OR_060) { int tmp; diff --git a/arch/m68k/platform/coldfire/Makefile b/arch/m68k/platform/coldfire/Makefile index 76d389d9a84e..02591a109f8c 100644 --- a/arch/m68k/platform/coldfire/Makefile +++ b/arch/m68k/platform/coldfire/Makefile @@ -20,6 +20,7 @@ obj-$(CONFIG_M5206e) += m5206.o timers.o intc.o reset.o obj-$(CONFIG_M520x) += m520x.o pit.o intc-simr.o reset.o obj-$(CONFIG_M523x) += m523x.o pit.o dma_timer.o intc-2.o reset.o obj-$(CONFIG_M5249) += m5249.o timers.o intc.o intc-5249.o reset.o +obj-$(CONFIG_M525x) += m525x.o timers.o intc.o intc-525x.o reset.o obj-$(CONFIG_M527x) += m527x.o pit.o intc-2.o reset.o obj-$(CONFIG_M5272) += m5272.o intc-5272.o timers.o obj-$(CONFIG_M528x) += m528x.o pit.o intc-2.o reset.o @@ -27,10 +28,14 @@ obj-$(CONFIG_M5307) += m5307.o timers.o intc.o reset.o obj-$(CONFIG_M532x) += m532x.o timers.o intc-simr.o reset.o obj-$(CONFIG_M5407) += m5407.o timers.o intc.o reset.o obj-$(CONFIG_M54xx) += m54xx.o sltimers.o intc-2.o +obj-$(CONFIG_M5441x) += m5441x.o pit.o intc-simr.o reset.o obj-$(CONFIG_NETtel) += nettel.o obj-$(CONFIG_CLEOPATRA) += nettel.o obj-$(CONFIG_FIREBEE) += firebee.o +obj-$(CONFIG_MCF8390) += mcf8390.o -obj-y += pinmux.o gpio.o +obj-$(CONFIG_PCI) += pci.o + +obj-y += gpio.o extra-y := head.o diff --git a/arch/m68k/platform/coldfire/clk.c b/arch/m68k/platform/coldfire/clk.c index 44da406897e5..75f9ee967ea7 100644 --- a/arch/m68k/platform/coldfire/clk.c +++ b/arch/m68k/platform/coldfire/clk.c @@ -10,11 +10,17 @@ #include <linux/kernel.h> #include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/mutex.h> #include <linux/clk.h> +#include <linux/io.h> +#include <linux/err.h> #include <asm/coldfire.h> +#include <asm/mcfsim.h> +#include <asm/mcfclk.h> /***************************************************************************/ - +#ifndef MCFPM_PPMCR0 struct clk *clk_get(struct device *dev, const char *id) { return NULL; @@ -42,11 +48,107 @@ unsigned long clk_get_rate(struct clk *clk) return MCF_CLK; } EXPORT_SYMBOL(clk_get_rate); +#else +static DEFINE_SPINLOCK(clk_lock); + +struct clk *clk_get(struct device *dev, const char *id) +{ + const char *clk_name = dev ? dev_name(dev) : id ? id : NULL; + struct clk *clk; + unsigned i; + + for (i = 0; (clk = mcf_clks[i]) != NULL; ++i) + if (!strcmp(clk->name, clk_name)) + return clk; + pr_warn("clk_get: didn't find clock %s\n", clk_name); + return ERR_PTR(-ENOENT); +} +EXPORT_SYMBOL(clk_get); + +int clk_enable(struct clk *clk) +{ + unsigned long flags; + spin_lock_irqsave(&clk_lock, flags); + if ((clk->enabled++ == 0) && clk->clk_ops) + clk->clk_ops->enable(clk); + spin_unlock_irqrestore(&clk_lock, flags); + + return 0; +} +EXPORT_SYMBOL(clk_enable); + +void clk_disable(struct clk *clk) +{ + unsigned long flags; + spin_lock_irqsave(&clk_lock, flags); + if ((--clk->enabled == 0) && clk->clk_ops) + clk->clk_ops->disable(clk); + spin_unlock_irqrestore(&clk_lock, flags); +} +EXPORT_SYMBOL(clk_disable); + +void clk_put(struct clk *clk) +{ + if (clk->enabled != 0) + pr_warn("clk_put %s still enabled\n", clk->name); +} +EXPORT_SYMBOL(clk_put); + +unsigned long clk_get_rate(struct clk *clk) +{ + return clk->rate; +} +EXPORT_SYMBOL(clk_get_rate); + +/***************************************************************************/ + +void __clk_init_enabled(struct clk *clk) +{ + clk->enabled = 1; + clk->clk_ops->enable(clk); +} + +void __clk_init_disabled(struct clk *clk) +{ + clk->enabled = 0; + clk->clk_ops->disable(clk); +} + +static void __clk_enable0(struct clk *clk) +{ + __raw_writeb(clk->slot, MCFPM_PPMCR0); +} + +static void __clk_disable0(struct clk *clk) +{ + __raw_writeb(clk->slot, MCFPM_PPMSR0); +} + +struct clk_ops clk_ops0 = { + .enable = __clk_enable0, + .disable = __clk_disable0, +}; + +#ifdef MCFPM_PPMCR1 +static void __clk_enable1(struct clk *clk) +{ + __raw_writeb(clk->slot, MCFPM_PPMCR1); +} + +static void __clk_disable1(struct clk *clk) +{ + __raw_writeb(clk->slot, MCFPM_PPMSR1); +} + +struct clk_ops clk_ops1 = { + .enable = __clk_enable1, + .disable = __clk_disable1, +}; +#endif /* MCFPM_PPMCR1 */ +#endif /* MCFPM_PPMCR0 */ struct clk *devm_clk_get(struct device *dev, const char *id) { return NULL; } EXPORT_SYMBOL(devm_clk_get); - -/***************************************************************************/ diff --git a/arch/m68k/platform/coldfire/device.c b/arch/m68k/platform/coldfire/device.c index 3aa77ddea89d..81f0fb5e51cf 100644 --- a/arch/m68k/platform/coldfire/device.c +++ b/arch/m68k/platform/coldfire/device.c @@ -13,6 +13,7 @@ #include <linux/io.h> #include <linux/spi/spi.h> #include <linux/gpio.h> +#include <linux/fec.h> #include <asm/traps.h> #include <asm/coldfire.h> #include <asm/mcfsim.h> @@ -20,7 +21,7 @@ #include <asm/mcfqspi.h> /* - * All current ColdFire parts contain from 2, 3 or 4 UARTS. + * All current ColdFire parts contain from 2, 3, 4 or 10 UARTS. */ static struct mcf_platform_uart mcf_uart_platform_data[] = { { @@ -43,6 +44,42 @@ static struct mcf_platform_uart mcf_uart_platform_data[] = { .irq = MCF_IRQ_UART3, }, #endif +#ifdef MCFUART_BASE4 + { + .mapbase = MCFUART_BASE4, + .irq = MCF_IRQ_UART4, + }, +#endif +#ifdef MCFUART_BASE5 + { + .mapbase = MCFUART_BASE5, + .irq = MCF_IRQ_UART5, + }, +#endif +#ifdef MCFUART_BASE6 + { + .mapbase = MCFUART_BASE6, + .irq = MCF_IRQ_UART6, + }, +#endif +#ifdef MCFUART_BASE7 + { + .mapbase = MCFUART_BASE7, + .irq = MCF_IRQ_UART7, + }, +#endif +#ifdef MCFUART_BASE8 + { + .mapbase = MCFUART_BASE8, + .irq = MCF_IRQ_UART8, + }, +#endif +#ifdef MCFUART_BASE9 + { + .mapbase = MCFUART_BASE9, + .irq = MCF_IRQ_UART9, + }, +#endif { }, }; @@ -53,6 +90,18 @@ static struct platform_device mcf_uart = { }; #ifdef CONFIG_FEC + +#ifdef CONFIG_M5441x +#define FEC_NAME "enet-fec" +static struct fec_platform_data fec_pdata = { + .phy = PHY_INTERFACE_MODE_RMII, +}; +#define FEC_PDATA (&fec_pdata) +#else +#define FEC_NAME "fec" +#define FEC_PDATA NULL +#endif + /* * Some ColdFire cores contain the Fast Ethernet Controller (FEC) * block. It is Freescale's own hardware block. Some ColdFires @@ -82,10 +131,11 @@ static struct resource mcf_fec0_resources[] = { }; static struct platform_device mcf_fec0 = { - .name = "fec", + .name = FEC_NAME, .id = 0, .num_resources = ARRAY_SIZE(mcf_fec0_resources), .resource = mcf_fec0_resources, + .dev.platform_data = FEC_PDATA, }; #ifdef MCFFEC_BASE1 @@ -113,10 +163,11 @@ static struct resource mcf_fec1_resources[] = { }; static struct platform_device mcf_fec1 = { - .name = "fec", + .name = FEC_NAME, .id = 1, .num_resources = ARRAY_SIZE(mcf_fec1_resources), .resource = mcf_fec1_resources, + .dev.platform_data = FEC_PDATA, }; #endif /* MCFFEC_BASE1 */ #endif /* CONFIG_FEC */ diff --git a/arch/m68k/platform/coldfire/gpio.c b/arch/m68k/platform/coldfire/gpio.c index 4c8c42450a4e..9cd2b5c70519 100644 --- a/arch/m68k/platform/coldfire/gpio.c +++ b/arch/m68k/platform/coldfire/gpio.c @@ -14,119 +14,161 @@ */ #include <linux/kernel.h> +#include <linux/module.h> #include <linux/init.h> #include <linux/device.h> -#include <asm/gpio.h> -#include <asm/pinmux.h> +#include <linux/io.h> +#include <asm/coldfire.h> +#include <asm/mcfsim.h> #include <asm/mcfgpio.h> -#define MCF_CHIP(chip) container_of(chip, struct mcf_gpio_chip, gpio_chip) +int __mcfgpio_get_value(unsigned gpio) +{ + return mcfgpio_read(__mcfgpio_ppdr(gpio)) & mcfgpio_bit(gpio); +} +EXPORT_SYMBOL(__mcfgpio_get_value); + +void __mcfgpio_set_value(unsigned gpio, int value) +{ + if (gpio < MCFGPIO_SCR_START) { + unsigned long flags; + MCFGPIO_PORTTYPE data; + + local_irq_save(flags); + data = mcfgpio_read(__mcfgpio_podr(gpio)); + if (value) + data |= mcfgpio_bit(gpio); + else + data &= ~mcfgpio_bit(gpio); + mcfgpio_write(data, __mcfgpio_podr(gpio)); + local_irq_restore(flags); + } else { + if (value) + mcfgpio_write(mcfgpio_bit(gpio), + MCFGPIO_SETR_PORT(gpio)); + else + mcfgpio_write(~mcfgpio_bit(gpio), + MCFGPIO_CLRR_PORT(gpio)); + } +} +EXPORT_SYMBOL(__mcfgpio_set_value); -int mcf_gpio_direction_input(struct gpio_chip *chip, unsigned offset) +int __mcfgpio_direction_input(unsigned gpio) { unsigned long flags; MCFGPIO_PORTTYPE dir; - struct mcf_gpio_chip *mcf_chip = MCF_CHIP(chip); local_irq_save(flags); - dir = mcfgpio_read(mcf_chip->pddr); - dir &= ~mcfgpio_bit(chip->base + offset); - mcfgpio_write(dir, mcf_chip->pddr); + dir = mcfgpio_read(__mcfgpio_pddr(gpio)); + dir &= ~mcfgpio_bit(gpio); + mcfgpio_write(dir, __mcfgpio_pddr(gpio)); local_irq_restore(flags); return 0; } +EXPORT_SYMBOL(__mcfgpio_direction_input); -int mcf_gpio_get_value(struct gpio_chip *chip, unsigned offset) -{ - struct mcf_gpio_chip *mcf_chip = MCF_CHIP(chip); - - return mcfgpio_read(mcf_chip->ppdr) & mcfgpio_bit(chip->base + offset); -} - -int mcf_gpio_direction_output(struct gpio_chip *chip, unsigned offset, - int value) +int __mcfgpio_direction_output(unsigned gpio, int value) { unsigned long flags; MCFGPIO_PORTTYPE data; - struct mcf_gpio_chip *mcf_chip = MCF_CHIP(chip); local_irq_save(flags); - /* write the value to the output latch */ - data = mcfgpio_read(mcf_chip->podr); + data = mcfgpio_read(__mcfgpio_pddr(gpio)); if (value) - data |= mcfgpio_bit(chip->base + offset); + data |= mcfgpio_bit(gpio); else - data &= ~mcfgpio_bit(chip->base + offset); - mcfgpio_write(data, mcf_chip->podr); - - /* now set the direction to output */ - data = mcfgpio_read(mcf_chip->pddr); - data |= mcfgpio_bit(chip->base + offset); - mcfgpio_write(data, mcf_chip->pddr); + data &= mcfgpio_bit(gpio); + mcfgpio_write(data, __mcfgpio_pddr(gpio)); + + /* now set the data to output */ + if (gpio < MCFGPIO_SCR_START) { + data = mcfgpio_read(__mcfgpio_podr(gpio)); + if (value) + data |= mcfgpio_bit(gpio); + else + data &= ~mcfgpio_bit(gpio); + mcfgpio_write(data, __mcfgpio_podr(gpio)); + } else { + if (value) + mcfgpio_write(mcfgpio_bit(gpio), + MCFGPIO_SETR_PORT(gpio)); + else + mcfgpio_write(~mcfgpio_bit(gpio), + MCFGPIO_CLRR_PORT(gpio)); + } local_irq_restore(flags); + return 0; +} +EXPORT_SYMBOL(__mcfgpio_direction_output); +int __mcfgpio_request(unsigned gpio) +{ return 0; } +EXPORT_SYMBOL(__mcfgpio_request); -void mcf_gpio_set_value(struct gpio_chip *chip, unsigned offset, int value) +void __mcfgpio_free(unsigned gpio) { - struct mcf_gpio_chip *mcf_chip = MCF_CHIP(chip); + __mcfgpio_direction_input(gpio); +} +EXPORT_SYMBOL(__mcfgpio_free); - unsigned long flags; - MCFGPIO_PORTTYPE data; +#ifdef CONFIG_GPIOLIB - local_irq_save(flags); - data = mcfgpio_read(mcf_chip->podr); - if (value) - data |= mcfgpio_bit(chip->base + offset); - else - data &= ~mcfgpio_bit(chip->base + offset); - mcfgpio_write(data, mcf_chip->podr); - local_irq_restore(flags); +int mcfgpio_direction_input(struct gpio_chip *chip, unsigned offset) +{ + return __mcfgpio_direction_input(offset); } -void mcf_gpio_set_value_fast(struct gpio_chip *chip, unsigned offset, int value) +int mcfgpio_get_value(struct gpio_chip *chip, unsigned offset) { - struct mcf_gpio_chip *mcf_chip = MCF_CHIP(chip); - - if (value) - mcfgpio_write(mcfgpio_bit(chip->base + offset), mcf_chip->setr); - else - mcfgpio_write(~mcfgpio_bit(chip->base + offset), mcf_chip->clrr); + return __mcfgpio_get_value(offset); } -int mcf_gpio_request(struct gpio_chip *chip, unsigned offset) +int mcfgpio_direction_output(struct gpio_chip *chip, unsigned offset, int value) { - struct mcf_gpio_chip *mcf_chip = MCF_CHIP(chip); - - return mcf_chip->gpio_to_pinmux ? - mcf_pinmux_request(mcf_chip->gpio_to_pinmux[offset], 0) : 0; + return __mcfgpio_direction_output(offset, value); } -void mcf_gpio_free(struct gpio_chip *chip, unsigned offset) +void mcfgpio_set_value(struct gpio_chip *chip, unsigned offset, int value) { - struct mcf_gpio_chip *mcf_chip = MCF_CHIP(chip); + __mcfgpio_set_value(offset, value); +} - mcf_gpio_direction_input(chip, offset); +int mcfgpio_request(struct gpio_chip *chip, unsigned offset) +{ + return __mcfgpio_request(offset); +} - if (mcf_chip->gpio_to_pinmux) - mcf_pinmux_release(mcf_chip->gpio_to_pinmux[offset], 0); +void mcfgpio_free(struct gpio_chip *chip, unsigned offset) +{ + __mcfgpio_free(offset); } -struct bus_type mcf_gpio_subsys = { +struct bus_type mcfgpio_subsys = { .name = "gpio", .dev_name = "gpio", }; -static int __init mcf_gpio_sysinit(void) -{ - unsigned int i = 0; +static struct gpio_chip mcfgpio_chip = { + .label = "mcfgpio", + .request = mcfgpio_request, + .free = mcfgpio_free, + .direction_input = mcfgpio_direction_input, + .direction_output = mcfgpio_direction_output, + .get = mcfgpio_get_value, + .set = mcfgpio_set_value, + .base = 0, + .ngpio = MCFGPIO_PIN_MAX, +}; - while (i < mcf_gpio_chips_size) - gpiochip_add((struct gpio_chip *)&mcf_gpio_chips[i++]); - return subsys_system_register(&mcf_gpio_subsys, NULL); +static int __init mcfgpio_sysinit(void) +{ + gpiochip_add(&mcfgpio_chip); + return subsys_system_register(&mcfgpio_subsys, NULL); } -core_initcall(mcf_gpio_sysinit); +core_initcall(mcfgpio_sysinit); +#endif diff --git a/arch/m68k/platform/coldfire/head.S b/arch/m68k/platform/coldfire/head.S index c3db70ed33b3..4e0c9eb3bd1f 100644 --- a/arch/m68k/platform/coldfire/head.S +++ b/arch/m68k/platform/coldfire/head.S @@ -31,9 +31,9 @@ .endm #elif defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \ - defined(CONFIG_M5249) || defined(CONFIG_M527x) || \ - defined(CONFIG_M528x) || defined(CONFIG_M5307) || \ - defined(CONFIG_M5407) + defined(CONFIG_M5249) || defined(CONFIG_M525x) || \ + defined(CONFIG_M527x) || defined(CONFIG_M528x) || \ + defined(CONFIG_M5307) || defined(CONFIG_M5407) /* * Not all these devices have exactly the same DRAM controller, * but the DCMR register is virtually identical - give or take diff --git a/arch/m68k/platform/coldfire/intc-525x.c b/arch/m68k/platform/coldfire/intc-525x.c new file mode 100644 index 000000000000..b23204d059ac --- /dev/null +++ b/arch/m68k/platform/coldfire/intc-525x.c @@ -0,0 +1,91 @@ +/* + * intc2.c -- support for the 2nd INTC controller of the 525x + * + * (C) Copyright 2012, Steven King <sfking@fdwdc.com> + * (C) Copyright 2009, Greg Ungerer <gerg@snapgear.com> + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file COPYING in the main directory of this archive + * for more details. + */ + +#include <linux/types.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/io.h> +#include <asm/coldfire.h> +#include <asm/mcfsim.h> + +static void intc2_irq_gpio_mask(struct irq_data *d) +{ + u32 imr = readl(MCFSIM2_GPIOINTENABLE); + u32 type = irqd_get_trigger_type(d); + int irq = d->irq - MCF_IRQ_GPIO0; + + if (type & IRQ_TYPE_EDGE_RISING) + imr &= ~(0x001 << irq); + if (type & IRQ_TYPE_EDGE_FALLING) + imr &= ~(0x100 << irq); + writel(imr, MCFSIM2_GPIOINTENABLE); +} + +static void intc2_irq_gpio_unmask(struct irq_data *d) +{ + u32 imr = readl(MCFSIM2_GPIOINTENABLE); + u32 type = irqd_get_trigger_type(d); + int irq = d->irq - MCF_IRQ_GPIO0; + + if (type & IRQ_TYPE_EDGE_RISING) + imr |= (0x001 << irq); + if (type & IRQ_TYPE_EDGE_FALLING) + imr |= (0x100 << irq); + writel(imr, MCFSIM2_GPIOINTENABLE); +} + +static void intc2_irq_gpio_ack(struct irq_data *d) +{ + u32 imr = 0; + u32 type = irqd_get_trigger_type(d); + int irq = d->irq - MCF_IRQ_GPIO0; + + if (type & IRQ_TYPE_EDGE_RISING) + imr |= (0x001 << irq); + if (type & IRQ_TYPE_EDGE_FALLING) + imr |= (0x100 << irq); + writel(imr, MCFSIM2_GPIOINTCLEAR); +} + +static int intc2_irq_gpio_set_type(struct irq_data *d, unsigned int f) +{ + if (f & ~IRQ_TYPE_EDGE_BOTH) + return -EINVAL; + return 0; +} + +static struct irq_chip intc2_irq_gpio_chip = { + .name = "CF-INTC2", + .irq_mask = intc2_irq_gpio_mask, + .irq_unmask = intc2_irq_gpio_unmask, + .irq_ack = intc2_irq_gpio_ack, + .irq_set_type = intc2_irq_gpio_set_type, +}; + +static int __init mcf_intc2_init(void) +{ + int irq; + + /* set the interrupt base for the second interrupt controller */ + writel(MCFINTC2_VECBASE, MCFINTC2_INTBASE); + + /* GPIO interrupt sources */ + for (irq = MCF_IRQ_GPIO0; (irq <= MCF_IRQ_GPIO6); irq++) { + irq_set_chip(irq, &intc2_irq_gpio_chip); + irq_set_handler(irq, handle_edge_irq); + } + + return 0; +} + +arch_initcall(mcf_intc2_init); diff --git a/arch/m68k/platform/coldfire/intc-simr.c b/arch/m68k/platform/coldfire/intc-simr.c index 650d52e2927e..7cf2c156f72d 100644 --- a/arch/m68k/platform/coldfire/intc-simr.c +++ b/arch/m68k/platform/coldfire/intc-simr.c @@ -59,16 +59,18 @@ static unsigned int inline irq2ebit(unsigned int irq) #endif /* - * There maybe one or two interrupt control units, each has 64 - * interrupts. If there is no second unit then MCFINTC1_* defines - * will be 0 (and code for them optimized away). + * There maybe one, two or three interrupt control units, each has 64 + * interrupts. If there is no second or third unit then MCFINTC1_* or + * MCFINTC2_* defines will be 0 (and code for them optimized away). */ static void intc_irq_mask(struct irq_data *d) { unsigned int irq = d->irq - MCFINT_VECBASE; - if (MCFINTC1_SIMR && (irq > 64)) + if (MCFINTC2_SIMR && (irq > 128)) + __raw_writeb(irq - 128, MCFINTC2_SIMR); + else if (MCFINTC1_SIMR && (irq > 64)) __raw_writeb(irq - 64, MCFINTC1_SIMR); else __raw_writeb(irq, MCFINTC0_SIMR); @@ -78,7 +80,9 @@ static void intc_irq_unmask(struct irq_data *d) { unsigned int irq = d->irq - MCFINT_VECBASE; - if (MCFINTC1_CIMR && (irq > 64)) + if (MCFINTC2_CIMR && (irq > 128)) + __raw_writeb(irq - 128, MCFINTC2_CIMR); + else if (MCFINTC1_CIMR && (irq > 64)) __raw_writeb(irq - 64, MCFINTC1_CIMR); else __raw_writeb(irq, MCFINTC0_CIMR); @@ -99,9 +103,11 @@ static unsigned int intc_irq_startup(struct irq_data *d) unsigned int ebit = irq2ebit(irq); u8 v; +#if defined(MCFEPORT_EPDDR) /* Set EPORT line as input */ v = __raw_readb(MCFEPORT_EPDDR); __raw_writeb(v & ~(0x1 << ebit), MCFEPORT_EPDDR); +#endif /* Set EPORT line as interrupt source */ v = __raw_readb(MCFEPORT_EPIER); @@ -109,12 +115,13 @@ static unsigned int intc_irq_startup(struct irq_data *d) } irq -= MCFINT_VECBASE; - if (MCFINTC1_ICR0 && (irq > 64)) + if (MCFINTC2_ICR0 && (irq > 128)) + __raw_writeb(5, MCFINTC2_ICR0 + irq - 128); + else if (MCFINTC1_ICR0 && (irq > 64)) __raw_writeb(5, MCFINTC1_ICR0 + irq - 64); else __raw_writeb(5, MCFINTC0_ICR0 + irq); - intc_irq_unmask(d); return 0; } @@ -175,8 +182,11 @@ void __init init_IRQ(void) __raw_writeb(0xff, MCFINTC0_SIMR); if (MCFINTC1_SIMR) __raw_writeb(0xff, MCFINTC1_SIMR); + if (MCFINTC2_SIMR) + __raw_writeb(0xff, MCFINTC2_SIMR); - eirq = MCFINT_VECBASE + 64 + (MCFINTC1_ICR0 ? 64 : 0); + eirq = MCFINT_VECBASE + 64 + (MCFINTC1_ICR0 ? 64 : 0) + + (MCFINTC2_ICR0 ? 64 : 0); for (irq = MCFINT_VECBASE; (irq < eirq); irq++) { if ((irq >= EINT1) && (irq <= EINT7)) irq_set_chip(irq, &intc_irq_chip_edge_port); diff --git a/arch/m68k/platform/coldfire/m5206.c b/arch/m68k/platform/coldfire/m5206.c index a8b81df653f0..6bfbeebd231b 100644 --- a/arch/m68k/platform/coldfire/m5206.c +++ b/arch/m68k/platform/coldfire/m5206.c @@ -16,15 +16,6 @@ #include <asm/machdep.h> #include <asm/coldfire.h> #include <asm/mcfsim.h> -#include <asm/mcfgpio.h> - -/***************************************************************************/ - -struct mcf_gpio_chip mcf_gpio_chips[] = { - MCFGPS(PP, 0, 8, MCFSIM_PADDR, MCFSIM_PADAT, MCFSIM_PADAT), -}; - -unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips); /***************************************************************************/ diff --git a/arch/m68k/platform/coldfire/m520x.c b/arch/m68k/platform/coldfire/m520x.c index 3264b8883d5f..ea1be0e98ad6 100644 --- a/arch/m68k/platform/coldfire/m520x.c +++ b/arch/m68k/platform/coldfire/m520x.c @@ -19,22 +19,102 @@ #include <asm/coldfire.h> #include <asm/mcfsim.h> #include <asm/mcfuart.h> -#include <asm/mcfgpio.h> +#include <asm/mcfclk.h> /***************************************************************************/ -struct mcf_gpio_chip mcf_gpio_chips[] = { - MCFGPS(PIRQ, 0, 8, MCFEPORT_EPDDR, MCFEPORT_EPDR, MCFEPORT_EPPDR), - MCFGPF(CS, 9, 3), - MCFGPF(FECI2C, 16, 4), - MCFGPF(QSPI, 24, 4), - MCFGPF(TIMER, 32, 4), - MCFGPF(UART, 40, 8), - MCFGPF(FECH, 48, 8), - MCFGPF(FECL, 56, 8), +DEFINE_CLK(0, "flexbus", 2, MCF_CLK); +DEFINE_CLK(0, "fec.0", 12, MCF_CLK); +DEFINE_CLK(0, "edma", 17, MCF_CLK); +DEFINE_CLK(0, "intc.0", 18, MCF_CLK); +DEFINE_CLK(0, "iack.0", 21, MCF_CLK); +DEFINE_CLK(0, "mcfi2c.0", 22, MCF_CLK); +DEFINE_CLK(0, "mcfqspi.0", 23, MCF_CLK); +DEFINE_CLK(0, "mcfuart.0", 24, MCF_BUSCLK); +DEFINE_CLK(0, "mcfuart.1", 25, MCF_BUSCLK); +DEFINE_CLK(0, "mcfuart.2", 26, MCF_BUSCLK); +DEFINE_CLK(0, "mcftmr.0", 28, MCF_CLK); +DEFINE_CLK(0, "mcftmr.1", 29, MCF_CLK); +DEFINE_CLK(0, "mcftmr.2", 30, MCF_CLK); +DEFINE_CLK(0, "mcftmr.3", 31, MCF_CLK); + +DEFINE_CLK(0, "mcfpit.0", 32, MCF_CLK); +DEFINE_CLK(0, "mcfpit.1", 33, MCF_CLK); +DEFINE_CLK(0, "mcfeport.0", 34, MCF_CLK); +DEFINE_CLK(0, "mcfwdt.0", 35, MCF_CLK); +DEFINE_CLK(0, "pll.0", 36, MCF_CLK); +DEFINE_CLK(0, "sys.0", 40, MCF_BUSCLK); +DEFINE_CLK(0, "gpio.0", 41, MCF_BUSCLK); +DEFINE_CLK(0, "sdram.0", 42, MCF_CLK); + +struct clk *mcf_clks[] = { + &__clk_0_2, /* flexbus */ + &__clk_0_12, /* fec.0 */ + &__clk_0_17, /* edma */ + &__clk_0_18, /* intc.0 */ + &__clk_0_21, /* iack.0 */ + &__clk_0_22, /* mcfi2c.0 */ + &__clk_0_23, /* mcfqspi.0 */ + &__clk_0_24, /* mcfuart.0 */ + &__clk_0_25, /* mcfuart.1 */ + &__clk_0_26, /* mcfuart.2 */ + &__clk_0_28, /* mcftmr.0 */ + &__clk_0_29, /* mcftmr.1 */ + &__clk_0_30, /* mcftmr.2 */ + &__clk_0_31, /* mcftmr.3 */ + + &__clk_0_32, /* mcfpit.0 */ + &__clk_0_33, /* mcfpit.1 */ + &__clk_0_34, /* mcfeport.0 */ + &__clk_0_35, /* mcfwdt.0 */ + &__clk_0_36, /* pll.0 */ + &__clk_0_40, /* sys.0 */ + &__clk_0_41, /* gpio.0 */ + &__clk_0_42, /* sdram.0 */ +NULL, }; -unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips); +static struct clk * const enable_clks[] __initconst = { + &__clk_0_2, /* flexbus */ + &__clk_0_18, /* intc.0 */ + &__clk_0_21, /* iack.0 */ + &__clk_0_24, /* mcfuart.0 */ + &__clk_0_25, /* mcfuart.1 */ + &__clk_0_26, /* mcfuart.2 */ + + &__clk_0_32, /* mcfpit.0 */ + &__clk_0_33, /* mcfpit.1 */ + &__clk_0_34, /* mcfeport.0 */ + &__clk_0_36, /* pll.0 */ + &__clk_0_40, /* sys.0 */ + &__clk_0_41, /* gpio.0 */ + &__clk_0_42, /* sdram.0 */ +}; + +static struct clk * const disable_clks[] __initconst = { + &__clk_0_12, /* fec.0 */ + &__clk_0_17, /* edma */ + &__clk_0_22, /* mcfi2c.0 */ + &__clk_0_23, /* mcfqspi.0 */ + &__clk_0_28, /* mcftmr.0 */ + &__clk_0_29, /* mcftmr.1 */ + &__clk_0_30, /* mcftmr.2 */ + &__clk_0_31, /* mcftmr.3 */ + &__clk_0_35, /* mcfwdt.0 */ +}; + + +static void __init m520x_clk_init(void) +{ + unsigned i; + + /* make sure these clocks are enabled */ + for (i = 0; i < ARRAY_SIZE(enable_clks); ++i) + __clk_init_enabled(enable_clks[i]); + /* make sure these clocks are disabled */ + for (i = 0; i < ARRAY_SIZE(disable_clks); ++i) + __clk_init_disabled(disable_clks[i]); +} /***************************************************************************/ @@ -93,6 +173,7 @@ static void __init m520x_fec_init(void) void __init config_BSP(char *commandp, int size) { mach_sched_init = hw_timer_init; + m520x_clk_init(); m520x_uarts_init(); m520x_fec_init(); #if IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI) diff --git a/arch/m68k/platform/coldfire/m523x.c b/arch/m68k/platform/coldfire/m523x.c index 5d57a4249412..d47dfd8f50a2 100644 --- a/arch/m68k/platform/coldfire/m523x.c +++ b/arch/m68k/platform/coldfire/m523x.c @@ -19,28 +19,6 @@ #include <asm/machdep.h> #include <asm/coldfire.h> #include <asm/mcfsim.h> -#include <asm/mcfgpio.h> - -/***************************************************************************/ - -struct mcf_gpio_chip mcf_gpio_chips[] = { - MCFGPS(PIRQ, 1, 7, MCFEPORT_EPDDR, MCFEPORT_EPDR, MCFEPORT_EPPDR), - MCFGPF(ADDR, 13, 3), - MCFGPF(DATAH, 16, 8), - MCFGPF(DATAL, 24, 8), - MCFGPF(BUSCTL, 32, 8), - MCFGPF(BS, 40, 4), - MCFGPF(CS, 49, 7), - MCFGPF(SDRAM, 56, 6), - MCFGPF(FECI2C, 64, 4), - MCFGPF(UARTH, 72, 2), - MCFGPF(UARTL, 80, 8), - MCFGPF(QSPI, 88, 5), - MCFGPF(TIMER, 96, 8), - MCFGPF(ETPU, 104, 3), -}; - -unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips); /***************************************************************************/ diff --git a/arch/m68k/platform/coldfire/m5249.c b/arch/m68k/platform/coldfire/m5249.c index fdfa1edfd1ac..300e729a58d0 100644 --- a/arch/m68k/platform/coldfire/m5249.c +++ b/arch/m68k/platform/coldfire/m5249.c @@ -16,16 +16,6 @@ #include <asm/machdep.h> #include <asm/coldfire.h> #include <asm/mcfsim.h> -#include <asm/mcfgpio.h> - -/***************************************************************************/ - -struct mcf_gpio_chip mcf_gpio_chips[] = { - MCFGPS(GPIO0, 0, 32, MCFSIM2_GPIOENABLE, MCFSIM2_GPIOWRITE, MCFSIM2_GPIOREAD), - MCFGPS(GPIO1, 32, 32, MCFSIM2_GPIO1ENABLE, MCFSIM2_GPIO1WRITE, MCFSIM2_GPIO1READ), -}; - -unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips); /***************************************************************************/ diff --git a/arch/m68k/platform/coldfire/m525x.c b/arch/m68k/platform/coldfire/m525x.c new file mode 100644 index 000000000000..8ce905f9b84f --- /dev/null +++ b/arch/m68k/platform/coldfire/m525x.c @@ -0,0 +1,66 @@ +/***************************************************************************/ + +/* + * 525x.c + * + * Copyright (C) 2012, Steven King <sfking@fdwdc.com> + */ + +/***************************************************************************/ + +#include <linux/kernel.h> +#include <linux/param.h> +#include <linux/init.h> +#include <linux/io.h> +#include <linux/platform_device.h> +#include <asm/machdep.h> +#include <asm/coldfire.h> +#include <asm/mcfsim.h> + +/***************************************************************************/ + +static void __init m525x_qspi_init(void) +{ +#if IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI) + /* set the GPIO function for the qspi cs gpios */ + /* FIXME: replace with pinmux/pinctl support */ + u32 f = readl(MCFSIM2_GPIOFUNC); + f |= (1 << MCFQSPI_CS2) | (1 << MCFQSPI_CS1) | (1 << MCFQSPI_CS0); + writel(f, MCFSIM2_GPIOFUNC); + + /* QSPI irq setup */ + writeb(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL4 | MCFSIM_ICR_PRI0, + MCF_MBAR + MCFSIM_QSPIICR); + mcf_mapirq2imr(MCF_IRQ_QSPI, MCFINTC_QSPI); +#endif /* IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI) */ +} + +static void __init m525x_i2c_init(void) +{ +#if IS_ENABLED(CONFIG_I2C_COLDFIRE) + u32 r; + + /* first I2C controller uses regular irq setup */ + writeb(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL5 | MCFSIM_ICR_PRI0, + MCF_MBAR + MCFSIM_I2CICR); + mcf_mapirq2imr(MCF_IRQ_I2C0, MCFINTC_I2C); + + /* second I2C controller is completely different */ + r = readl(MCFINTC2_INTPRI_REG(MCF_IRQ_I2C1)); + r &= ~MCFINTC2_INTPRI_BITS(0xf, MCF_IRQ_I2C1); + r |= MCFINTC2_INTPRI_BITS(0x5, MCF_IRQ_I2C1); + writel(r, MCFINTC2_INTPRI_REG(MCF_IRQ_I2C1)); +#endif /* IS_ENABLED(CONFIG_I2C_COLDFIRE) */ +} + +/***************************************************************************/ + +void __init config_BSP(char *commandp, int size) +{ + mach_sched_init = hw_timer_init; + + m525x_qspi_init(); + m525x_i2c_init(); +} + +/***************************************************************************/ diff --git a/arch/m68k/platform/coldfire/m5272.c b/arch/m68k/platform/coldfire/m5272.c index 43e36060da18..e68bc7a148eb 100644 --- a/arch/m68k/platform/coldfire/m5272.c +++ b/arch/m68k/platform/coldfire/m5272.c @@ -19,7 +19,6 @@ #include <asm/coldfire.h> #include <asm/mcfsim.h> #include <asm/mcfuart.h> -#include <asm/mcfgpio.h> /***************************************************************************/ @@ -31,16 +30,6 @@ unsigned char ledbank = 0xff; /***************************************************************************/ -struct mcf_gpio_chip mcf_gpio_chips[] = { - MCFGPS(PA, 0, 16, MCFSIM_PADDR, MCFSIM_PADAT, MCFSIM_PADAT), - MCFGPS(PB, 16, 16, MCFSIM_PBDDR, MCFSIM_PBDAT, MCFSIM_PBDAT), - MCFGPS(Pc, 32, 16, MCFSIM_PCDDR, MCFSIM_PCDAT, MCFSIM_PCDAT), -}; - -unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips); - -/***************************************************************************/ - static void __init m5272_uarts_init(void) { u32 v; diff --git a/arch/m68k/platform/coldfire/m527x.c b/arch/m68k/platform/coldfire/m527x.c index 9b0b66aabd1b..b3cb378c5e94 100644 --- a/arch/m68k/platform/coldfire/m527x.c +++ b/arch/m68k/platform/coldfire/m527x.c @@ -20,49 +20,6 @@ #include <asm/coldfire.h> #include <asm/mcfsim.h> #include <asm/mcfuart.h> -#include <asm/mcfgpio.h> - -/***************************************************************************/ - -struct mcf_gpio_chip mcf_gpio_chips[] = { -#if defined(CONFIG_M5271) - MCFGPS(PIRQ, 1, 7, MCFEPORT_EPDDR, MCFEPORT_EPDR, MCFEPORT_EPPDR), - MCFGPF(ADDR, 13, 3), - MCFGPF(DATAH, 16, 8), - MCFGPF(DATAL, 24, 8), - MCFGPF(BUSCTL, 32, 8), - MCFGPF(BS, 40, 4), - MCFGPF(CS, 49, 7), - MCFGPF(SDRAM, 56, 6), - MCFGPF(FECI2C, 64, 4), - MCFGPF(UARTH, 72, 2), - MCFGPF(UARTL, 80, 8), - MCFGPF(QSPI, 88, 5), - MCFGPF(TIMER, 96, 8), -#elif defined(CONFIG_M5275) - MCFGPS(PIRQ, 1, 7, MCFEPORT_EPDDR, MCFEPORT_EPDR, MCFEPORT_EPPDR), - MCFGPF(BUSCTL, 8, 8), - MCFGPF(ADDR, 21, 3), - MCFGPF(CS, 25, 7), - MCFGPF(FEC0H, 32, 8), - MCFGPF(FEC0L, 40, 8), - MCFGPF(FECI2C, 48, 6), - MCFGPF(QSPI, 56, 7), - MCFGPF(SDRAM, 64, 8), - MCFGPF(TIMERH, 72, 4), - MCFGPF(TIMERL, 80, 4), - MCFGPF(UARTL, 88, 8), - MCFGPF(FEC1H, 96, 8), - MCFGPF(FEC1L, 104, 8), - MCFGPF(BS, 114, 2), - MCFGPF(IRQ, 121, 7), - MCFGPF(USBH, 128, 1), - MCFGPF(USBL, 136, 8), - MCFGPF(UARTH, 144, 4), -#endif -}; - -unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips); /***************************************************************************/ diff --git a/arch/m68k/platform/coldfire/m528x.c b/arch/m68k/platform/coldfire/m528x.c index 7ed1276b29dc..f1319e5d2546 100644 --- a/arch/m68k/platform/coldfire/m528x.c +++ b/arch/m68k/platform/coldfire/m528x.c @@ -21,37 +21,6 @@ #include <asm/coldfire.h> #include <asm/mcfsim.h> #include <asm/mcfuart.h> -#include <asm/mcfgpio.h> - -/***************************************************************************/ - -struct mcf_gpio_chip mcf_gpio_chips[] = { - MCFGPS(NQ, 1, 7, MCFEPORT_EPDDR, MCFEPORT_EPDR, MCFEPORT_EPPDR), - MCFGPS(TA, 8, 4, MCFGPTA_GPTDDR, MCFGPTA_GPTPORT, MCFGPTB_GPTPORT), - MCFGPS(TB, 16, 4, MCFGPTB_GPTDDR, MCFGPTB_GPTPORT, MCFGPTB_GPTPORT), - MCFGPS(QA, 24, 4, MCFQADC_DDRQA, MCFQADC_PORTQA, MCFQADC_PORTQA), - MCFGPS(QB, 32, 4, MCFQADC_DDRQB, MCFQADC_PORTQB, MCFQADC_PORTQB), - MCFGPF(A, 40, 8), - MCFGPF(B, 48, 8), - MCFGPF(C, 56, 8), - MCFGPF(D, 64, 8), - MCFGPF(E, 72, 8), - MCFGPF(F, 80, 8), - MCFGPF(G, 88, 8), - MCFGPF(H, 96, 8), - MCFGPF(J, 104, 8), - MCFGPF(DD, 112, 8), - MCFGPF(EH, 120, 8), - MCFGPF(EL, 128, 8), - MCFGPF(AS, 136, 6), - MCFGPF(QS, 144, 7), - MCFGPF(SD, 152, 6), - MCFGPF(TC, 160, 4), - MCFGPF(TD, 168, 4), - MCFGPF(UA, 176, 4), -}; - -unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips); /***************************************************************************/ @@ -74,7 +43,7 @@ static void __init m528x_uarts_init(void) /* make sure PUAPAR is set for UART0 and UART1 */ port = readb(MCF5282_GPIO_PUAPAR); port |= 0x03 | (0x03 << 2); - writeb(port, MCF5282_GPIO_PUAPAR); + writeb(port, MCFGPIO_PUAPAR); } /***************************************************************************/ diff --git a/arch/m68k/platform/coldfire/m5307.c b/arch/m68k/platform/coldfire/m5307.c index 93b484976ab3..a568d2870d15 100644 --- a/arch/m68k/platform/coldfire/m5307.c +++ b/arch/m68k/platform/coldfire/m5307.c @@ -16,7 +16,6 @@ #include <asm/machdep.h> #include <asm/coldfire.h> #include <asm/mcfsim.h> -#include <asm/mcfgpio.h> #include <asm/mcfwdebug.h> /***************************************************************************/ @@ -29,14 +28,6 @@ unsigned char ledbank = 0xff; /***************************************************************************/ -struct mcf_gpio_chip mcf_gpio_chips[] = { - MCFGPS(PP, 0, 16, MCFSIM_PADDR, MCFSIM_PADAT, MCFSIM_PADAT), -}; - -unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips); - -/***************************************************************************/ - void __init config_BSP(char *commandp, int size) { #if defined(CONFIG_NETtel) || \ diff --git a/arch/m68k/platform/coldfire/m532x.c b/arch/m68k/platform/coldfire/m532x.c index 5394223639f8..4819a44991ed 100644 --- a/arch/m68k/platform/coldfire/m532x.c +++ b/arch/m68k/platform/coldfire/m532x.c @@ -26,32 +26,144 @@ #include <asm/mcfsim.h> #include <asm/mcfuart.h> #include <asm/mcfdma.h> -#include <asm/mcfgpio.h> #include <asm/mcfwdebug.h> +#include <asm/mcfclk.h> /***************************************************************************/ -struct mcf_gpio_chip mcf_gpio_chips[] = { - MCFGPS(PIRQ, 0, 8, MCFEPORT_EPDDR, MCFEPORT_EPDR, MCFEPORT_EPPDR), - MCFGPF(FECH, 8, 8), - MCFGPF(FECL, 16, 8), - MCFGPF(SSI, 24, 5), - MCFGPF(BUSCTL, 32, 4), - MCFGPF(BE, 40, 4), - MCFGPF(CS, 49, 5), - MCFGPF(PWM, 58, 4), - MCFGPF(FECI2C, 64, 4), - MCFGPF(UART, 72, 8), - MCFGPF(QSPI, 80, 6), - MCFGPF(TIMER, 88, 4), - MCFGPF(LCDDATAH, 96, 2), - MCFGPF(LCDDATAM, 104, 8), - MCFGPF(LCDDATAL, 112, 8), - MCFGPF(LCDCTLH, 120, 1), - MCFGPF(LCDCTLL, 128, 8), +DEFINE_CLK(0, "flexbus", 2, MCF_CLK); +DEFINE_CLK(0, "mcfcan.0", 8, MCF_CLK); +DEFINE_CLK(0, "fec.0", 12, MCF_CLK); +DEFINE_CLK(0, "edma", 17, MCF_CLK); +DEFINE_CLK(0, "intc.0", 18, MCF_CLK); +DEFINE_CLK(0, "intc.1", 19, MCF_CLK); +DEFINE_CLK(0, "iack.0", 21, MCF_CLK); +DEFINE_CLK(0, "mcfi2c.0", 22, MCF_CLK); +DEFINE_CLK(0, "mcfqspi.0", 23, MCF_CLK); +DEFINE_CLK(0, "mcfuart.0", 24, MCF_BUSCLK); +DEFINE_CLK(0, "mcfuart.1", 25, MCF_BUSCLK); +DEFINE_CLK(0, "mcfuart.2", 26, MCF_BUSCLK); +DEFINE_CLK(0, "mcftmr.0", 28, MCF_CLK); +DEFINE_CLK(0, "mcftmr.1", 29, MCF_CLK); +DEFINE_CLK(0, "mcftmr.2", 30, MCF_CLK); +DEFINE_CLK(0, "mcftmr.3", 31, MCF_CLK); + +DEFINE_CLK(0, "mcfpit.0", 32, MCF_CLK); +DEFINE_CLK(0, "mcfpit.1", 33, MCF_CLK); +DEFINE_CLK(0, "mcfpit.2", 34, MCF_CLK); +DEFINE_CLK(0, "mcfpit.3", 35, MCF_CLK); +DEFINE_CLK(0, "mcfpwm.0", 36, MCF_CLK); +DEFINE_CLK(0, "mcfeport.0", 37, MCF_CLK); +DEFINE_CLK(0, "mcfwdt.0", 38, MCF_CLK); +DEFINE_CLK(0, "sys.0", 40, MCF_BUSCLK); +DEFINE_CLK(0, "gpio.0", 41, MCF_BUSCLK); +DEFINE_CLK(0, "mcfrtc.0", 42, MCF_CLK); +DEFINE_CLK(0, "mcflcd.0", 43, MCF_CLK); +DEFINE_CLK(0, "mcfusb-otg.0", 44, MCF_CLK); +DEFINE_CLK(0, "mcfusb-host.0", 45, MCF_CLK); +DEFINE_CLK(0, "sdram.0", 46, MCF_CLK); +DEFINE_CLK(0, "ssi.0", 47, MCF_CLK); +DEFINE_CLK(0, "pll.0", 48, MCF_CLK); + +DEFINE_CLK(1, "mdha.0", 32, MCF_CLK); +DEFINE_CLK(1, "skha.0", 33, MCF_CLK); +DEFINE_CLK(1, "rng.0", 34, MCF_CLK); + +struct clk *mcf_clks[] = { + &__clk_0_2, /* flexbus */ + &__clk_0_8, /* mcfcan.0 */ + &__clk_0_12, /* fec.0 */ + &__clk_0_17, /* edma */ + &__clk_0_18, /* intc.0 */ + &__clk_0_19, /* intc.1 */ + &__clk_0_21, /* iack.0 */ + &__clk_0_22, /* mcfi2c.0 */ + &__clk_0_23, /* mcfqspi.0 */ + &__clk_0_24, /* mcfuart.0 */ + &__clk_0_25, /* mcfuart.1 */ + &__clk_0_26, /* mcfuart.2 */ + &__clk_0_28, /* mcftmr.0 */ + &__clk_0_29, /* mcftmr.1 */ + &__clk_0_30, /* mcftmr.2 */ + &__clk_0_31, /* mcftmr.3 */ + + &__clk_0_32, /* mcfpit.0 */ + &__clk_0_33, /* mcfpit.1 */ + &__clk_0_34, /* mcfpit.2 */ + &__clk_0_35, /* mcfpit.3 */ + &__clk_0_36, /* mcfpwm.0 */ + &__clk_0_37, /* mcfeport.0 */ + &__clk_0_38, /* mcfwdt.0 */ + &__clk_0_40, /* sys.0 */ + &__clk_0_41, /* gpio.0 */ + &__clk_0_42, /* mcfrtc.0 */ + &__clk_0_43, /* mcflcd.0 */ + &__clk_0_44, /* mcfusb-otg.0 */ + &__clk_0_45, /* mcfusb-host.0 */ + &__clk_0_46, /* sdram.0 */ + &__clk_0_47, /* ssi.0 */ + &__clk_0_48, /* pll.0 */ + + &__clk_1_32, /* mdha.0 */ + &__clk_1_33, /* skha.0 */ + &__clk_1_34, /* rng.0 */ + NULL, +}; + +static struct clk * const enable_clks[] __initconst = { + &__clk_0_2, /* flexbus */ + &__clk_0_18, /* intc.0 */ + &__clk_0_19, /* intc.1 */ + &__clk_0_21, /* iack.0 */ + &__clk_0_24, /* mcfuart.0 */ + &__clk_0_25, /* mcfuart.1 */ + &__clk_0_26, /* mcfuart.2 */ + + &__clk_0_32, /* mcfpit.0 */ + &__clk_0_33, /* mcfpit.1 */ + &__clk_0_37, /* mcfeport.0 */ + &__clk_0_40, /* sys.0 */ + &__clk_0_41, /* gpio.0 */ + &__clk_0_46, /* sdram.0 */ + &__clk_0_48, /* pll.0 */ +}; + +static struct clk * const disable_clks[] __initconst = { + &__clk_0_8, /* mcfcan.0 */ + &__clk_0_12, /* fec.0 */ + &__clk_0_17, /* edma */ + &__clk_0_22, /* mcfi2c.0 */ + &__clk_0_23, /* mcfqspi.0 */ + &__clk_0_28, /* mcftmr.0 */ + &__clk_0_29, /* mcftmr.1 */ + &__clk_0_30, /* mcftmr.2 */ + &__clk_0_31, /* mcftmr.3 */ + &__clk_0_34, /* mcfpit.2 */ + &__clk_0_35, /* mcfpit.3 */ + &__clk_0_36, /* mcfpwm.0 */ + &__clk_0_38, /* mcfwdt.0 */ + &__clk_0_42, /* mcfrtc.0 */ + &__clk_0_43, /* mcflcd.0 */ + &__clk_0_44, /* mcfusb-otg.0 */ + &__clk_0_45, /* mcfusb-host.0 */ + &__clk_0_47, /* ssi.0 */ + &__clk_1_32, /* mdha.0 */ + &__clk_1_33, /* skha.0 */ + &__clk_1_34, /* rng.0 */ }; -unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips); + +static void __init m532x_clk_init(void) +{ + unsigned i; + + /* make sure these clocks are enabled */ + for (i = 0; i < ARRAY_SIZE(enable_clks); ++i) + __clk_init_enabled(enable_clks[i]); + /* make sure these clocks are disabled */ + for (i = 0; i < ARRAY_SIZE(disable_clks); ++i) + __clk_init_disabled(disable_clks[i]); +} /***************************************************************************/ @@ -98,8 +210,8 @@ void __init config_BSP(char *commandp, int size) memset(commandp, 0, size); } #endif - mach_sched_init = hw_timer_init; + m532x_clk_init(); m532x_uarts_init(); m532x_fec_init(); #if IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI) diff --git a/arch/m68k/platform/coldfire/m5407.c b/arch/m68k/platform/coldfire/m5407.c index faa6680b3404..bb6c746ae819 100644 --- a/arch/m68k/platform/coldfire/m5407.c +++ b/arch/m68k/platform/coldfire/m5407.c @@ -16,15 +16,6 @@ #include <asm/machdep.h> #include <asm/coldfire.h> #include <asm/mcfsim.h> -#include <asm/mcfgpio.h> - -/***************************************************************************/ - -struct mcf_gpio_chip mcf_gpio_chips[] = { - MCFGPS(PP, 0, 16, MCFSIM_PADDR, MCFSIM_PADAT, MCFSIM_PADAT), -}; - -unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips); /***************************************************************************/ diff --git a/arch/m68k/platform/coldfire/m5441x.c b/arch/m68k/platform/coldfire/m5441x.c new file mode 100644 index 000000000000..98a13cce93d8 --- /dev/null +++ b/arch/m68k/platform/coldfire/m5441x.c @@ -0,0 +1,261 @@ +/* + * m5441x.c -- support for Coldfire m5441x processors + * + * (C) Copyright Steven King <sfking@fdwdc.com> + */ + +#include <linux/kernel.h> +#include <linux/param.h> +#include <linux/init.h> +#include <linux/io.h> +#include <linux/clk.h> +#include <asm/machdep.h> +#include <asm/coldfire.h> +#include <asm/mcfsim.h> +#include <asm/mcfuart.h> +#include <asm/mcfdma.h> +#include <asm/mcfclk.h> + +DEFINE_CLK(0, "flexbus", 2, MCF_CLK); +DEFINE_CLK(0, "mcfcan.0", 8, MCF_CLK); +DEFINE_CLK(0, "mcfcan.1", 9, MCF_CLK); +DEFINE_CLK(0, "mcfi2c.1", 14, MCF_CLK); +DEFINE_CLK(0, "mcfdspi.1", 15, MCF_CLK); +DEFINE_CLK(0, "edma", 17, MCF_CLK); +DEFINE_CLK(0, "intc.0", 18, MCF_CLK); +DEFINE_CLK(0, "intc.1", 19, MCF_CLK); +DEFINE_CLK(0, "intc.2", 20, MCF_CLK); +DEFINE_CLK(0, "mcfi2c.0", 22, MCF_CLK); +DEFINE_CLK(0, "mcfdspi.0", 23, MCF_CLK); +DEFINE_CLK(0, "mcfuart.0", 24, MCF_BUSCLK); +DEFINE_CLK(0, "mcfuart.1", 25, MCF_BUSCLK); +DEFINE_CLK(0, "mcfuart.2", 26, MCF_BUSCLK); +DEFINE_CLK(0, "mcfuart.3", 27, MCF_BUSCLK); +DEFINE_CLK(0, "mcftmr.0", 28, MCF_CLK); +DEFINE_CLK(0, "mcftmr.1", 29, MCF_CLK); +DEFINE_CLK(0, "mcftmr.2", 30, MCF_CLK); +DEFINE_CLK(0, "mcftmr.3", 31, MCF_CLK); +DEFINE_CLK(0, "mcfpit.0", 32, MCF_CLK); +DEFINE_CLK(0, "mcfpit.1", 33, MCF_CLK); +DEFINE_CLK(0, "mcfpit.2", 34, MCF_CLK); +DEFINE_CLK(0, "mcfpit.3", 35, MCF_CLK); +DEFINE_CLK(0, "mcfeport.0", 37, MCF_CLK); +DEFINE_CLK(0, "mcfadc.0", 38, MCF_CLK); +DEFINE_CLK(0, "mcfdac.0", 39, MCF_CLK); +DEFINE_CLK(0, "mcfrtc.0", 42, MCF_CLK); +DEFINE_CLK(0, "mcfsim.0", 43, MCF_CLK); +DEFINE_CLK(0, "mcfusb-otg.0", 44, MCF_CLK); +DEFINE_CLK(0, "mcfusb-host.0", 45, MCF_CLK); +DEFINE_CLK(0, "mcfddr-sram.0", 46, MCF_CLK); +DEFINE_CLK(0, "mcfssi.0", 47, MCF_CLK); +DEFINE_CLK(0, "pll.0", 48, MCF_CLK); +DEFINE_CLK(0, "mcfrng.0", 49, MCF_CLK); +DEFINE_CLK(0, "mcfssi.1", 50, MCF_CLK); +DEFINE_CLK(0, "mcfsdhc.0", 51, MCF_CLK); +DEFINE_CLK(0, "enet-fec.0", 53, MCF_CLK); +DEFINE_CLK(0, "enet-fec.1", 54, MCF_CLK); +DEFINE_CLK(0, "switch.0", 55, MCF_CLK); +DEFINE_CLK(0, "switch.1", 56, MCF_CLK); +DEFINE_CLK(0, "nand.0", 63, MCF_CLK); + +DEFINE_CLK(1, "mcfow.0", 2, MCF_CLK); +DEFINE_CLK(1, "mcfi2c.2", 4, MCF_CLK); +DEFINE_CLK(1, "mcfi2c.3", 5, MCF_CLK); +DEFINE_CLK(1, "mcfi2c.4", 6, MCF_CLK); +DEFINE_CLK(1, "mcfi2c.5", 7, MCF_CLK); +DEFINE_CLK(1, "mcfuart.4", 24, MCF_BUSCLK); +DEFINE_CLK(1, "mcfuart.5", 25, MCF_BUSCLK); +DEFINE_CLK(1, "mcfuart.6", 26, MCF_BUSCLK); +DEFINE_CLK(1, "mcfuart.7", 27, MCF_BUSCLK); +DEFINE_CLK(1, "mcfuart.8", 28, MCF_BUSCLK); +DEFINE_CLK(1, "mcfuart.9", 29, MCF_BUSCLK); +DEFINE_CLK(1, "mcfpwm.0", 34, MCF_BUSCLK); +DEFINE_CLK(1, "sys.0", 36, MCF_BUSCLK); +DEFINE_CLK(1, "gpio.0", 37, MCF_BUSCLK); + +struct clk *mcf_clks[] = { + &__clk_0_2, + &__clk_0_8, + &__clk_0_9, + &__clk_0_14, + &__clk_0_15, + &__clk_0_17, + &__clk_0_18, + &__clk_0_19, + &__clk_0_20, + &__clk_0_22, + &__clk_0_23, + &__clk_0_24, + &__clk_0_25, + &__clk_0_26, + &__clk_0_27, + &__clk_0_28, + &__clk_0_29, + &__clk_0_30, + &__clk_0_31, + &__clk_0_32, + &__clk_0_33, + &__clk_0_34, + &__clk_0_35, + &__clk_0_37, + &__clk_0_38, + &__clk_0_39, + &__clk_0_42, + &__clk_0_43, + &__clk_0_44, + &__clk_0_45, + &__clk_0_46, + &__clk_0_47, + &__clk_0_48, + &__clk_0_49, + &__clk_0_50, + &__clk_0_51, + &__clk_0_53, + &__clk_0_54, + &__clk_0_55, + &__clk_0_56, + &__clk_0_63, + + &__clk_1_2, + &__clk_1_4, + &__clk_1_5, + &__clk_1_6, + &__clk_1_7, + &__clk_1_24, + &__clk_1_25, + &__clk_1_26, + &__clk_1_27, + &__clk_1_28, + &__clk_1_29, + &__clk_1_34, + &__clk_1_36, + &__clk_1_37, + NULL, +}; + + +static struct clk * const enable_clks[] __initconst = { + /* make sure these clocks are enabled */ + &__clk_0_18, /* intc0 */ + &__clk_0_19, /* intc0 */ + &__clk_0_20, /* intc0 */ + &__clk_0_24, /* uart0 */ + &__clk_0_25, /* uart1 */ + &__clk_0_26, /* uart2 */ + &__clk_0_27, /* uart3 */ + + &__clk_0_33, /* pit.1 */ + &__clk_0_37, /* eport */ + &__clk_0_48, /* pll */ + + &__clk_1_36, /* CCM/reset module/Power management */ + &__clk_1_37, /* gpio */ +}; +static struct clk * const disable_clks[] __initconst = { + &__clk_0_8, /* can.0 */ + &__clk_0_9, /* can.1 */ + &__clk_0_14, /* i2c.1 */ + &__clk_0_15, /* dspi.1 */ + &__clk_0_17, /* eDMA */ + &__clk_0_22, /* i2c.0 */ + &__clk_0_23, /* dspi.0 */ + &__clk_0_28, /* tmr.1 */ + &__clk_0_29, /* tmr.2 */ + &__clk_0_30, /* tmr.2 */ + &__clk_0_31, /* tmr.3 */ + &__clk_0_32, /* pit.0 */ + &__clk_0_34, /* pit.2 */ + &__clk_0_35, /* pit.3 */ + &__clk_0_38, /* adc */ + &__clk_0_39, /* dac */ + &__clk_0_44, /* usb otg */ + &__clk_0_45, /* usb host */ + &__clk_0_47, /* ssi.0 */ + &__clk_0_49, /* rng */ + &__clk_0_50, /* ssi.1 */ + &__clk_0_51, /* eSDHC */ + &__clk_0_53, /* enet-fec */ + &__clk_0_54, /* enet-fec */ + &__clk_0_55, /* switch.0 */ + &__clk_0_56, /* switch.1 */ + + &__clk_1_2, /* 1-wire */ + &__clk_1_4, /* i2c.2 */ + &__clk_1_5, /* i2c.3 */ + &__clk_1_6, /* i2c.4 */ + &__clk_1_7, /* i2c.5 */ + &__clk_1_24, /* uart 4 */ + &__clk_1_25, /* uart 5 */ + &__clk_1_26, /* uart 6 */ + &__clk_1_27, /* uart 7 */ + &__clk_1_28, /* uart 8 */ + &__clk_1_29, /* uart 9 */ +}; + +static void __init m5441x_clk_init(void) +{ + unsigned i; + + for (i = 0; i < ARRAY_SIZE(enable_clks); ++i) + __clk_init_enabled(enable_clks[i]); + /* make sure these clocks are disabled */ + for (i = 0; i < ARRAY_SIZE(disable_clks); ++i) + __clk_init_disabled(disable_clks[i]); +} + +static void __init m5441x_uarts_init(void) +{ + __raw_writeb(0x0f, MCFGPIO_PAR_UART0); + __raw_writeb(0x00, MCFGPIO_PAR_UART1); + __raw_writeb(0x00, MCFGPIO_PAR_UART2); +} + +static void __init m5441x_fec_init(void) +{ + __raw_writeb(0x03, MCFGPIO_PAR_FEC); +} + +void __init config_BSP(char *commandp, int size) +{ + m5441x_clk_init(); + mach_sched_init = hw_timer_init; + m5441x_uarts_init(); + m5441x_fec_init(); +} + + +#if IS_ENABLED(CONFIG_RTC_DRV_M5441x) +static struct resource m5441x_rtc_resources[] = { + { + .start = MCFRTC_BASE, + .end = MCFRTC_BASE + MCFRTC_SIZE - 1, + .flags = IORESOURCE_MEM, + }, + { + .start = MCF_IRQ_RTC, + .end = MCF_IRQ_RTC, + .flags = IORESOURCE_IRQ, + }, +}; + +static struct platform_device m5441x_rtc = { + .name = "mcfrtc", + .id = 0, + .resource = m5441x_rtc_resources, + .num_resources = ARRAY_SIZE(m5441x_rtc_resources), +}; +#endif + +static struct platform_device *m5441x_devices[] __initdata = { +#if IS_ENABLED(CONFIG_RTC_DRV_M5441x) + &m5441x_rtc, +#endif +}; + +static int __init init_BSP(void) +{ + platform_add_devices(m5441x_devices, ARRAY_SIZE(m5441x_devices)); + return 0; +} + +arch_initcall(init_BSP); diff --git a/arch/m68k/platform/coldfire/m54xx.c b/arch/m68k/platform/coldfire/m54xx.c index 20672dadb252..2081c6cbb3de 100644 --- a/arch/m68k/platform/coldfire/m54xx.c +++ b/arch/m68k/platform/coldfire/m54xx.c @@ -21,19 +21,12 @@ #include <asm/m54xxsim.h> #include <asm/mcfuart.h> #include <asm/m54xxgpt.h> -#include <asm/mcfgpio.h> #ifdef CONFIG_MMU #include <asm/mmu_context.h> #endif /***************************************************************************/ -struct mcf_gpio_chip mcf_gpio_chips[] = { }; - -unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips); - -/***************************************************************************/ - static void __init m54xx_uarts_init(void) { /* enable io pins */ diff --git a/arch/m68k/platform/coldfire/mcf8390.c b/arch/m68k/platform/coldfire/mcf8390.c new file mode 100644 index 000000000000..23a6874a3248 --- /dev/null +++ b/arch/m68k/platform/coldfire/mcf8390.c @@ -0,0 +1,38 @@ +/* + * mcf8390.c -- platform support for 8390 ethernet on many boards + * + * (C) Copyright 2012, Greg Ungerer <gerg@uclinux.org> + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file COPYING in the main directory of this archive + * for more details. + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/resource.h> +#include <linux/platform_device.h> +#include <asm/mcf8390.h> + +static struct resource mcf8390_resources[] = { + { + .start = NE2000_ADDR, + .end = NE2000_ADDR + NE2000_ADDRSIZE - 1, + .flags = IORESOURCE_MEM, + }, + { + .start = NE2000_IRQ_VECTOR, + .end = NE2000_IRQ_VECTOR, + .flags = IORESOURCE_IRQ, + }, +}; + +static int __init mcf8390_platform_init(void) +{ + platform_device_register_simple("mcf8390", -1, mcf8390_resources, + ARRAY_SIZE(mcf8390_resources)); + return 0; +} + +arch_initcall(mcf8390_platform_init); diff --git a/arch/m68k/platform/coldfire/pci.c b/arch/m68k/platform/coldfire/pci.c new file mode 100644 index 000000000000..553210d3d4c1 --- /dev/null +++ b/arch/m68k/platform/coldfire/pci.c @@ -0,0 +1,327 @@ +/* + * pci.c -- PCI bus support for ColdFire processors + * + * (C) Copyright 2012, Greg Ungerer <gerg@uclinux.com> + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file COPYING in the main directory of this archive + * for more details. + */ + +#include <linux/types.h> +#include <linux/module.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/io.h> +#include <linux/pci.h> +#include <linux/delay.h> +#include <asm/coldfire.h> +#include <asm/mcfsim.h> +#include <asm/m54xxpci.h> + +/* + * Memory and IO mappings. We use a 1:1 mapping for local host memory to + * PCI bus memory (no reason not to really). IO space doesn't matter, we + * always use access functions for that. The device configuration space is + * mapped over the IO map space when we enable it in the PCICAR register. + */ +#define PCI_MEM_PA 0xf0000000 /* Host physical address */ +#define PCI_MEM_BA 0xf0000000 /* Bus physical address */ +#define PCI_MEM_SIZE 0x08000000 /* 128 MB */ +#define PCI_MEM_MASK (PCI_MEM_SIZE - 1) + +#define PCI_IO_PA 0xf8000000 /* Host physical address */ +#define PCI_IO_BA 0x00000000 /* Bus physical address */ +#define PCI_IO_SIZE 0x00010000 /* 64k */ +#define PCI_IO_MASK (PCI_IO_SIZE - 1) + +static struct pci_bus *rootbus; +static unsigned long iospace; + +/* + * We need to be carefull probing on bus 0 (directly connected to host + * bridge). We should only acccess the well defined possible devices in + * use, ignore aliases and the like. + */ +static unsigned char mcf_host_slot2sid[32] = { + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 2, 0, 3, 4, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, +}; + +static unsigned char mcf_host_irq[] = { + 0, 69, 69, 71, 71, +}; + + +static inline void syncio(void) +{ + /* The ColdFire "nop" instruction waits for all bus IO to complete */ + __asm__ __volatile__ ("nop"); +} + +/* + * Configuration space access functions. Configuration space access is + * through the IO mapping window, enabling it via the PCICAR register. + */ +static unsigned long mcf_mk_pcicar(int bus, unsigned int devfn, int where) +{ + return (bus << PCICAR_BUSN) | (devfn << PCICAR_DEVFNN) | (where & 0xfc); +} + +static int mcf_pci_readconfig(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 *value) +{ + unsigned long addr; + + *value = 0xffffffff; + + if (bus->number == 0) { + if (mcf_host_slot2sid[PCI_SLOT(devfn)] == 0) + return PCIBIOS_SUCCESSFUL; + } + + syncio(); + addr = mcf_mk_pcicar(bus->number, devfn, where); + __raw_writel(PCICAR_E | addr, PCICAR); + addr = iospace + (where & 0x3); + + switch (size) { + case 1: + *value = __raw_readb(addr); + break; + case 2: + *value = le16_to_cpu(__raw_readw(addr)); + break; + default: + *value = le32_to_cpu(__raw_readl(addr)); + break; + } + + syncio(); + __raw_writel(0, PCICAR); + return PCIBIOS_SUCCESSFUL; +} + +static int mcf_pci_writeconfig(struct pci_bus *bus, unsigned int devfn, + int where, int size, u32 value) +{ + unsigned long addr; + + if (bus->number == 0) { + if (mcf_host_slot2sid[PCI_SLOT(devfn)] == 0) + return PCIBIOS_SUCCESSFUL; + } + + syncio(); + addr = mcf_mk_pcicar(bus->number, devfn, where); + __raw_writel(PCICAR_E | addr, PCICAR); + addr = iospace + (where & 0x3); + + switch (size) { + case 1: + __raw_writeb(value, addr); + break; + case 2: + __raw_writew(cpu_to_le16(value), addr); + break; + default: + __raw_writel(cpu_to_le32(value), addr); + break; + } + + syncio(); + __raw_writel(0, PCICAR); + return PCIBIOS_SUCCESSFUL; +} + +static struct pci_ops mcf_pci_ops = { + .read = mcf_pci_readconfig, + .write = mcf_pci_writeconfig, +}; + +/* + * IO address space access functions. Pretty strait forward, these are + * directly mapped in to the IO mapping window. And that is mapped into + * virtual address space. + */ +u8 mcf_pci_inb(u32 addr) +{ + return __raw_readb(iospace + (addr & PCI_IO_MASK)); +} +EXPORT_SYMBOL(mcf_pci_inb); + +u16 mcf_pci_inw(u32 addr) +{ + return le16_to_cpu(__raw_readw(iospace + (addr & PCI_IO_MASK))); +} +EXPORT_SYMBOL(mcf_pci_inw); + +u32 mcf_pci_inl(u32 addr) +{ + return le32_to_cpu(__raw_readl(iospace + (addr & PCI_IO_MASK))); +} +EXPORT_SYMBOL(mcf_pci_inl); + +void mcf_pci_insb(u32 addr, u8 *buf, u32 len) +{ + for (; len; len--) + *buf++ = mcf_pci_inb(addr); +} +EXPORT_SYMBOL(mcf_pci_insb); + +void mcf_pci_insw(u32 addr, u16 *buf, u32 len) +{ + for (; len; len--) + *buf++ = mcf_pci_inw(addr); +} +EXPORT_SYMBOL(mcf_pci_insw); + +void mcf_pci_insl(u32 addr, u32 *buf, u32 len) +{ + for (; len; len--) + *buf++ = mcf_pci_inl(addr); +} +EXPORT_SYMBOL(mcf_pci_insl); + +void mcf_pci_outb(u8 v, u32 addr) +{ + __raw_writeb(v, iospace + (addr & PCI_IO_MASK)); +} +EXPORT_SYMBOL(mcf_pci_outb); + +void mcf_pci_outw(u16 v, u32 addr) +{ + __raw_writew(cpu_to_le16(v), iospace + (addr & PCI_IO_MASK)); +} +EXPORT_SYMBOL(mcf_pci_outw); + +void mcf_pci_outl(u32 v, u32 addr) +{ + __raw_writel(cpu_to_le32(v), iospace + (addr & PCI_IO_MASK)); +} +EXPORT_SYMBOL(mcf_pci_outl); + +void mcf_pci_outsb(u32 addr, const u8 *buf, u32 len) +{ + for (; len; len--) + mcf_pci_outb(*buf++, addr); +} +EXPORT_SYMBOL(mcf_pci_outsb); + +void mcf_pci_outsw(u32 addr, const u16 *buf, u32 len) +{ + for (; len; len--) + mcf_pci_outw(*buf++, addr); +} +EXPORT_SYMBOL(mcf_pci_outsw); + +void mcf_pci_outsl(u32 addr, const u32 *buf, u32 len) +{ + for (; len; len--) + mcf_pci_outl(*buf++, addr); +} +EXPORT_SYMBOL(mcf_pci_outsl); + +/* + * Initialize the PCI bus registers, and scan the bus. + */ +static struct resource mcf_pci_mem = { + .name = "PCI Memory space", + .start = PCI_MEM_PA, + .end = PCI_MEM_PA + PCI_MEM_SIZE - 1, + .flags = IORESOURCE_MEM, +}; + +static struct resource mcf_pci_io = { + .name = "PCI IO space", + .start = 0x400, + .end = 0x10000 - 1, + .flags = IORESOURCE_IO, +}; + +/* + * Interrupt mapping and setting. + */ +static int mcf_pci_map_irq(const struct pci_dev *dev, u8 slot, u8 pin) +{ + int sid; + + sid = mcf_host_slot2sid[slot]; + if (sid) + return mcf_host_irq[sid]; + return 0; +} + +static int __init mcf_pci_init(void) +{ + pr_info("ColdFire: PCI bus initialization...\n"); + + /* Reset the external PCI bus */ + __raw_writel(PCIGSCR_RESET, PCIGSCR); + __raw_writel(0, PCITCR); + + request_resource(&iomem_resource, &mcf_pci_mem); + request_resource(&iomem_resource, &mcf_pci_io); + + /* Configure PCI arbiter */ + __raw_writel(PACR_INTMPRI | PACR_INTMINTE | PACR_EXTMPRI(0x1f) | + PACR_EXTMINTE(0x1f), PACR); + + /* Set required multi-function pins for PCI bus use */ + __raw_writew(0x3ff, MCF_PAR_PCIBG); + __raw_writew(0x3ff, MCF_PAR_PCIBR); + + /* Set up config space for local host bus controller */ + __raw_writel(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER | + PCI_COMMAND_INVALIDATE, PCISCR); + __raw_writel(PCICR1_LT(32) | PCICR1_CL(8), PCICR1); + __raw_writel(0, PCICR2); + + /* + * Set up the initiator windows for memory and IO mapping. + * These give the CPU bus access onto the PCI bus. One for each of + * PCI memory and IO address spaces. + */ + __raw_writel(WXBTAR(PCI_MEM_PA, PCI_MEM_BA, PCI_MEM_SIZE), + PCIIW0BTAR); + __raw_writel(WXBTAR(PCI_IO_PA, PCI_IO_BA, PCI_IO_SIZE), + PCIIW1BTAR); + __raw_writel(PCIIWCR_W0_MEM /*| PCIIWCR_W0_MRDL*/ | PCIIWCR_W0_E | + PCIIWCR_W1_IO | PCIIWCR_W1_E, PCIIWCR); + + /* + * Set up the target windows for access from the PCI bus back to the + * CPU bus. All we need is access to system RAM (for mastering). + */ + __raw_writel(CONFIG_RAMBASE, PCIBAR1); + __raw_writel(CONFIG_RAMBASE | PCITBATR1_E, PCITBATR1); + + /* Keep a virtual mapping to IO/config space active */ + iospace = (unsigned long) ioremap(PCI_IO_PA, PCI_IO_SIZE); + if (iospace == 0) + return -ENODEV; + pr_info("Coldfire: PCI IO/config window mapped to 0x%x\n", + (u32) iospace); + + /* Turn of PCI reset, and wait for devices to settle */ + __raw_writel(0, PCIGSCR); + set_current_state(TASK_UNINTERRUPTIBLE); + schedule_timeout(msecs_to_jiffies(200)); + + rootbus = pci_scan_bus(0, &mcf_pci_ops, NULL); + rootbus->resource[0] = &mcf_pci_io; + rootbus->resource[1] = &mcf_pci_mem; + + pci_fixup_irqs(pci_common_swizzle, mcf_pci_map_irq); + pci_bus_size_bridges(rootbus); + pci_bus_assign_resources(rootbus); + pci_enable_bridges(rootbus); + pci_bus_add_devices(rootbus); + return 0; +} + +subsys_initcall(mcf_pci_init); diff --git a/arch/m68k/platform/coldfire/pinmux.c b/arch/m68k/platform/coldfire/pinmux.c deleted file mode 100644 index 8c62b825939f..000000000000 --- a/arch/m68k/platform/coldfire/pinmux.c +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Coldfire generic GPIO pinmux support. - * - * (C) Copyright 2009, Steven King <sfking@fdwdc.com> - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; version 2 of the License. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - */ - -#include <linux/kernel.h> - -#include <asm/pinmux.h> - -int mcf_pinmux_request(unsigned pinmux, unsigned func) -{ - return 0; -} - -void mcf_pinmux_release(unsigned pinmux, unsigned func) -{ -} diff --git a/arch/m68k/platform/coldfire/pit.c b/arch/m68k/platform/coldfire/pit.c index e62dbbcb10f6..e8f3b97b0f77 100644 --- a/arch/m68k/platform/coldfire/pit.c +++ b/arch/m68k/platform/coldfire/pit.c @@ -93,7 +93,7 @@ struct clock_event_device cf_pit_clockevent = { .set_mode = init_cf_pit_timer, .set_next_event = cf_pit_next_event, .shift = 32, - .irq = MCFINT_VECBASE + MCFINT_PIT1, + .irq = MCF_IRQ_PIT1, }; @@ -159,7 +159,7 @@ void hw_timer_init(irq_handler_t handler) clockevent_delta2ns(0x3f, &cf_pit_clockevent); clockevents_register_device(&cf_pit_clockevent); - setup_irq(MCFINT_VECBASE + MCFINT_PIT1, &pit_irq); + setup_irq(MCF_IRQ_PIT1, &pit_irq); clocksource_register_hz(&pit_clk, FREQ); } diff --git a/arch/m68k/platform/coldfire/timers.c b/arch/m68k/platform/coldfire/timers.c index ed96ce50d79f..0a273e75408c 100644 --- a/arch/m68k/platform/coldfire/timers.c +++ b/arch/m68k/platform/coldfire/timers.c @@ -36,7 +36,7 @@ */ void coldfire_profile_init(void); -#if defined(CONFIG_M532x) +#if defined(CONFIG_M532x) || defined(CONFIG_M5441x) #define __raw_readtrr __raw_readl #define __raw_writetrr __raw_writel #else diff --git a/arch/mips/kernel/kspd.c b/arch/mips/kernel/kspd.c index 84d0639e4580..b77f56bbb477 100644 --- a/arch/mips/kernel/kspd.c +++ b/arch/mips/kernel/kspd.c @@ -323,7 +323,7 @@ static void sp_cleanup(void) fdt = files_fdtable(files); for (;;) { unsigned long set; - i = j * __NFDBITS; + i = j * BITS_PER_LONG; if (i >= fdt->max_fds) break; set = fdt->open_fds[j++]; diff --git a/arch/powerpc/Makefile b/arch/powerpc/Makefile index 950d1f7a5a39..159e94f4b22a 100644 --- a/arch/powerpc/Makefile +++ b/arch/powerpc/Makefile @@ -149,7 +149,6 @@ core-$(CONFIG_KVM) += arch/powerpc/kvm/ core-$(CONFIG_PERF_EVENTS) += arch/powerpc/perf/ drivers-$(CONFIG_OPROFILE) += arch/powerpc/oprofile/ -drivers-$(CONFIG_CRYPTO_DEV_NX) += drivers/crypto/nx/ # Default to zImage, override when needed all: zImage diff --git a/arch/s390/crypto/crypto_des.h b/arch/s390/crypto/crypto_des.h deleted file mode 100644 index 6210457ceebb..000000000000 --- a/arch/s390/crypto/crypto_des.h +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Cryptographic API. - * - * Function for checking keys for the DES and Tripple DES Encryption - * algorithms. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - */ -#ifndef __CRYPTO_DES_H__ -#define __CRYPTO_DES_H__ - -extern int crypto_des_check_key(const u8*, unsigned int, u32*); - -#endif /*__CRYPTO_DES_H__*/ diff --git a/arch/sh/include/asm/siu.h b/arch/sh/include/asm/siu.h index 1d95c78808d1..580b7ac228b7 100644 --- a/arch/sh/include/asm/siu.h +++ b/arch/sh/include/asm/siu.h @@ -14,7 +14,6 @@ struct device; struct siu_platform { - struct device *dma_dev; unsigned int dma_slave_tx_a; unsigned int dma_slave_rx_a; unsigned int dma_slave_tx_b; diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7722.c b/arch/sh/kernel/cpu/sh4a/setup-sh7722.c index 0f5a21907da6..65786c7f5ded 100644 --- a/arch/sh/kernel/cpu/sh4a/setup-sh7722.c +++ b/arch/sh/kernel/cpu/sh4a/setup-sh7722.c @@ -512,7 +512,6 @@ static struct platform_device tmu2_device = { }; static struct siu_platform siu_platform_data = { - .dma_dev = &dma_device.dev, .dma_slave_tx_a = SHDMA_SLAVE_SIUA_TX, .dma_slave_rx_a = SHDMA_SLAVE_SIUA_RX, .dma_slave_tx_b = SHDMA_SLAVE_SIUB_TX, diff --git a/arch/sparc/include/asm/fixmap.h b/arch/sparc/include/asm/fixmap.h deleted file mode 100644 index f18fc0755adf..000000000000 --- a/arch/sparc/include/asm/fixmap.h +++ /dev/null @@ -1,110 +0,0 @@ -/* - * fixmap.h: compile-time virtual memory allocation - * - * This file is subject to the terms and conditions of the GNU General Public - * License. See the file "COPYING" in the main directory of this archive - * for more details. - * - * Copyright (C) 1998 Ingo Molnar - * - * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999 - */ - -#ifndef _ASM_FIXMAP_H -#define _ASM_FIXMAP_H - -#include <linux/kernel.h> -#include <asm/page.h> -#ifdef CONFIG_HIGHMEM -#include <linux/threads.h> -#include <asm/kmap_types.h> -#endif - -/* - * Here we define all the compile-time 'special' virtual - * addresses. The point is to have a constant address at - * compile time, but to set the physical address only - * in the boot process. We allocate these special addresses - * from the top of unused virtual memory (0xfd000000 - 1 page) backwards. - * Also this lets us do fail-safe vmalloc(), we - * can guarantee that these special addresses and - * vmalloc()-ed addresses never overlap. - * - * these 'compile-time allocated' memory buffers are - * fixed-size 4k pages. (or larger if used with an increment - * highger than 1) use fixmap_set(idx,phys) to associate - * physical memory with fixmap indices. - * - * TLB entries of such buffers will not be flushed across - * task switches. - */ - -/* - * on UP currently we will have no trace of the fixmap mechanism, - * no page table allocations, etc. This might change in the - * future, say framebuffers for the console driver(s) could be - * fix-mapped? - */ -enum fixed_addresses { - FIX_HOLE, -#ifdef CONFIG_HIGHMEM - FIX_KMAP_BEGIN, - FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*NR_CPUS)-1, -#endif - __end_of_fixed_addresses -}; - -extern void __set_fixmap (enum fixed_addresses idx, - unsigned long phys, pgprot_t flags); - -#define set_fixmap(idx, phys) \ - __set_fixmap(idx, phys, PAGE_KERNEL) -/* - * Some hardware wants to get fixmapped without caching. - */ -#define set_fixmap_nocache(idx, phys) \ - __set_fixmap(idx, phys, PAGE_KERNEL_NOCACHE) -/* - * used by vmalloc.c. - * - * Leave one empty page between IO pages at 0xfd000000 and - * the start of the fixmap. - */ -#define FIXADDR_TOP (0xfcfff000UL) -#define FIXADDR_SIZE ((__end_of_fixed_addresses) << PAGE_SHIFT) -#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE) - -#define __fix_to_virt(x) (FIXADDR_TOP - ((x) << PAGE_SHIFT)) -#define __virt_to_fix(x) ((FIXADDR_TOP - ((x)&PAGE_MASK)) >> PAGE_SHIFT) - -extern void __this_fixmap_does_not_exist(void); - -/* - * 'index to address' translation. If anyone tries to use the idx - * directly without tranlation, we catch the bug with a NULL-deference - * kernel oops. Illegal ranges of incoming indices are caught too. - */ -static inline unsigned long fix_to_virt(const unsigned int idx) -{ - /* - * this branch gets completely eliminated after inlining, - * except when someone tries to use fixaddr indices in an - * illegal way. (such as mixing up address types or using - * out-of-range indices). - * - * If it doesn't get removed, the linker will complain - * loudly with a reasonably clear error message.. - */ - if (idx >= __end_of_fixed_addresses) - __this_fixmap_does_not_exist(); - - return __fix_to_virt(idx); -} - -static inline unsigned long virt_to_fix(const unsigned long vaddr) -{ - BUG_ON(vaddr >= FIXADDR_TOP || vaddr < FIXADDR_START); - return __virt_to_fix(vaddr); -} - -#endif diff --git a/arch/sparc/include/asm/highmem.h b/arch/sparc/include/asm/highmem.h index 3b6e00dd96e5..4f9e15c757e2 100644 --- a/arch/sparc/include/asm/highmem.h +++ b/arch/sparc/include/asm/highmem.h @@ -21,7 +21,6 @@ #ifdef __KERNEL__ #include <linux/interrupt.h> -#include <asm/fixmap.h> #include <asm/vaddrs.h> #include <asm/kmap_types.h> #include <asm/pgtable.h> @@ -29,7 +28,6 @@ /* declarations for highmem.c */ extern unsigned long highstart_pfn, highend_pfn; -extern pte_t *kmap_pte; extern pgprot_t kmap_prot; extern pte_t *pkmap_page_table; @@ -72,7 +70,6 @@ static inline void kunmap(struct page *page) extern void *kmap_atomic(struct page *page); extern void __kunmap_atomic(void *kvaddr); -extern struct page *kmap_atomic_to_page(void *vaddr); #define flush_cache_kmaps() flush_cache_all() diff --git a/arch/sparc/include/asm/leon.h b/arch/sparc/include/asm/leon.h index 3375c6293893..15a716934e4d 100644 --- a/arch/sparc/include/asm/leon.h +++ b/arch/sparc/include/asm/leon.h @@ -82,7 +82,6 @@ static inline unsigned long leon_load_reg(unsigned long paddr) #define LEON_BYPASS_LOAD_PA(x) leon_load_reg((unsigned long)(x)) #define LEON_BYPASS_STORE_PA(x, v) leon_store_reg((unsigned long)(x), (unsigned long)(v)) -extern void leon_init(void); extern void leon_switch_mm(void); extern void leon_init_IRQ(void); diff --git a/arch/sparc/include/asm/mmu_context_32.h b/arch/sparc/include/asm/mmu_context_32.h index 01456c900720..2df2a9be8f6d 100644 --- a/arch/sparc/include/asm/mmu_context_32.h +++ b/arch/sparc/include/asm/mmu_context_32.h @@ -9,14 +9,12 @@ static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk) { } -/* - * Initialize a new mmu context. This is invoked when a new +/* Initialize a new mmu context. This is invoked when a new * address space instance (unique or shared) is instantiated. */ -#define init_new_context(tsk, mm) (((mm)->context = NO_CONTEXT), 0) +int init_new_context(struct task_struct *tsk, struct mm_struct *mm); -/* - * Destroy a dead context. This occurs when mmput drops the +/* Destroy a dead context. This occurs when mmput drops the * mm_users count to zero, the mmaps have been released, and * all the page tables have been flushed. Our job is to destroy * any remaining processor-specific state. diff --git a/arch/sparc/include/asm/page_32.h b/arch/sparc/include/asm/page_32.h index fab78a308ebf..f82a1f36b655 100644 --- a/arch/sparc/include/asm/page_32.h +++ b/arch/sparc/include/asm/page_32.h @@ -107,8 +107,7 @@ typedef unsigned long iopgprot_t; typedef struct page *pgtable_t; -extern unsigned long sparc_unmapped_base; -#define TASK_UNMAPPED_BASE sparc_unmapped_base +#define TASK_UNMAPPED_BASE 0x50000000 #else /* !(__ASSEMBLY__) */ diff --git a/arch/sparc/include/asm/pgalloc_32.h b/arch/sparc/include/asm/pgalloc_32.h index e5b169b46d21..9b1c36de0f18 100644 --- a/arch/sparc/include/asm/pgalloc_32.h +++ b/arch/sparc/include/asm/pgalloc_32.h @@ -11,28 +11,15 @@ struct page; -extern struct pgtable_cache_struct { - unsigned long *pgd_cache; - unsigned long *pte_cache; - unsigned long pgtable_cache_sz; - unsigned long pgd_cache_sz; -} pgt_quicklists; - -unsigned long srmmu_get_nocache(int size, int align); -void srmmu_free_nocache(unsigned long vaddr, int size); - -#define pgd_quicklist (pgt_quicklists.pgd_cache) -#define pmd_quicklist ((unsigned long *)0) -#define pte_quicklist (pgt_quicklists.pte_cache) -#define pgtable_cache_size (pgt_quicklists.pgtable_cache_sz) -#define pgd_cache_size (pgt_quicklists.pgd_cache_sz) +void *srmmu_get_nocache(int size, int align); +void srmmu_free_nocache(void *addr, int size); #define check_pgt_cache() do { } while (0) pgd_t *get_pgd_fast(void); static inline void free_pgd_fast(pgd_t *pgd) { - srmmu_free_nocache((unsigned long)pgd, SRMMU_PGD_TABLE_SIZE); + srmmu_free_nocache(pgd, SRMMU_PGD_TABLE_SIZE); } #define pgd_free(mm, pgd) free_pgd_fast(pgd) @@ -50,13 +37,13 @@ static inline void pgd_set(pgd_t * pgdp, pmd_t * pmdp) static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long address) { - return (pmd_t *)srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, - SRMMU_PMD_TABLE_SIZE); + return srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, + SRMMU_PMD_TABLE_SIZE); } static inline void free_pmd_fast(pmd_t * pmd) { - srmmu_free_nocache((unsigned long)pmd, SRMMU_PMD_TABLE_SIZE); + srmmu_free_nocache(pmd, SRMMU_PMD_TABLE_SIZE); } #define pmd_free(mm, pmd) free_pmd_fast(pmd) @@ -73,13 +60,13 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address); static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { - return (pte_t *)srmmu_get_nocache(PTE_SIZE, PTE_SIZE); + return srmmu_get_nocache(PTE_SIZE, PTE_SIZE); } static inline void free_pte_fast(pte_t *pte) { - srmmu_free_nocache((unsigned long)pte, PTE_SIZE); + srmmu_free_nocache(pte, PTE_SIZE); } #define pte_free_kernel(mm, pte) free_pte_fast(pte) diff --git a/arch/sparc/include/asm/pgtable_32.h b/arch/sparc/include/asm/pgtable_32.h index cbbbed5cb3aa..6fc13483f702 100644 --- a/arch/sparc/include/asm/pgtable_32.h +++ b/arch/sparc/include/asm/pgtable_32.h @@ -52,8 +52,9 @@ extern unsigned long calc_highpages(void); #define PAGE_READONLY SRMMU_PAGE_RDONLY #define PAGE_KERNEL SRMMU_PAGE_KERNEL -/* Top-level page directory */ -extern pgd_t swapper_pg_dir[1024]; +/* Top-level page directory - dummy used by init-mm. + * srmmu.c will assign the real one (which is dynamically sized) */ +#define swapper_pg_dir NULL extern void paging_init(void); @@ -78,8 +79,6 @@ extern unsigned long ptr_in_current_pgd; #define __S110 PAGE_SHARED #define __S111 PAGE_SHARED -extern int num_contexts; - /* First physical page can be anywhere, the following is needed so that * va-->pa and vice versa conversions work properly without performance * hit for all __pa()/__va() operations. @@ -88,18 +87,11 @@ extern unsigned long phys_base; extern unsigned long pfn_base; /* - * BAD_PAGETABLE is used when we need a bogus page-table, while - * BAD_PAGE is used for a bogus page. - * * ZERO_PAGE is a global shared page that is always zero: used * for zero-mapped memory areas etc.. */ -extern pte_t * __bad_pagetable(void); -extern pte_t __bad_page(void); extern unsigned long empty_zero_page; -#define BAD_PAGETABLE __bad_pagetable() -#define BAD_PAGE __bad_page() #define ZERO_PAGE(vaddr) (virt_to_page(&empty_zero_page)) /* @@ -398,36 +390,6 @@ static inline pte_t pgoff_to_pte(unsigned long pgoff) */ #define PTE_FILE_MAX_BITS 24 -/* - */ -struct ctx_list { - struct ctx_list *next; - struct ctx_list *prev; - unsigned int ctx_number; - struct mm_struct *ctx_mm; -}; - -extern struct ctx_list *ctx_list_pool; /* Dynamically allocated */ -extern struct ctx_list ctx_free; /* Head of free list */ -extern struct ctx_list ctx_used; /* Head of used contexts list */ - -#define NO_CONTEXT -1 - -static inline void remove_from_ctx_list(struct ctx_list *entry) -{ - entry->next->prev = entry->prev; - entry->prev->next = entry->next; -} - -static inline void add_to_ctx_list(struct ctx_list *head, struct ctx_list *entry) -{ - entry->next = head; - (entry->prev = head->prev)->next = entry; - head->prev = entry; -} -#define add_to_free_ctxlist(entry) add_to_ctx_list(&ctx_free, entry) -#define add_to_used_ctxlist(entry) add_to_ctx_list(&ctx_used, entry) - static inline unsigned long __get_phys (unsigned long addr) { diff --git a/arch/sparc/include/asm/vaddrs.h b/arch/sparc/include/asm/vaddrs.h index da6535d88a72..c3dbcf902034 100644 --- a/arch/sparc/include/asm/vaddrs.h +++ b/arch/sparc/include/asm/vaddrs.h @@ -30,6 +30,28 @@ */ #define SRMMU_NOCACHE_ALCRATIO 64 /* 256 pages per 64MB of system RAM */ +#ifndef __ASSEMBLY__ +#include <asm/kmap_types.h> + +enum fixed_addresses { + FIX_HOLE, +#ifdef CONFIG_HIGHMEM + FIX_KMAP_BEGIN, + FIX_KMAP_END = (KM_TYPE_NR * NR_CPUS), +#endif + __end_of_fixed_addresses +}; +#endif + +/* Leave one empty page between IO pages at 0xfd000000 and + * the top of the fixmap. + */ +#define FIXADDR_TOP (0xfcfff000UL) +#define FIXADDR_SIZE ((FIX_KMAP_END + 1) << PAGE_SHIFT) +#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE) + +#define __fix_to_virt(x) (FIXADDR_TOP - ((x) << PAGE_SHIFT)) + #define SUN4M_IOBASE_VADDR 0xfd000000 /* Base for mapping pages */ #define IOBASE_VADDR 0xfe000000 #define IOBASE_END 0xfe600000 diff --git a/arch/sparc/kernel/head_32.S b/arch/sparc/kernel/head_32.S index afeb1d770303..3d92c0a8f6c4 100644 --- a/arch/sparc/kernel/head_32.S +++ b/arch/sparc/kernel/head_32.S @@ -58,8 +58,6 @@ sun4e_notsup: /* This was the only reasonable way I could think of to properly align * these page-table data structures. */ - .globl swapper_pg_dir -swapper_pg_dir: .skip PAGE_SIZE .globl empty_zero_page empty_zero_page: .skip PAGE_SIZE diff --git a/arch/sparc/kernel/leon_kernel.c b/arch/sparc/kernel/leon_kernel.c index e34e2c40c060..f8b6eee40bde 100644 --- a/arch/sparc/kernel/leon_kernel.c +++ b/arch/sparc/kernel/leon_kernel.c @@ -486,17 +486,6 @@ void __init leon_trans_init(struct device_node *dp) } } -void __initdata (*prom_amba_init)(struct device_node *dp, struct device_node ***nextp) = 0; - -void __init leon_node_init(struct device_node *dp, struct device_node ***nextp) -{ - if (prom_amba_init && - strcmp(dp->type, "ambapp") == 0 && - strcmp(dp->name, "ambapp0") == 0) { - prom_amba_init(dp, nextp); - } -} - #ifdef CONFIG_SMP void leon_clear_profile_irq(int cpu) { @@ -522,8 +511,3 @@ void __init leon_init_IRQ(void) sparc_config.clear_clock_irq = leon_clear_clock_irq; sparc_config.load_profile_irq = leon_load_profile_irq; } - -void __init leon_init(void) -{ - of_pdt_build_more = &leon_node_init; -} diff --git a/arch/sparc/kernel/process_32.c b/arch/sparc/kernel/process_32.c index cb36e82dcd5d..14006d8aca28 100644 --- a/arch/sparc/kernel/process_32.c +++ b/arch/sparc/kernel/process_32.c @@ -333,9 +333,6 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, put_psr(get_psr() | PSR_EF); fpsave(&p->thread.float_regs[0], &p->thread.fsr, &p->thread.fpqueue[0], &p->thread.fpqdepth); -#ifdef CONFIG_SMP - clear_thread_flag(TIF_USEDFPU); -#endif } /* @@ -413,6 +410,7 @@ int copy_thread(unsigned long clone_flags, unsigned long sp, #ifdef CONFIG_SMP /* FPU must be disabled on SMP. */ childregs->psr &= ~PSR_EF; + clear_tsk_thread_flag(p, TIF_USEDFPU); #endif /* Set the return value for the child. */ diff --git a/arch/sparc/kernel/setup_32.c b/arch/sparc/kernel/setup_32.c index efe3e64bba38..38bf80a22f02 100644 --- a/arch/sparc/kernel/setup_32.c +++ b/arch/sparc/kernel/setup_32.c @@ -371,7 +371,6 @@ void __init setup_arch(char **cmdline_p) (*(linux_dbvec->teach_debugger))(); } - init_mm.context = (unsigned long) NO_CONTEXT; init_task.thread.kregs = &fake_swapper_regs; /* Run-time patch instructions to match the cpu model */ diff --git a/arch/sparc/kernel/sys_sparc_64.c b/arch/sparc/kernel/sys_sparc_64.c index 275f74fd6f6a..c38e5aaae56f 100644 --- a/arch/sparc/kernel/sys_sparc_64.c +++ b/arch/sparc/kernel/sys_sparc_64.c @@ -66,23 +66,6 @@ static inline int invalid_64bit_range(unsigned long addr, unsigned long len) return 0; } -/* Does start,end straddle the VA-space hole? */ -static inline int straddles_64bit_va_hole(unsigned long start, unsigned long end) -{ - unsigned long va_exclude_start, va_exclude_end; - - va_exclude_start = VA_EXCLUDE_START; - va_exclude_end = VA_EXCLUDE_END; - - if (likely(start < va_exclude_start && end < va_exclude_start)) - return 0; - - if (likely(start >= va_exclude_end && end >= va_exclude_end)) - return 0; - - return 1; -} - /* These functions differ from the default implementations in * mm/mmap.c in two ways: * diff --git a/arch/sparc/lib/NG2memcpy.S b/arch/sparc/lib/NG2memcpy.S index 0aed75653b50..03eadf66b0d3 100644 --- a/arch/sparc/lib/NG2memcpy.S +++ b/arch/sparc/lib/NG2memcpy.S @@ -90,49 +90,49 @@ faligndata %x7, %x8, %f14; #define FREG_MOVE_1(x0) \ - fmovd %x0, %f0; + fsrc2 %x0, %f0; #define FREG_MOVE_2(x0, x1) \ - fmovd %x0, %f0; \ - fmovd %x1, %f2; + fsrc2 %x0, %f0; \ + fsrc2 %x1, %f2; #define FREG_MOVE_3(x0, x1, x2) \ - fmovd %x0, %f0; \ - fmovd %x1, %f2; \ - fmovd %x2, %f4; + fsrc2 %x0, %f0; \ + fsrc2 %x1, %f2; \ + fsrc2 %x2, %f4; #define FREG_MOVE_4(x0, x1, x2, x3) \ - fmovd %x0, %f0; \ - fmovd %x1, %f2; \ - fmovd %x2, %f4; \ - fmovd %x3, %f6; + fsrc2 %x0, %f0; \ + fsrc2 %x1, %f2; \ + fsrc2 %x2, %f4; \ + fsrc2 %x3, %f6; #define FREG_MOVE_5(x0, x1, x2, x3, x4) \ - fmovd %x0, %f0; \ - fmovd %x1, %f2; \ - fmovd %x2, %f4; \ - fmovd %x3, %f6; \ - fmovd %x4, %f8; + fsrc2 %x0, %f0; \ + fsrc2 %x1, %f2; \ + fsrc2 %x2, %f4; \ + fsrc2 %x3, %f6; \ + fsrc2 %x4, %f8; #define FREG_MOVE_6(x0, x1, x2, x3, x4, x5) \ - fmovd %x0, %f0; \ - fmovd %x1, %f2; \ - fmovd %x2, %f4; \ - fmovd %x3, %f6; \ - fmovd %x4, %f8; \ - fmovd %x5, %f10; + fsrc2 %x0, %f0; \ + fsrc2 %x1, %f2; \ + fsrc2 %x2, %f4; \ + fsrc2 %x3, %f6; \ + fsrc2 %x4, %f8; \ + fsrc2 %x5, %f10; #define FREG_MOVE_7(x0, x1, x2, x3, x4, x5, x6) \ - fmovd %x0, %f0; \ - fmovd %x1, %f2; \ - fmovd %x2, %f4; \ - fmovd %x3, %f6; \ - fmovd %x4, %f8; \ - fmovd %x5, %f10; \ - fmovd %x6, %f12; + fsrc2 %x0, %f0; \ + fsrc2 %x1, %f2; \ + fsrc2 %x2, %f4; \ + fsrc2 %x3, %f6; \ + fsrc2 %x4, %f8; \ + fsrc2 %x5, %f10; \ + fsrc2 %x6, %f12; #define FREG_MOVE_8(x0, x1, x2, x3, x4, x5, x6, x7) \ - fmovd %x0, %f0; \ - fmovd %x1, %f2; \ - fmovd %x2, %f4; \ - fmovd %x3, %f6; \ - fmovd %x4, %f8; \ - fmovd %x5, %f10; \ - fmovd %x6, %f12; \ - fmovd %x7, %f14; + fsrc2 %x0, %f0; \ + fsrc2 %x1, %f2; \ + fsrc2 %x2, %f4; \ + fsrc2 %x3, %f6; \ + fsrc2 %x4, %f8; \ + fsrc2 %x5, %f10; \ + fsrc2 %x6, %f12; \ + fsrc2 %x7, %f14; #define FREG_LOAD_1(base, x0) \ EX_LD(LOAD(ldd, base + 0x00, %x0)) #define FREG_LOAD_2(base, x0, x1) \ diff --git a/arch/sparc/lib/U1memcpy.S b/arch/sparc/lib/U1memcpy.S index bafd2fc07acb..b67142b7768e 100644 --- a/arch/sparc/lib/U1memcpy.S +++ b/arch/sparc/lib/U1memcpy.S @@ -109,7 +109,7 @@ #define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \ subcc %left, 8, %left; \ bl,pn %xcc, 95f; \ - fsrc1 %f0, %f1; + fsrc2 %f0, %f1; #define UNEVEN_VISCHUNK(dest, f0, f1, left) \ UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \ @@ -201,7 +201,7 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ andn %o1, (0x40 - 1), %o1 and %g2, 7, %g2 andncc %g3, 0x7, %g3 - fmovd %f0, %f2 + fsrc2 %f0, %f2 sub %g3, 0x8, %g3 sub %o2, %GLOBAL_SPARE, %o2 diff --git a/arch/sparc/lib/copy_page.S b/arch/sparc/lib/copy_page.S index b243d3b606ba..4d2df328e514 100644 --- a/arch/sparc/lib/copy_page.S +++ b/arch/sparc/lib/copy_page.S @@ -34,10 +34,10 @@ #endif #define TOUCH(reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7) \ - fmovd %reg0, %f48; fmovd %reg1, %f50; \ - fmovd %reg2, %f52; fmovd %reg3, %f54; \ - fmovd %reg4, %f56; fmovd %reg5, %f58; \ - fmovd %reg6, %f60; fmovd %reg7, %f62; + fsrc2 %reg0, %f48; fsrc2 %reg1, %f50; \ + fsrc2 %reg2, %f52; fsrc2 %reg3, %f54; \ + fsrc2 %reg4, %f56; fsrc2 %reg5, %f58; \ + fsrc2 %reg6, %f60; fsrc2 %reg7, %f62; .text @@ -104,60 +104,60 @@ cheetah_copy_page_insn: prefetch [%o1 + 0x140], #one_read ldd [%o1 + 0x010], %f4 prefetch [%o1 + 0x180], #one_read - fmovd %f0, %f16 + fsrc2 %f0, %f16 ldd [%o1 + 0x018], %f6 - fmovd %f2, %f18 + fsrc2 %f2, %f18 ldd [%o1 + 0x020], %f8 - fmovd %f4, %f20 + fsrc2 %f4, %f20 ldd [%o1 + 0x028], %f10 - fmovd %f6, %f22 + fsrc2 %f6, %f22 ldd [%o1 + 0x030], %f12 - fmovd %f8, %f24 + fsrc2 %f8, %f24 ldd [%o1 + 0x038], %f14 - fmovd %f10, %f26 + fsrc2 %f10, %f26 ldd [%o1 + 0x040], %f0 1: ldd [%o1 + 0x048], %f2 - fmovd %f12, %f28 + fsrc2 %f12, %f28 ldd [%o1 + 0x050], %f4 - fmovd %f14, %f30 + fsrc2 %f14, %f30 stda %f16, [%o0] ASI_BLK_P ldd [%o1 + 0x058], %f6 - fmovd %f0, %f16 + fsrc2 %f0, %f16 ldd [%o1 + 0x060], %f8 - fmovd %f2, %f18 + fsrc2 %f2, %f18 ldd [%o1 + 0x068], %f10 - fmovd %f4, %f20 + fsrc2 %f4, %f20 ldd [%o1 + 0x070], %f12 - fmovd %f6, %f22 + fsrc2 %f6, %f22 ldd [%o1 + 0x078], %f14 - fmovd %f8, %f24 + fsrc2 %f8, %f24 ldd [%o1 + 0x080], %f0 prefetch [%o1 + 0x180], #one_read - fmovd %f10, %f26 + fsrc2 %f10, %f26 subcc %o2, 1, %o2 add %o0, 0x40, %o0 bne,pt %xcc, 1b add %o1, 0x40, %o1 ldd [%o1 + 0x048], %f2 - fmovd %f12, %f28 + fsrc2 %f12, %f28 ldd [%o1 + 0x050], %f4 - fmovd %f14, %f30 + fsrc2 %f14, %f30 stda %f16, [%o0] ASI_BLK_P ldd [%o1 + 0x058], %f6 - fmovd %f0, %f16 + fsrc2 %f0, %f16 ldd [%o1 + 0x060], %f8 - fmovd %f2, %f18 + fsrc2 %f2, %f18 ldd [%o1 + 0x068], %f10 - fmovd %f4, %f20 + fsrc2 %f4, %f20 ldd [%o1 + 0x070], %f12 - fmovd %f6, %f22 + fsrc2 %f6, %f22 add %o0, 0x40, %o0 ldd [%o1 + 0x078], %f14 - fmovd %f8, %f24 - fmovd %f10, %f26 - fmovd %f12, %f28 - fmovd %f14, %f30 + fsrc2 %f8, %f24 + fsrc2 %f10, %f26 + fsrc2 %f12, %f28 + fsrc2 %f14, %f30 stda %f16, [%o0] ASI_BLK_P membar #Sync VISExitHalf diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c index f46cf6be3370..77ac917be152 100644 --- a/arch/sparc/mm/fault_32.c +++ b/arch/sparc/mm/fault_32.c @@ -32,24 +32,6 @@ int show_unhandled_signals = 1; -/* At boot time we determine these two values necessary for setting - * up the segment maps and page table entries (pte's). - */ - -int num_contexts; - -/* Return how much physical memory we have. */ -unsigned long probe_memory(void) -{ - unsigned long total = 0; - int i; - - for (i = 0; sp_banks[i].num_bytes; i++) - total += sp_banks[i].num_bytes; - - return total; -} - static void unhandled_fault(unsigned long, struct task_struct *, struct pt_regs *) __attribute__ ((noreturn)); diff --git a/arch/sparc/mm/highmem.c b/arch/sparc/mm/highmem.c index 055c66cf1bf4..449f864f0cef 100644 --- a/arch/sparc/mm/highmem.c +++ b/arch/sparc/mm/highmem.c @@ -22,13 +22,31 @@ * shared by CPUs, and so precious, and establishing them requires IPI. * Atomic kmaps are lightweight and we may have NCPUS more of them. */ -#include <linux/mm.h> #include <linux/highmem.h> #include <linux/export.h> -#include <asm/pgalloc.h> +#include <linux/mm.h> + #include <asm/cacheflush.h> #include <asm/tlbflush.h> -#include <asm/fixmap.h> +#include <asm/pgalloc.h> +#include <asm/vaddrs.h> + +pgprot_t kmap_prot; + +static pte_t *kmap_pte; + +void __init kmap_init(void) +{ + unsigned long address; + pmd_t *dir; + + address = __fix_to_virt(FIX_KMAP_BEGIN); + dir = pmd_offset(pgd_offset_k(address), address); + + /* cache the first kmap pte */ + kmap_pte = pte_offset_kernel(dir, address); + kmap_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV | SRMMU_CACHE); +} void *kmap_atomic(struct page *page) { @@ -110,21 +128,3 @@ void __kunmap_atomic(void *kvaddr) pagefault_enable(); } EXPORT_SYMBOL(__kunmap_atomic); - -/* We may be fed a pagetable here by ptep_to_xxx and others. */ -struct page *kmap_atomic_to_page(void *ptr) -{ - unsigned long idx, vaddr = (unsigned long)ptr; - pte_t *pte; - - if (vaddr < SRMMU_NOCACHE_VADDR) - return virt_to_page(ptr); - if (vaddr < PKMAP_BASE) - return pfn_to_page(__nocache_pa(vaddr) >> PAGE_SHIFT); - BUG_ON(vaddr < FIXADDR_START); - BUG_ON(vaddr > FIXADDR_TOP); - - idx = virt_to_fix(vaddr); - pte = kmap_pte - (idx - FIX_KMAP_BEGIN); - return pte_page(*pte); -} diff --git a/arch/sparc/mm/init_32.c b/arch/sparc/mm/init_32.c index ef5c779ec855..dde85ef1c56d 100644 --- a/arch/sparc/mm/init_32.c +++ b/arch/sparc/mm/init_32.c @@ -45,9 +45,6 @@ unsigned long pfn_base; EXPORT_SYMBOL(pfn_base); struct sparc_phys_banks sp_banks[SPARC_PHYS_BANKS+1]; -unsigned long sparc_unmapped_base; - -struct pgtable_cache_struct pgt_quicklists; /* Initial ramdisk setup */ extern unsigned int sparc_ramdisk_image; @@ -55,19 +52,6 @@ extern unsigned int sparc_ramdisk_size; unsigned long highstart_pfn, highend_pfn; -pte_t *kmap_pte; -pgprot_t kmap_prot; - -#define kmap_get_fixmap_pte(vaddr) \ - pte_offset_kernel(pmd_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr)) - -void __init kmap_init(void) -{ - /* cache the first kmap pte */ - kmap_pte = kmap_get_fixmap_pte(__fix_to_virt(FIX_KMAP_BEGIN)); - kmap_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV | SRMMU_CACHE); -} - void show_mem(unsigned int filter) { printk("Mem-info:\n"); @@ -76,33 +60,8 @@ void show_mem(unsigned int filter) nr_swap_pages << (PAGE_SHIFT-10)); printk("%ld pages of RAM\n", totalram_pages); printk("%ld free pages\n", nr_free_pages()); -#if 0 /* undefined pgtable_cache_size, pgd_cache_size */ - printk("%ld pages in page table cache\n",pgtable_cache_size); -#ifndef CONFIG_SMP - if (sparc_cpu_model == sun4m || sparc_cpu_model == sun4d) - printk("%ld entries in page dir cache\n",pgd_cache_size); -#endif -#endif } -void __init sparc_context_init(int numctx) -{ - int ctx; - - ctx_list_pool = __alloc_bootmem(numctx * sizeof(struct ctx_list), SMP_CACHE_BYTES, 0UL); - - for(ctx = 0; ctx < numctx; ctx++) { - struct ctx_list *clist; - - clist = (ctx_list_pool + ctx); - clist->ctx_number = ctx; - clist->ctx_mm = NULL; - } - ctx_free.next = ctx_free.prev = &ctx_free; - ctx_used.next = ctx_used.prev = &ctx_used; - for(ctx = 0; ctx < numctx; ctx++) - add_to_free_ctxlist(ctx_list_pool + ctx); -} extern unsigned long cmdline_memory_size; unsigned long last_valid_pfn; @@ -292,22 +251,7 @@ extern void device_scan(void); void __init paging_init(void) { - switch(sparc_cpu_model) { - case sparc_leon: - leon_init(); - /* fall through */ - case sun4m: - case sun4d: - srmmu_paging_init(); - sparc_unmapped_base = 0x50000000; - break; - default: - prom_printf("paging_init: Cannot init paging on this Sparc\n"); - prom_printf("paging_init: sparc_cpu_model = %d\n", sparc_cpu_model); - prom_printf("paging_init: Halting...\n"); - prom_halt(); - } - + srmmu_paging_init(); prom_build_devicetree(); of_fill_in_cpu_data(); device_scan(); diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c index 62e3f5773303..c38bb72e3e80 100644 --- a/arch/sparc/mm/srmmu.c +++ b/arch/sparc/mm/srmmu.c @@ -8,45 +8,45 @@ * Copyright (C) 1999,2000 Anton Blanchard (anton@samba.org) */ -#include <linux/kernel.h> -#include <linux/mm.h> -#include <linux/vmalloc.h> -#include <linux/pagemap.h> -#include <linux/init.h> +#include <linux/seq_file.h> #include <linux/spinlock.h> #include <linux/bootmem.h> -#include <linux/fs.h> -#include <linux/seq_file.h> +#include <linux/pagemap.h> +#include <linux/vmalloc.h> #include <linux/kdebug.h> +#include <linux/kernel.h> +#include <linux/init.h> #include <linux/log2.h> #include <linux/gfp.h> +#include <linux/fs.h> +#include <linux/mm.h> -#include <asm/bitext.h> -#include <asm/page.h> +#include <asm/mmu_context.h> +#include <asm/cacheflush.h> +#include <asm/tlbflush.h> +#include <asm/io-unit.h> #include <asm/pgalloc.h> #include <asm/pgtable.h> -#include <asm/io.h> +#include <asm/bitext.h> #include <asm/vaddrs.h> -#include <asm/traps.h> -#include <asm/smp.h> -#include <asm/mbus.h> #include <asm/cache.h> +#include <asm/traps.h> #include <asm/oplib.h> +#include <asm/mbus.h> +#include <asm/page.h> #include <asm/asi.h> #include <asm/msi.h> -#include <asm/mmu_context.h> -#include <asm/io-unit.h> -#include <asm/cacheflush.h> -#include <asm/tlbflush.h> +#include <asm/smp.h> +#include <asm/io.h> /* Now the cpu specific definitions. */ -#include <asm/viking.h> -#include <asm/mxcc.h> -#include <asm/ross.h> +#include <asm/turbosparc.h> #include <asm/tsunami.h> +#include <asm/viking.h> #include <asm/swift.h> -#include <asm/turbosparc.h> #include <asm/leon.h> +#include <asm/mxcc.h> +#include <asm/ross.h> #include "srmmu.h" @@ -55,10 +55,6 @@ static unsigned int hwbug_bitmask; int vac_cache_size; int vac_line_size; -struct ctx_list *ctx_list_pool; -struct ctx_list ctx_free; -struct ctx_list ctx_used; - extern struct resource sparc_iomap; extern unsigned long last_valid_pfn; @@ -136,8 +132,8 @@ void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, struct page *ptep) } } -/* Find an entry in the third-level page table.. */ -pte_t *pte_offset_kernel(pmd_t * dir, unsigned long address) +/* Find an entry in the third-level page table.. */ +pte_t *pte_offset_kernel(pmd_t *dir, unsigned long address) { void *pte; @@ -151,55 +147,61 @@ pte_t *pte_offset_kernel(pmd_t * dir, unsigned long address) * align: bytes, number to align at. * Returns the virtual address of the allocated area. */ -static unsigned long __srmmu_get_nocache(int size, int align) +static void *__srmmu_get_nocache(int size, int align) { int offset; + unsigned long addr; if (size < SRMMU_NOCACHE_BITMAP_SHIFT) { - printk("Size 0x%x too small for nocache request\n", size); + printk(KERN_ERR "Size 0x%x too small for nocache request\n", + size); size = SRMMU_NOCACHE_BITMAP_SHIFT; } - if (size & (SRMMU_NOCACHE_BITMAP_SHIFT-1)) { - printk("Size 0x%x unaligned int nocache request\n", size); - size += SRMMU_NOCACHE_BITMAP_SHIFT-1; + if (size & (SRMMU_NOCACHE_BITMAP_SHIFT - 1)) { + printk(KERN_ERR "Size 0x%x unaligned int nocache request\n", + size); + size += SRMMU_NOCACHE_BITMAP_SHIFT - 1; } BUG_ON(align > SRMMU_NOCACHE_ALIGN_MAX); offset = bit_map_string_get(&srmmu_nocache_map, - size >> SRMMU_NOCACHE_BITMAP_SHIFT, - align >> SRMMU_NOCACHE_BITMAP_SHIFT); + size >> SRMMU_NOCACHE_BITMAP_SHIFT, + align >> SRMMU_NOCACHE_BITMAP_SHIFT); if (offset == -1) { - printk("srmmu: out of nocache %d: %d/%d\n", - size, (int) srmmu_nocache_size, - srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT); + printk(KERN_ERR "srmmu: out of nocache %d: %d/%d\n", + size, (int) srmmu_nocache_size, + srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT); return 0; } - return (SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT)); + addr = SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT); + return (void *)addr; } -unsigned long srmmu_get_nocache(int size, int align) +void *srmmu_get_nocache(int size, int align) { - unsigned long tmp; + void *tmp; tmp = __srmmu_get_nocache(size, align); if (tmp) - memset((void *)tmp, 0, size); + memset(tmp, 0, size); return tmp; } -void srmmu_free_nocache(unsigned long vaddr, int size) +void srmmu_free_nocache(void *addr, int size) { + unsigned long vaddr; int offset; + vaddr = (unsigned long)addr; if (vaddr < SRMMU_NOCACHE_VADDR) { printk("Vaddr %lx is smaller than nocache base 0x%lx\n", vaddr, (unsigned long)SRMMU_NOCACHE_VADDR); BUG(); } - if (vaddr+size > srmmu_nocache_end) { + if (vaddr + size > srmmu_nocache_end) { printk("Vaddr %lx is bigger than nocache end 0x%lx\n", vaddr, srmmu_nocache_end); BUG(); @@ -212,7 +214,7 @@ void srmmu_free_nocache(unsigned long vaddr, int size) printk("Size 0x%x is too small\n", size); BUG(); } - if (vaddr & (size-1)) { + if (vaddr & (size - 1)) { printk("Vaddr %lx is not aligned to size 0x%x\n", vaddr, size); BUG(); } @@ -226,13 +228,23 @@ void srmmu_free_nocache(unsigned long vaddr, int size) static void srmmu_early_allocate_ptable_skeleton(unsigned long start, unsigned long end); -extern unsigned long probe_memory(void); /* in fault.c */ +/* Return how much physical memory we have. */ +static unsigned long __init probe_memory(void) +{ + unsigned long total = 0; + int i; + + for (i = 0; sp_banks[i].num_bytes; i++) + total += sp_banks[i].num_bytes; + + return total; +} /* * Reserve nocache dynamically proportionally to the amount of * system RAM. -- Tomas Szepe <szepe@pinerecords.com>, June 2002 */ -static void srmmu_nocache_calcsize(void) +static void __init srmmu_nocache_calcsize(void) { unsigned long sysmemavail = probe_memory() / 1024; int srmmu_nocache_npages; @@ -271,7 +283,7 @@ static void __init srmmu_nocache_init(void) srmmu_nocache_bitmap = __alloc_bootmem(bitmap_bits >> 3, SMP_CACHE_BYTES, 0UL); bit_map_init(&srmmu_nocache_map, srmmu_nocache_bitmap, bitmap_bits); - srmmu_swapper_pg_dir = (pgd_t *)__srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE); + srmmu_swapper_pg_dir = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE); memset(__nocache_fix(srmmu_swapper_pg_dir), 0, SRMMU_PGD_TABLE_SIZE); init_mm.pgd = srmmu_swapper_pg_dir; @@ -304,7 +316,7 @@ pgd_t *get_pgd_fast(void) { pgd_t *pgd = NULL; - pgd = (pgd_t *)__srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE); + pgd = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE); if (pgd) { pgd_t *init = pgd_offset_k(0); memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t)); @@ -330,7 +342,7 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) if ((pte = (unsigned long)pte_alloc_one_kernel(mm, address)) == 0) return NULL; - page = pfn_to_page( __nocache_pa(pte) >> PAGE_SHIFT ); + page = pfn_to_page(__nocache_pa(pte) >> PAGE_SHIFT); pgtable_page_ctor(page); return page; } @@ -344,18 +356,50 @@ void pte_free(struct mm_struct *mm, pgtable_t pte) if (p == 0) BUG(); p = page_to_pfn(pte) << PAGE_SHIFT; /* Physical address */ - p = (unsigned long) __nocache_va(p); /* Nocached virtual */ - srmmu_free_nocache(p, PTE_SIZE); + + /* free non cached virtual address*/ + srmmu_free_nocache(__nocache_va(p), PTE_SIZE); } -/* - */ +/* context handling - a dynamically sized pool is used */ +#define NO_CONTEXT -1 + +struct ctx_list { + struct ctx_list *next; + struct ctx_list *prev; + unsigned int ctx_number; + struct mm_struct *ctx_mm; +}; + +static struct ctx_list *ctx_list_pool; +static struct ctx_list ctx_free; +static struct ctx_list ctx_used; + +/* At boot time we determine the number of contexts */ +static int num_contexts; + +static inline void remove_from_ctx_list(struct ctx_list *entry) +{ + entry->next->prev = entry->prev; + entry->prev->next = entry->next; +} + +static inline void add_to_ctx_list(struct ctx_list *head, struct ctx_list *entry) +{ + entry->next = head; + (entry->prev = head->prev)->next = entry; + head->prev = entry; +} +#define add_to_free_ctxlist(entry) add_to_ctx_list(&ctx_free, entry) +#define add_to_used_ctxlist(entry) add_to_ctx_list(&ctx_used, entry) + + static inline void alloc_context(struct mm_struct *old_mm, struct mm_struct *mm) { struct ctx_list *ctxp; ctxp = ctx_free.next; - if(ctxp != &ctx_free) { + if (ctxp != &ctx_free) { remove_from_ctx_list(ctxp); add_to_used_ctxlist(ctxp); mm->context = ctxp->ctx_number; @@ -363,9 +407,9 @@ static inline void alloc_context(struct mm_struct *old_mm, struct mm_struct *mm) return; } ctxp = ctx_used.next; - if(ctxp->ctx_mm == old_mm) + if (ctxp->ctx_mm == old_mm) ctxp = ctxp->next; - if(ctxp == &ctx_used) + if (ctxp == &ctx_used) panic("out of mmu contexts"); flush_cache_mm(ctxp->ctx_mm); flush_tlb_mm(ctxp->ctx_mm); @@ -385,11 +429,31 @@ static inline void free_context(int context) add_to_free_ctxlist(ctx_old); } +static void __init sparc_context_init(int numctx) +{ + int ctx; + unsigned long size; + + size = numctx * sizeof(struct ctx_list); + ctx_list_pool = __alloc_bootmem(size, SMP_CACHE_BYTES, 0UL); + + for (ctx = 0; ctx < numctx; ctx++) { + struct ctx_list *clist; + + clist = (ctx_list_pool + ctx); + clist->ctx_number = ctx; + clist->ctx_mm = NULL; + } + ctx_free.next = ctx_free.prev = &ctx_free; + ctx_used.next = ctx_used.prev = &ctx_used; + for (ctx = 0; ctx < numctx; ctx++) + add_to_free_ctxlist(ctx_list_pool + ctx); +} void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, struct task_struct *tsk) { - if(mm->context == NO_CONTEXT) { + if (mm->context == NO_CONTEXT) { spin_lock(&srmmu_context_spinlock); alloc_context(old_mm, mm); spin_unlock(&srmmu_context_spinlock); @@ -407,7 +471,7 @@ void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm, /* Low level IO area allocation on the SRMMU. */ static inline void srmmu_mapioaddr(unsigned long physaddr, - unsigned long virt_addr, int bus_type) + unsigned long virt_addr, int bus_type) { pgd_t *pgdp; pmd_t *pmdp; @@ -420,8 +484,7 @@ static inline void srmmu_mapioaddr(unsigned long physaddr, ptep = pte_offset_kernel(pmdp, virt_addr); tmp = (physaddr >> 4) | SRMMU_ET_PTE; - /* - * I need to test whether this is consistent over all + /* I need to test whether this is consistent over all * sun4m's. The bus_type represents the upper 4 bits of * 36-bit physical address on the I/O space lines... */ @@ -591,10 +654,10 @@ static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start, pmd_t *pmdp; pte_t *ptep; - while(start < end) { + while (start < end) { pgdp = pgd_offset_k(start); if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) { - pmdp = (pmd_t *) __srmmu_get_nocache( + pmdp = __srmmu_get_nocache( SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); if (pmdp == NULL) early_pgtable_allocfail("pmd"); @@ -602,8 +665,8 @@ static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start, pgd_set(__nocache_fix(pgdp), pmdp); } pmdp = pmd_offset(__nocache_fix(pgdp), start); - if(srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { - ptep = (pte_t *)__srmmu_get_nocache(PTE_SIZE, PTE_SIZE); + if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { + ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE); if (ptep == NULL) early_pgtable_allocfail("pte"); memset(__nocache_fix(ptep), 0, PTE_SIZE); @@ -622,18 +685,18 @@ static void __init srmmu_allocate_ptable_skeleton(unsigned long start, pmd_t *pmdp; pte_t *ptep; - while(start < end) { + while (start < end) { pgdp = pgd_offset_k(start); if (pgd_none(*pgdp)) { - pmdp = (pmd_t *)__srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); + pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); if (pmdp == NULL) early_pgtable_allocfail("pmd"); memset(pmdp, 0, SRMMU_PMD_TABLE_SIZE); pgd_set(pgdp, pmdp); } pmdp = pmd_offset(pgdp, start); - if(srmmu_pmd_none(*pmdp)) { - ptep = (pte_t *) __srmmu_get_nocache(PTE_SIZE, + if (srmmu_pmd_none(*pmdp)) { + ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE); if (ptep == NULL) early_pgtable_allocfail("pte"); @@ -671,72 +734,76 @@ static inline unsigned long srmmu_probe(unsigned long vaddr) static void __init srmmu_inherit_prom_mappings(unsigned long start, unsigned long end) { + unsigned long probed; + unsigned long addr; pgd_t *pgdp; pmd_t *pmdp; pte_t *ptep; - int what = 0; /* 0 = normal-pte, 1 = pmd-level pte, 2 = pgd-level pte */ - unsigned long prompte; + int what; /* 0 = normal-pte, 1 = pmd-level pte, 2 = pgd-level pte */ - while(start <= end) { + while (start <= end) { if (start == 0) break; /* probably wrap around */ - if(start == 0xfef00000) + if (start == 0xfef00000) start = KADB_DEBUGGER_BEGVM; - if(!(prompte = srmmu_probe(start))) { + probed = srmmu_probe(start); + if (!probed) { + /* continue probing until we find an entry */ start += PAGE_SIZE; continue; } - + /* A red snapper, see what it really is. */ what = 0; - - if(!(start & ~(SRMMU_REAL_PMD_MASK))) { - if(srmmu_probe((start-PAGE_SIZE) + SRMMU_REAL_PMD_SIZE) == prompte) + addr = start - PAGE_SIZE; + + if (!(start & ~(SRMMU_REAL_PMD_MASK))) { + if (srmmu_probe(addr + SRMMU_REAL_PMD_SIZE) == probed) what = 1; } - - if(!(start & ~(SRMMU_PGDIR_MASK))) { - if(srmmu_probe((start-PAGE_SIZE) + SRMMU_PGDIR_SIZE) == - prompte) + + if (!(start & ~(SRMMU_PGDIR_MASK))) { + if (srmmu_probe(addr + SRMMU_PGDIR_SIZE) == probed) what = 2; } - + pgdp = pgd_offset_k(start); - if(what == 2) { - *(pgd_t *)__nocache_fix(pgdp) = __pgd(prompte); + if (what == 2) { + *(pgd_t *)__nocache_fix(pgdp) = __pgd(probed); start += SRMMU_PGDIR_SIZE; continue; } if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) { - pmdp = (pmd_t *)__srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE); + pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, + SRMMU_PMD_TABLE_SIZE); if (pmdp == NULL) early_pgtable_allocfail("pmd"); memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE); pgd_set(__nocache_fix(pgdp), pmdp); } pmdp = pmd_offset(__nocache_fix(pgdp), start); - if(srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { - ptep = (pte_t *) __srmmu_get_nocache(PTE_SIZE, - PTE_SIZE); + if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) { + ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE); if (ptep == NULL) early_pgtable_allocfail("pte"); memset(__nocache_fix(ptep), 0, PTE_SIZE); pmd_set(__nocache_fix(pmdp), ptep); } - if(what == 1) { - /* - * We bend the rule where all 16 PTPs in a pmd_t point + if (what == 1) { + /* We bend the rule where all 16 PTPs in a pmd_t point * inside the same PTE page, and we leak a perfectly * good hardware PTE piece. Alternatives seem worse. */ unsigned int x; /* Index of HW PMD in soft cluster */ + unsigned long *val; x = (start >> PMD_SHIFT) & 15; - *(unsigned long *)__nocache_fix(&pmdp->pmdv[x]) = prompte; + val = &pmdp->pmdv[x]; + *(unsigned long *)__nocache_fix(val) = probed; start += SRMMU_REAL_PMD_SIZE; continue; } ptep = pte_offset_kernel(__nocache_fix(pmdp), start); - *(pte_t *)__nocache_fix(ptep) = __pte(prompte); + *(pte_t *)__nocache_fix(ptep) = __pte(probed); start += PAGE_SIZE; } } @@ -765,18 +832,18 @@ static unsigned long __init map_spbank(unsigned long vbase, int sp_entry) if (vstart < min_vaddr || vstart >= max_vaddr) return vstart; - + if (vend > max_vaddr || vend < min_vaddr) vend = max_vaddr; - while(vstart < vend) { + while (vstart < vend) { do_large_mapping(vstart, pstart); vstart += SRMMU_PGDIR_SIZE; pstart += SRMMU_PGDIR_SIZE; } return vstart; } -static inline void map_kernel(void) +static void __init map_kernel(void) { int i; @@ -789,9 +856,6 @@ static inline void map_kernel(void) } } -/* Paging initialization on the Sparc Reference MMU. */ -extern void sparc_context_init(int); - void (*poke_srmmu)(void) __cpuinitdata = NULL; extern unsigned long bootmem_init(unsigned long *pages_avail); @@ -806,6 +870,7 @@ void __init srmmu_paging_init(void) pte_t *pte; unsigned long pages_avail; + init_mm.context = (unsigned long) NO_CONTEXT; sparc_iomap.start = SUN4M_IOBASE_VADDR; /* 16MB of IOSPACE on all sun4m's. */ if (sparc_cpu_model == sun4d) @@ -814,9 +879,9 @@ void __init srmmu_paging_init(void) /* Find the number of contexts on the srmmu. */ cpunode = prom_getchild(prom_root_node); num_contexts = 0; - while(cpunode != 0) { + while (cpunode != 0) { prom_getstring(cpunode, "device_type", node_str, sizeof(node_str)); - if(!strcmp(node_str, "cpu")) { + if (!strcmp(node_str, "cpu")) { num_contexts = prom_getintdefault(cpunode, "mmu-nctx", 0x8); break; } @@ -824,7 +889,7 @@ void __init srmmu_paging_init(void) } } - if(!num_contexts) { + if (!num_contexts) { prom_printf("Something wrong, can't find cpu node in paging_init.\n"); prom_halt(); } @@ -834,14 +899,14 @@ void __init srmmu_paging_init(void) srmmu_nocache_calcsize(); srmmu_nocache_init(); - srmmu_inherit_prom_mappings(0xfe400000,(LINUX_OPPROM_ENDVM-PAGE_SIZE)); + srmmu_inherit_prom_mappings(0xfe400000, (LINUX_OPPROM_ENDVM - PAGE_SIZE)); map_kernel(); /* ctx table has to be physically aligned to its size */ - srmmu_context_table = (ctxd_t *)__srmmu_get_nocache(num_contexts*sizeof(ctxd_t), num_contexts*sizeof(ctxd_t)); + srmmu_context_table = __srmmu_get_nocache(num_contexts * sizeof(ctxd_t), num_contexts * sizeof(ctxd_t)); srmmu_ctx_table_phys = (ctxd_t *)__nocache_pa((unsigned long)srmmu_context_table); - for(i = 0; i < num_contexts; i++) + for (i = 0; i < num_contexts; i++) srmmu_ctxd_set((ctxd_t *)__nocache_fix(&srmmu_context_table[i]), srmmu_swapper_pg_dir); flush_cache_all(); @@ -897,7 +962,7 @@ void __init srmmu_paging_init(void) void mmu_info(struct seq_file *m) { - seq_printf(m, + seq_printf(m, "MMU type\t: %s\n" "contexts\t: %d\n" "nocache total\t: %ld\n" @@ -908,10 +973,16 @@ void mmu_info(struct seq_file *m) srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT); } +int init_new_context(struct task_struct *tsk, struct mm_struct *mm) +{ + mm->context = NO_CONTEXT; + return 0; +} + void destroy_context(struct mm_struct *mm) { - if(mm->context != NO_CONTEXT) { + if (mm->context != NO_CONTEXT) { flush_cache_mm(mm); srmmu_ctxd_set(&srmmu_context_table[mm->context], srmmu_swapper_pg_dir); flush_tlb_mm(mm); @@ -941,13 +1012,12 @@ static void __init init_vac_layout(void) #endif nd = prom_getchild(prom_root_node); - while((nd = prom_getsibling(nd)) != 0) { + while ((nd = prom_getsibling(nd)) != 0) { prom_getstring(nd, "device_type", node_str, sizeof(node_str)); - if(!strcmp(node_str, "cpu")) { + if (!strcmp(node_str, "cpu")) { vac_line_size = prom_getint(nd, "cache-line-size"); if (vac_line_size == -1) { - prom_printf("can't determine cache-line-size, " - "halting.\n"); + prom_printf("can't determine cache-line-size, halting.\n"); prom_halt(); } cache_lines = prom_getint(nd, "cache-nlines"); @@ -958,9 +1028,9 @@ static void __init init_vac_layout(void) vac_cache_size = cache_lines * vac_line_size; #ifdef CONFIG_SMP - if(vac_cache_size > max_size) + if (vac_cache_size > max_size) max_size = vac_cache_size; - if(vac_line_size < min_line_size) + if (vac_line_size < min_line_size) min_line_size = vac_line_size; //FIXME: cpus not contiguous!! cpu++; @@ -971,7 +1041,7 @@ static void __init init_vac_layout(void) #endif } } - if(nd == 0) { + if (nd == 0) { prom_printf("No CPU nodes found, halting.\n"); prom_halt(); } @@ -1082,7 +1152,7 @@ static void __init init_swift(void) "=r" (swift_rev) : "r" (SWIFT_MASKID_ADDR), "i" (ASI_M_BYPASS)); srmmu_name = "Fujitsu Swift"; - switch(swift_rev) { + switch (swift_rev) { case 0x11: case 0x20: case 0x23: @@ -1222,10 +1292,11 @@ static void __cpuinit poke_turbosparc(void) /* Clear any crap from the cache or else... */ turbosparc_flush_cache_all(); - mreg &= ~(TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* Temporarily disable I & D caches */ + /* Temporarily disable I & D caches */ + mreg &= ~(TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); mreg &= ~(TURBOSPARC_PCENABLE); /* Don't check parity */ srmmu_set_mmureg(mreg); - + ccreg = turbosparc_get_ccreg(); #ifdef TURBOSPARC_WRITEBACK @@ -1248,7 +1319,7 @@ static void __cpuinit poke_turbosparc(void) default: ccreg |= (TURBOSPARC_SCENABLE); } - turbosparc_set_ccreg (ccreg); + turbosparc_set_ccreg(ccreg); mreg |= (TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* I & D caches on */ mreg |= (TURBOSPARC_ICSNOOP); /* Icache snooping on */ @@ -1342,7 +1413,7 @@ static void __cpuinit poke_viking(void) unsigned long bpreg; mreg &= ~(VIKING_TCENABLE); - if(smp_catch++) { + if (smp_catch++) { /* Must disable mixed-cmd mode here for other cpu's. */ bpreg = viking_get_bpreg(); bpreg &= ~(VIKING_ACTION_MIX); @@ -1411,7 +1482,7 @@ static void __init init_viking(void) unsigned long mreg = srmmu_get_mmureg(); /* Ahhh, the viking. SRMMU VLSI abortion number two... */ - if(mreg & VIKING_MMODE) { + if (mreg & VIKING_MMODE) { srmmu_name = "TI Viking"; viking_mxcc_present = 0; msi_set_sync(); @@ -1467,8 +1538,8 @@ static void __init get_srmmu_type(void) } /* Second, check for HyperSparc or Cypress. */ - if(mod_typ == 1) { - switch(mod_rev) { + if (mod_typ == 1) { + switch (mod_rev) { case 7: /* UP or MP Hypersparc */ init_hypersparc(); @@ -1488,9 +1559,8 @@ static void __init get_srmmu_type(void) } return; } - - /* - * Now Fujitsu TurboSparc. It might happen that it is + + /* Now Fujitsu TurboSparc. It might happen that it is * in Swift emulation mode, so we will check later... */ if (psr_typ == 0 && psr_vers == 5) { @@ -1499,15 +1569,15 @@ static void __init get_srmmu_type(void) } /* Next check for Fujitsu Swift. */ - if(psr_typ == 0 && psr_vers == 4) { + if (psr_typ == 0 && psr_vers == 4) { phandle cpunode; char node_str[128]; /* Look if it is not a TurboSparc emulating Swift... */ cpunode = prom_getchild(prom_root_node); - while((cpunode = prom_getsibling(cpunode)) != 0) { + while ((cpunode = prom_getsibling(cpunode)) != 0) { prom_getstring(cpunode, "device_type", node_str, sizeof(node_str)); - if(!strcmp(node_str, "cpu")) { + if (!strcmp(node_str, "cpu")) { if (!prom_getintdefault(cpunode, "psr-implementation", 1) && prom_getintdefault(cpunode, "psr-version", 1) == 5) { init_turbosparc(); @@ -1516,13 +1586,13 @@ static void __init get_srmmu_type(void) break; } } - + init_swift(); return; } /* Now the Viking family of srmmu. */ - if(psr_typ == 4 && + if (psr_typ == 4 && ((psr_vers == 0) || ((psr_vers == 1) && (mod_typ == 0) && (mod_rev == 0)))) { init_viking(); @@ -1530,7 +1600,7 @@ static void __init get_srmmu_type(void) } /* Finally the Tsunami. */ - if(psr_typ == 4 && psr_vers == 1 && (mod_typ || mod_rev)) { + if (psr_typ == 4 && psr_vers == 1 && (mod_typ || mod_rev)) { init_tsunami(); return; } diff --git a/arch/sparc/prom/init_32.c b/arch/sparc/prom/init_32.c index 26c64cea3c9c..9ac30c2b7dba 100644 --- a/arch/sparc/prom/init_32.c +++ b/arch/sparc/prom/init_32.c @@ -27,13 +27,10 @@ EXPORT_SYMBOL(prom_root_node); struct linux_nodeops *prom_nodeops; /* You must call prom_init() before you attempt to use any of the - * routines in the prom library. It returns 0 on success, 1 on - * failure. It gets passed the pointer to the PROM vector. + * routines in the prom library. + * It gets passed the pointer to the PROM vector. */ -extern void prom_meminit(void); -extern void prom_ranges_init(void); - void __init prom_init(struct linux_romvec *rp) { romvec = rp; diff --git a/arch/sparc/prom/init_64.c b/arch/sparc/prom/init_64.c index 5016c5e20575..d95db755828f 100644 --- a/arch/sparc/prom/init_64.c +++ b/arch/sparc/prom/init_64.c @@ -22,8 +22,8 @@ int prom_stdout; phandle prom_chosen_node; /* You must call prom_init() before you attempt to use any of the - * routines in the prom library. It returns 0 on success, 1 on - * failure. It gets passed the pointer to the PROM vector. + * routines in the prom library. + * It gets passed the pointer to the PROM vector. */ extern void prom_cif_init(void *, void *); diff --git a/arch/tile/kernel/pci.c b/arch/tile/kernel/pci.c index 0fdd99d0d8b7..33c10864d2f7 100644 --- a/arch/tile/kernel/pci.c +++ b/arch/tile/kernel/pci.c @@ -369,7 +369,7 @@ int __init pcibios_init(void) */ if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && (PCI_SLOT(dev->devfn) == 0)) { - next_bus = dev->busn_res.end; + next_bus = dev->subordinate; controllers[i].mem_resources[0] = *next_bus->resource[0]; controllers[i].mem_resources[1] = diff --git a/arch/tile/kernel/pci_gx.c b/arch/tile/kernel/pci_gx.c index fa75264a82ae..0e213e35ffc3 100644 --- a/arch/tile/kernel/pci_gx.c +++ b/arch/tile/kernel/pci_gx.c @@ -853,7 +853,7 @@ int __init pcibios_init(void) bus = pci_scan_root_bus(NULL, next_busno, controller->ops, controller, &resources); controller->root_bus = bus; - next_busno = bus->subordinate + 1; + next_busno = bus->busn_res.end + 1; } diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug index e46c2147397f..b322f124ee3c 100644 --- a/arch/x86/Kconfig.debug +++ b/arch/x86/Kconfig.debug @@ -129,6 +129,25 @@ config DOUBLEFAULT option saves about 4k and might cause you much additional grey hair. +config DEBUG_TLBFLUSH + bool "Set upper limit of TLB entries to flush one-by-one" + depends on DEBUG_KERNEL && (X86_64 || X86_INVLPG) + ---help--- + + X86-only for now. + + This option allows the user to tune the amount of TLB entries the + kernel flushes one-by-one instead of doing a full TLB flush. In + certain situations, the former is cheaper. This is controlled by the + tlb_flushall_shift knob under /sys/kernel/debug/x86. If you set it + to -1, the code flushes the whole TLB unconditionally. Otherwise, + for positive values of it, the kernel will use single TLB entry + invalidating instructions according to the following formula: + + flush_entries <= active_tlb_entries / 2^tlb_flushall_shift + + If in doubt, say "N". + config IOMMU_DEBUG bool "Enable IOMMU debugging" depends on GART_IOMMU && DEBUG_KERNEL diff --git a/arch/x86/boot/compressed/cmdline.c b/arch/x86/boot/compressed/cmdline.c index cb62f786990d..10f6b1178c68 100644 --- a/arch/x86/boot/compressed/cmdline.c +++ b/arch/x86/boot/compressed/cmdline.c @@ -1,5 +1,7 @@ #include "misc.h" +#ifdef CONFIG_EARLY_PRINTK + static unsigned long fs; static inline void set_fs(unsigned long seg) { @@ -19,3 +21,5 @@ int cmdline_find_option_bool(const char *option) { return __cmdline_find_option_bool(real_mode->hdr.cmd_line_ptr, option); } + +#endif diff --git a/arch/x86/boot/compressed/early_serial_console.c b/arch/x86/boot/compressed/early_serial_console.c index 261e81fb9582..d3d003cb5481 100644 --- a/arch/x86/boot/compressed/early_serial_console.c +++ b/arch/x86/boot/compressed/early_serial_console.c @@ -1,5 +1,9 @@ #include "misc.h" +#ifdef CONFIG_EARLY_PRINTK + int early_serial_base; #include "../early_serial_console.c" + +#endif diff --git a/arch/x86/boot/compressed/eboot.c b/arch/x86/boot/compressed/eboot.c index 4e85f5f85837..b3e0227df2c9 100644 --- a/arch/x86/boot/compressed/eboot.c +++ b/arch/x86/boot/compressed/eboot.c @@ -729,32 +729,68 @@ fail: * need to create one ourselves (usually the bootloader would create * one for us). */ -static efi_status_t make_boot_params(struct boot_params *boot_params, - efi_loaded_image_t *image, - void *handle) +struct boot_params *make_boot_params(void *handle, efi_system_table_t *_table) { - struct efi_info *efi = &boot_params->efi_info; - struct apm_bios_info *bi = &boot_params->apm_bios_info; - struct sys_desc_table *sdt = &boot_params->sys_desc_table; - struct e820entry *e820_map = &boot_params->e820_map[0]; - struct e820entry *prev = NULL; - struct setup_header *hdr = &boot_params->hdr; - unsigned long size, key, desc_size, _size; - efi_memory_desc_t *mem_map; - void *options = image->load_options; - u32 load_options_size = image->load_options_size / 2; /* ASCII */ + struct boot_params *boot_params; + struct sys_desc_table *sdt; + struct apm_bios_info *bi; + struct setup_header *hdr; + struct efi_info *efi; + efi_loaded_image_t *image; + void *options; + u32 load_options_size; + efi_guid_t proto = LOADED_IMAGE_PROTOCOL_GUID; int options_size = 0; efi_status_t status; - __u32 desc_version; unsigned long cmdline; - u8 nr_entries; u16 *s2; u8 *s1; int i; + sys_table = _table; + + /* Check if we were booted by the EFI firmware */ + if (sys_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) + return NULL; + + status = efi_call_phys3(sys_table->boottime->handle_protocol, + handle, &proto, (void *)&image); + if (status != EFI_SUCCESS) { + efi_printk("Failed to get handle for LOADED_IMAGE_PROTOCOL\n"); + return NULL; + } + + status = low_alloc(0x4000, 1, (unsigned long *)&boot_params); + if (status != EFI_SUCCESS) { + efi_printk("Failed to alloc lowmem for boot params\n"); + return NULL; + } + + memset(boot_params, 0x0, 0x4000); + + hdr = &boot_params->hdr; + efi = &boot_params->efi_info; + bi = &boot_params->apm_bios_info; + sdt = &boot_params->sys_desc_table; + + /* Copy the second sector to boot_params */ + memcpy(&hdr->jump, image->image_base + 512, 512); + + /* + * Fill out some of the header fields ourselves because the + * EFI firmware loader doesn't load the first sector. + */ + hdr->root_flags = 1; + hdr->vid_mode = 0xffff; + hdr->boot_flag = 0xAA55; + + hdr->code32_start = (__u64)(unsigned long)image->image_base; + hdr->type_of_loader = 0x21; /* Convert unicode cmdline to ascii */ + options = image->load_options; + load_options_size = image->load_options_size / 2; /* ASCII */ cmdline = 0; s2 = (u16 *)options; @@ -791,18 +827,36 @@ static efi_status_t make_boot_params(struct boot_params *boot_params, hdr->ramdisk_image = 0; hdr->ramdisk_size = 0; - status = handle_ramdisks(image, hdr); - if (status != EFI_SUCCESS) - goto free_cmdline; - - setup_graphics(boot_params); - /* Clear APM BIOS info */ memset(bi, 0, sizeof(*bi)); memset(sdt, 0, sizeof(*sdt)); - memcpy(&efi->efi_loader_signature, EFI_LOADER_SIGNATURE, sizeof(__u32)); + status = handle_ramdisks(image, hdr); + if (status != EFI_SUCCESS) + goto fail2; + + return boot_params; +fail2: + if (options_size) + low_free(options_size, hdr->cmd_line_ptr); +fail: + low_free(0x4000, (unsigned long)boot_params); + return NULL; +} + +static efi_status_t exit_boot(struct boot_params *boot_params, + void *handle) +{ + struct efi_info *efi = &boot_params->efi_info; + struct e820entry *e820_map = &boot_params->e820_map[0]; + struct e820entry *prev = NULL; + unsigned long size, key, desc_size, _size; + efi_memory_desc_t *mem_map; + efi_status_t status; + __u32 desc_version; + u8 nr_entries; + int i; size = sizeof(*mem_map) * 32; @@ -811,7 +865,7 @@ again: _size = size; status = low_alloc(size, 1, (unsigned long *)&mem_map); if (status != EFI_SUCCESS) - goto free_cmdline; + return status; status = efi_call_phys5(sys_table->boottime->get_memory_map, &size, mem_map, &key, &desc_size, &desc_version); @@ -823,6 +877,7 @@ again: if (status != EFI_SUCCESS) goto free_mem_map; + memcpy(&efi->efi_loader_signature, EFI_LOADER_SIGNATURE, sizeof(__u32)); efi->efi_systab = (unsigned long)sys_table; efi->efi_memdesc_size = desc_size; efi->efi_memdesc_version = desc_version; @@ -906,61 +961,13 @@ again: free_mem_map: low_free(_size, (unsigned long)mem_map); -free_cmdline: - if (options_size) - low_free(options_size, hdr->cmd_line_ptr); -fail: return status; } -/* - * On success we return a pointer to a boot_params structure, and NULL - * on failure. - */ -struct boot_params *efi_main(void *handle, efi_system_table_t *_table) +static efi_status_t relocate_kernel(struct setup_header *hdr) { - struct boot_params *boot_params; unsigned long start, nr_pages; - struct desc_ptr *gdt, *idt; - efi_loaded_image_t *image; - struct setup_header *hdr; efi_status_t status; - efi_guid_t proto = LOADED_IMAGE_PROTOCOL_GUID; - struct desc_struct *desc; - - sys_table = _table; - - /* Check if we were booted by the EFI firmware */ - if (sys_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) - goto fail; - - status = efi_call_phys3(sys_table->boottime->handle_protocol, - handle, &proto, (void *)&image); - if (status != EFI_SUCCESS) { - efi_printk("Failed to get handle for LOADED_IMAGE_PROTOCOL\n"); - goto fail; - } - - status = low_alloc(0x4000, 1, (unsigned long *)&boot_params); - if (status != EFI_SUCCESS) { - efi_printk("Failed to alloc lowmem for boot params\n"); - goto fail; - } - - memset(boot_params, 0x0, 0x4000); - - hdr = &boot_params->hdr; - - /* Copy the second sector to boot_params */ - memcpy(&hdr->jump, image->image_base + 512, 512); - - /* - * Fill out some of the header fields ourselves because the - * EFI firmware loader doesn't load the first sector. - */ - hdr->root_flags = 1; - hdr->vid_mode = 0xffff; - hdr->boot_flag = 0xAA55; /* * The EFI firmware loader could have placed the kernel image @@ -978,16 +985,40 @@ struct boot_params *efi_main(void *handle, efi_system_table_t *_table) if (status != EFI_SUCCESS) { status = low_alloc(hdr->init_size, hdr->kernel_alignment, &start); - if (status != EFI_SUCCESS) { + if (status != EFI_SUCCESS) efi_printk("Failed to alloc mem for kernel\n"); - goto fail; - } } + if (status == EFI_SUCCESS) + memcpy((void *)start, (void *)(unsigned long)hdr->code32_start, + hdr->init_size); + + hdr->pref_address = hdr->code32_start; hdr->code32_start = (__u32)start; - hdr->pref_address = (__u64)(unsigned long)image->image_base; - memcpy((void *)start, image->image_base, image->image_size); + return status; +} + +/* + * On success we return a pointer to a boot_params structure, and NULL + * on failure. + */ +struct boot_params *efi_main(void *handle, efi_system_table_t *_table, + struct boot_params *boot_params) +{ + struct desc_ptr *gdt, *idt; + efi_loaded_image_t *image; + struct setup_header *hdr = &boot_params->hdr; + efi_status_t status; + struct desc_struct *desc; + + sys_table = _table; + + /* Check if we were booted by the EFI firmware */ + if (sys_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE) + goto fail; + + setup_graphics(boot_params); status = efi_call_phys3(sys_table->boottime->allocate_pool, EFI_LOADER_DATA, sizeof(*gdt), @@ -1015,7 +1046,18 @@ struct boot_params *efi_main(void *handle, efi_system_table_t *_table) idt->size = 0; idt->address = 0; - status = make_boot_params(boot_params, image, handle); + /* + * If the kernel isn't already loaded at the preferred load + * address, relocate it. + */ + if (hdr->pref_address != hdr->code32_start) { + status = relocate_kernel(hdr); + + if (status != EFI_SUCCESS) + goto fail; + } + + status = exit_boot(boot_params, handle); if (status != EFI_SUCCESS) goto fail; diff --git a/arch/x86/boot/compressed/head_32.S b/arch/x86/boot/compressed/head_32.S index c85e3ac99bba..aa4aaf1b2380 100644 --- a/arch/x86/boot/compressed/head_32.S +++ b/arch/x86/boot/compressed/head_32.S @@ -42,6 +42,16 @@ ENTRY(startup_32) */ add $0x4, %esp + call make_boot_params + cmpl $0, %eax + je 1f + movl 0x4(%esp), %esi + movl (%esp), %ecx + pushl %eax + pushl %esi + pushl %ecx + + .org 0x30,0x90 call efi_main cmpl $0, %eax movl %eax, %esi diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S index 87e03a13d8e3..2c4b171eec33 100644 --- a/arch/x86/boot/compressed/head_64.S +++ b/arch/x86/boot/compressed/head_64.S @@ -209,6 +209,16 @@ ENTRY(startup_64) .org 0x210 mov %rcx, %rdi mov %rdx, %rsi + pushq %rdi + pushq %rsi + call make_boot_params + cmpq $0,%rax + je 1f + mov %rax, %rdx + popq %rsi + popq %rdi + + .org 0x230,0x90 call efi_main movq %rax,%rsi cmpq $0,%rax diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c index 7116dcba0c9e..88f7ff6da404 100644 --- a/arch/x86/boot/compressed/misc.c +++ b/arch/x86/boot/compressed/misc.c @@ -108,8 +108,6 @@ static void error(char *m); * This is set up by the setup-routine at boot-time */ struct boot_params *real_mode; /* Pointer to real-mode data */ -static int quiet; -static int debug; void *memset(void *s, int c, size_t n); void *memcpy(void *dest, const void *src, size_t n); @@ -170,15 +168,11 @@ static void serial_putchar(int ch) outb(ch, early_serial_base + TXR); } -void __putstr(int error, const char *s) +void __putstr(const char *s) { int x, y, pos; char c; -#ifndef CONFIG_X86_VERBOSE_BOOTUP - if (!error) - return; -#endif if (early_serial_base) { const char *str = s; while (*str) { @@ -265,9 +259,9 @@ void *memcpy(void *dest, const void *src, size_t n) static void error(char *x) { - __putstr(1, "\n\n"); - __putstr(1, x); - __putstr(1, "\n\n -- System halted"); + error_putstr("\n\n"); + error_putstr(x); + error_putstr("\n\n -- System halted"); while (1) asm("hlt"); @@ -294,8 +288,7 @@ static void parse_elf(void *output) return; } - if (!quiet) - putstr("Parsing ELF... "); + debug_putstr("Parsing ELF... "); phdrs = malloc(sizeof(*phdrs) * ehdr.e_phnum); if (!phdrs) @@ -332,11 +325,6 @@ asmlinkage void decompress_kernel(void *rmode, memptr heap, { real_mode = rmode; - if (cmdline_find_option_bool("quiet")) - quiet = 1; - if (cmdline_find_option_bool("debug")) - debug = 1; - if (real_mode->screen_info.orig_video_mode == 7) { vidmem = (char *) 0xb0000; vidport = 0x3b4; @@ -349,8 +337,7 @@ asmlinkage void decompress_kernel(void *rmode, memptr heap, cols = real_mode->screen_info.orig_video_cols; console_init(); - if (debug) - putstr("early console in decompress_kernel\n"); + debug_putstr("early console in decompress_kernel\n"); free_mem_ptr = heap; /* Heap */ free_mem_end_ptr = heap + BOOT_HEAP_SIZE; @@ -369,11 +356,9 @@ asmlinkage void decompress_kernel(void *rmode, memptr heap, error("Wrong destination address"); #endif - if (!quiet) - putstr("\nDecompressing Linux... "); + debug_putstr("\nDecompressing Linux... "); decompress(input_data, input_len, NULL, NULL, output, NULL, error); parse_elf(output); - if (!quiet) - putstr("done.\nBooting the kernel.\n"); + debug_putstr("done.\nBooting the kernel.\n"); return; } diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h index 3f19c81a6203..0e6dc0ee0eea 100644 --- a/arch/x86/boot/compressed/misc.h +++ b/arch/x86/boot/compressed/misc.h @@ -24,9 +24,21 @@ /* misc.c */ extern struct boot_params *real_mode; /* Pointer to real-mode data */ -void __putstr(int error, const char *s); -#define putstr(__x) __putstr(0, __x) -#define puts(__x) __putstr(0, __x) +void __putstr(const char *s); +#define error_putstr(__x) __putstr(__x) + +#ifdef CONFIG_X86_VERBOSE_BOOTUP + +#define debug_putstr(__x) __putstr(__x) + +#else + +static inline void debug_putstr(const char *s) +{ } + +#endif + +#ifdef CONFIG_EARLY_PRINTK /* cmdline.c */ int cmdline_find_option(const char *option, char *buffer, int bufsize); @@ -36,4 +48,13 @@ int cmdline_find_option_bool(const char *option); extern int early_serial_base; void console_init(void); +#else + +/* early_serial_console.c */ +static const int early_serial_base; +static inline void console_init(void) +{ } + +#endif + #endif diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S index efe5acfc79c3..b4e15dd6786a 100644 --- a/arch/x86/boot/header.S +++ b/arch/x86/boot/header.S @@ -283,7 +283,7 @@ _start: # Part 2 of the header, from the old setup.S .ascii "HdrS" # header signature - .word 0x020a # header version number (>= 0x0105) + .word 0x020b # header version number (>= 0x0105) # or else old loadlin-1.5 will fail) .globl realmode_swtch realmode_swtch: .word 0, 0 # default_switch, SETUPSEG @@ -401,18 +401,13 @@ pref_address: .quad LOAD_PHYSICAL_ADDR # preferred load addr #define INIT_SIZE VO_INIT_SIZE #endif init_size: .long INIT_SIZE # kernel initialization size +handover_offset: .long 0x30 # offset to the handover + # protocol entry point # End of setup header ##################################################### .section ".entrytext", "ax" start_of_setup: -#ifdef SAFE_RESET_DISK_CONTROLLER -# Reset the disk controller. - movw $0x0000, %ax # Reset disk controller - movb $0x80, %dl # All disks - int $0x13 -#endif - # Force %es = %ds movw %ds, %ax movw %ax, %es diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile index e191ac048b59..e908e5de82d3 100644 --- a/arch/x86/crypto/Makefile +++ b/arch/x86/crypto/Makefile @@ -2,6 +2,9 @@ # Arch-specific CryptoAPI modules. # +obj-$(CONFIG_CRYPTO_ABLK_HELPER_X86) += ablk_helper.o +obj-$(CONFIG_CRYPTO_GLUE_HELPER_X86) += glue_helper.o + obj-$(CONFIG_CRYPTO_AES_586) += aes-i586.o obj-$(CONFIG_CRYPTO_TWOFISH_586) += twofish-i586.o obj-$(CONFIG_CRYPTO_SALSA20_586) += salsa20-i586.o @@ -12,8 +15,10 @@ obj-$(CONFIG_CRYPTO_CAMELLIA_X86_64) += camellia-x86_64.o obj-$(CONFIG_CRYPTO_BLOWFISH_X86_64) += blowfish-x86_64.o obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o obj-$(CONFIG_CRYPTO_TWOFISH_X86_64_3WAY) += twofish-x86_64-3way.o +obj-$(CONFIG_CRYPTO_TWOFISH_AVX_X86_64) += twofish-avx-x86_64.o obj-$(CONFIG_CRYPTO_SALSA20_X86_64) += salsa20-x86_64.o obj-$(CONFIG_CRYPTO_SERPENT_SSE2_X86_64) += serpent-sse2-x86_64.o +obj-$(CONFIG_CRYPTO_SERPENT_AVX_X86_64) += serpent-avx-x86_64.o obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o @@ -30,16 +35,11 @@ camellia-x86_64-y := camellia-x86_64-asm_64.o camellia_glue.o blowfish-x86_64-y := blowfish-x86_64-asm_64.o blowfish_glue.o twofish-x86_64-y := twofish-x86_64-asm_64.o twofish_glue.o twofish-x86_64-3way-y := twofish-x86_64-asm_64-3way.o twofish_glue_3way.o +twofish-avx-x86_64-y := twofish-avx-x86_64-asm_64.o twofish_avx_glue.o salsa20-x86_64-y := salsa20-x86_64-asm_64.o salsa20_glue.o serpent-sse2-x86_64-y := serpent-sse2-x86_64-asm_64.o serpent_sse2_glue.o +serpent-avx-x86_64-y := serpent-avx-x86_64-asm_64.o serpent_avx_glue.o aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o fpu.o - ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o - -# enable AVX support only when $(AS) can actually assemble the instructions -ifeq ($(call as-instr,vpxor %xmm0$(comma)%xmm1$(comma)%xmm2,yes,no),yes) -AFLAGS_sha1_ssse3_asm.o += -DSHA1_ENABLE_AVX_SUPPORT -CFLAGS_sha1_ssse3_glue.o += -DSHA1_ENABLE_AVX_SUPPORT -endif sha1-ssse3-y := sha1_ssse3_asm.o sha1_ssse3_glue.o diff --git a/arch/x86/crypto/ablk_helper.c b/arch/x86/crypto/ablk_helper.c new file mode 100644 index 000000000000..43282fe04a8b --- /dev/null +++ b/arch/x86/crypto/ablk_helper.c @@ -0,0 +1,149 @@ +/* + * Shared async block cipher helpers + * + * Copyright (c) 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> + * + * Based on aesni-intel_glue.c by: + * Copyright (C) 2008, Intel Corp. + * Author: Huang Ying <ying.huang@intel.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA + * + */ + +#include <linux/kernel.h> +#include <linux/crypto.h> +#include <linux/init.h> +#include <linux/module.h> +#include <crypto/algapi.h> +#include <crypto/cryptd.h> +#include <asm/i387.h> +#include <asm/crypto/ablk_helper.h> + +int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int key_len) +{ + struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base; + int err; + + crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); + crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm) + & CRYPTO_TFM_REQ_MASK); + err = crypto_ablkcipher_setkey(child, key, key_len); + crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child) + & CRYPTO_TFM_RES_MASK); + return err; +} +EXPORT_SYMBOL_GPL(ablk_set_key); + +int __ablk_encrypt(struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct blkcipher_desc desc; + + desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); + desc.info = req->info; + desc.flags = 0; + + return crypto_blkcipher_crt(desc.tfm)->encrypt( + &desc, req->dst, req->src, req->nbytes); +} +EXPORT_SYMBOL_GPL(__ablk_encrypt); + +int ablk_encrypt(struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm); + + if (!irq_fpu_usable()) { + struct ablkcipher_request *cryptd_req = + ablkcipher_request_ctx(req); + + memcpy(cryptd_req, req, sizeof(*req)); + ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); + + return crypto_ablkcipher_encrypt(cryptd_req); + } else { + return __ablk_encrypt(req); + } +} +EXPORT_SYMBOL_GPL(ablk_encrypt); + +int ablk_decrypt(struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); + struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm); + + if (!irq_fpu_usable()) { + struct ablkcipher_request *cryptd_req = + ablkcipher_request_ctx(req); + + memcpy(cryptd_req, req, sizeof(*req)); + ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); + + return crypto_ablkcipher_decrypt(cryptd_req); + } else { + struct blkcipher_desc desc; + + desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); + desc.info = req->info; + desc.flags = 0; + + return crypto_blkcipher_crt(desc.tfm)->decrypt( + &desc, req->dst, req->src, req->nbytes); + } +} +EXPORT_SYMBOL_GPL(ablk_decrypt); + +void ablk_exit(struct crypto_tfm *tfm) +{ + struct async_helper_ctx *ctx = crypto_tfm_ctx(tfm); + + cryptd_free_ablkcipher(ctx->cryptd_tfm); +} +EXPORT_SYMBOL_GPL(ablk_exit); + +int ablk_init_common(struct crypto_tfm *tfm, const char *drv_name) +{ + struct async_helper_ctx *ctx = crypto_tfm_ctx(tfm); + struct cryptd_ablkcipher *cryptd_tfm; + + cryptd_tfm = cryptd_alloc_ablkcipher(drv_name, 0, 0); + if (IS_ERR(cryptd_tfm)) + return PTR_ERR(cryptd_tfm); + + ctx->cryptd_tfm = cryptd_tfm; + tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) + + crypto_ablkcipher_reqsize(&cryptd_tfm->base); + + return 0; +} +EXPORT_SYMBOL_GPL(ablk_init_common); + +int ablk_init(struct crypto_tfm *tfm) +{ + char drv_name[CRYPTO_MAX_ALG_NAME]; + + snprintf(drv_name, sizeof(drv_name), "__driver-%s", + crypto_tfm_alg_driver_name(tfm)); + + return ablk_init_common(tfm, drv_name); +} +EXPORT_SYMBOL_GPL(ablk_init); + +MODULE_LICENSE("GPL"); diff --git a/arch/x86/crypto/aes_glue.c b/arch/x86/crypto/aes_glue.c index 8efcf42a9d7e..59b37deb8c8d 100644 --- a/arch/x86/crypto/aes_glue.c +++ b/arch/x86/crypto/aes_glue.c @@ -5,7 +5,7 @@ #include <linux/module.h> #include <crypto/aes.h> -#include <asm/aes.h> +#include <asm/crypto/aes.h> asmlinkage void aes_enc_blk(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in); asmlinkage void aes_dec_blk(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in); diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index ac7f5cd019e8..34fdcff4d2c8 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -30,7 +30,8 @@ #include <crypto/ctr.h> #include <asm/cpu_device_id.h> #include <asm/i387.h> -#include <asm/aes.h> +#include <asm/crypto/aes.h> +#include <asm/crypto/ablk_helper.h> #include <crypto/scatterwalk.h> #include <crypto/internal/aead.h> #include <linux/workqueue.h> @@ -52,10 +53,6 @@ #define HAS_XTS #endif -struct async_aes_ctx { - struct cryptd_ablkcipher *cryptd_tfm; -}; - /* This data is stored at the end of the crypto_tfm struct. * It's a type of per "session" data storage location. * This needs to be 16 byte aligned. @@ -377,87 +374,6 @@ static int ctr_crypt(struct blkcipher_desc *desc, } #endif -static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key, - unsigned int key_len) -{ - struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); - struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base; - int err; - - crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); - crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm) - & CRYPTO_TFM_REQ_MASK); - err = crypto_ablkcipher_setkey(child, key, key_len); - crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child) - & CRYPTO_TFM_RES_MASK); - return err; -} - -static int ablk_encrypt(struct ablkcipher_request *req) -{ - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); - - if (!irq_fpu_usable()) { - struct ablkcipher_request *cryptd_req = - ablkcipher_request_ctx(req); - memcpy(cryptd_req, req, sizeof(*req)); - ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); - return crypto_ablkcipher_encrypt(cryptd_req); - } else { - struct blkcipher_desc desc; - desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); - desc.info = req->info; - desc.flags = 0; - return crypto_blkcipher_crt(desc.tfm)->encrypt( - &desc, req->dst, req->src, req->nbytes); - } -} - -static int ablk_decrypt(struct ablkcipher_request *req) -{ - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); - - if (!irq_fpu_usable()) { - struct ablkcipher_request *cryptd_req = - ablkcipher_request_ctx(req); - memcpy(cryptd_req, req, sizeof(*req)); - ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); - return crypto_ablkcipher_decrypt(cryptd_req); - } else { - struct blkcipher_desc desc; - desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); - desc.info = req->info; - desc.flags = 0; - return crypto_blkcipher_crt(desc.tfm)->decrypt( - &desc, req->dst, req->src, req->nbytes); - } -} - -static void ablk_exit(struct crypto_tfm *tfm) -{ - struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm); - - cryptd_free_ablkcipher(ctx->cryptd_tfm); -} - -static int ablk_init_common(struct crypto_tfm *tfm, const char *drv_name) -{ - struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm); - struct cryptd_ablkcipher *cryptd_tfm; - - cryptd_tfm = cryptd_alloc_ablkcipher(drv_name, 0, 0); - if (IS_ERR(cryptd_tfm)) - return PTR_ERR(cryptd_tfm); - - ctx->cryptd_tfm = cryptd_tfm; - tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) + - crypto_ablkcipher_reqsize(&cryptd_tfm->base); - - return 0; -} - static int ablk_ecb_init(struct crypto_tfm *tfm) { return ablk_init_common(tfm, "__driver-ecb-aes-aesni"); @@ -613,7 +529,7 @@ static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key, struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm); struct aesni_rfc4106_gcm_ctx *child_ctx = aesni_rfc4106_gcm_ctx_get(cryptd_child); - u8 *new_key_mem = NULL; + u8 *new_key_align, *new_key_mem = NULL; if (key_len < 4) { crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); @@ -637,9 +553,9 @@ static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key, if (!new_key_mem) return -ENOMEM; - new_key_mem = PTR_ALIGN(new_key_mem, AESNI_ALIGN); - memcpy(new_key_mem, key, key_len); - key = new_key_mem; + new_key_align = PTR_ALIGN(new_key_mem, AESNI_ALIGN); + memcpy(new_key_align, key, key_len); + key = new_key_align; } if (!irq_fpu_usable()) @@ -968,7 +884,7 @@ static struct crypto_alg aesni_algs[] = { { .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct async_aes_ctx), + .cra_ctxsize = sizeof(struct async_helper_ctx), .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, @@ -989,7 +905,7 @@ static struct crypto_alg aesni_algs[] = { { .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct async_aes_ctx), + .cra_ctxsize = sizeof(struct async_helper_ctx), .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, @@ -1033,7 +949,7 @@ static struct crypto_alg aesni_algs[] = { { .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct async_aes_ctx), + .cra_ctxsize = sizeof(struct async_helper_ctx), .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, @@ -1098,7 +1014,7 @@ static struct crypto_alg aesni_algs[] = { { .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct async_aes_ctx), + .cra_ctxsize = sizeof(struct async_helper_ctx), .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, @@ -1126,7 +1042,7 @@ static struct crypto_alg aesni_algs[] = { { .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct async_aes_ctx), + .cra_ctxsize = sizeof(struct async_helper_ctx), .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, @@ -1150,7 +1066,7 @@ static struct crypto_alg aesni_algs[] = { { .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct async_aes_ctx), + .cra_ctxsize = sizeof(struct async_helper_ctx), .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, @@ -1174,7 +1090,7 @@ static struct crypto_alg aesni_algs[] = { { .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct async_aes_ctx), + .cra_ctxsize = sizeof(struct async_helper_ctx), .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, diff --git a/arch/x86/crypto/camellia_glue.c b/arch/x86/crypto/camellia_glue.c index 3306dc0b139e..eeb2b3b743e9 100644 --- a/arch/x86/crypto/camellia_glue.c +++ b/arch/x86/crypto/camellia_glue.c @@ -5,10 +5,6 @@ * * Camellia parts based on code by: * Copyright (C) 2006 NTT (Nippon Telegraph and Telephone Corporation) - * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: - * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> - * CTR part based on code (crypto/ctr.c) by: - * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com> * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -34,9 +30,9 @@ #include <linux/module.h> #include <linux/types.h> #include <crypto/algapi.h> -#include <crypto/b128ops.h> #include <crypto/lrw.h> #include <crypto/xts.h> +#include <asm/crypto/glue_helper.h> #define CAMELLIA_MIN_KEY_SIZE 16 #define CAMELLIA_MAX_KEY_SIZE 32 @@ -1312,307 +1308,128 @@ static int camellia_setkey(struct crypto_tfm *tfm, const u8 *in_key, &tfm->crt_flags); } -static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk, - void (*fn)(struct camellia_ctx *, u8 *, const u8 *), - void (*fn_2way)(struct camellia_ctx *, u8 *, const u8 *)) +static void camellia_decrypt_cbc_2way(void *ctx, u128 *dst, const u128 *src) { - struct camellia_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - unsigned int bsize = CAMELLIA_BLOCK_SIZE; - unsigned int nbytes; - int err; - - err = blkcipher_walk_virt(desc, walk); - - while ((nbytes = walk->nbytes)) { - u8 *wsrc = walk->src.virt.addr; - u8 *wdst = walk->dst.virt.addr; - - /* Process two block batch */ - if (nbytes >= bsize * 2) { - do { - fn_2way(ctx, wdst, wsrc); - - wsrc += bsize * 2; - wdst += bsize * 2; - nbytes -= bsize * 2; - } while (nbytes >= bsize * 2); - - if (nbytes < bsize) - goto done; - } - - /* Handle leftovers */ - do { - fn(ctx, wdst, wsrc); - - wsrc += bsize; - wdst += bsize; - nbytes -= bsize; - } while (nbytes >= bsize); - -done: - err = blkcipher_walk_done(desc, walk, nbytes); - } - - return err; -} - -static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) -{ - struct blkcipher_walk walk; - - blkcipher_walk_init(&walk, dst, src, nbytes); - return ecb_crypt(desc, &walk, camellia_enc_blk, camellia_enc_blk_2way); -} + u128 iv = *src; -static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) -{ - struct blkcipher_walk walk; - - blkcipher_walk_init(&walk, dst, src, nbytes); - return ecb_crypt(desc, &walk, camellia_dec_blk, camellia_dec_blk_2way); -} + camellia_dec_blk_2way(ctx, (u8 *)dst, (u8 *)src); -static unsigned int __cbc_encrypt(struct blkcipher_desc *desc, - struct blkcipher_walk *walk) -{ - struct camellia_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - unsigned int bsize = CAMELLIA_BLOCK_SIZE; - unsigned int nbytes = walk->nbytes; - u128 *src = (u128 *)walk->src.virt.addr; - u128 *dst = (u128 *)walk->dst.virt.addr; - u128 *iv = (u128 *)walk->iv; - - do { - u128_xor(dst, src, iv); - camellia_enc_blk(ctx, (u8 *)dst, (u8 *)dst); - iv = dst; - - src += 1; - dst += 1; - nbytes -= bsize; - } while (nbytes >= bsize); - - u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv); - return nbytes; + u128_xor(&dst[1], &dst[1], &iv); } -static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) +static void camellia_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv) { - struct blkcipher_walk walk; - int err; + be128 ctrblk; - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); + if (dst != src) + *dst = *src; - while ((nbytes = walk.nbytes)) { - nbytes = __cbc_encrypt(desc, &walk); - err = blkcipher_walk_done(desc, &walk, nbytes); - } + u128_to_be128(&ctrblk, iv); + u128_inc(iv); - return err; + camellia_enc_blk_xor(ctx, (u8 *)dst, (u8 *)&ctrblk); } -static unsigned int __cbc_decrypt(struct blkcipher_desc *desc, - struct blkcipher_walk *walk) +static void camellia_crypt_ctr_2way(void *ctx, u128 *dst, const u128 *src, + u128 *iv) { - struct camellia_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - unsigned int bsize = CAMELLIA_BLOCK_SIZE; - unsigned int nbytes = walk->nbytes; - u128 *src = (u128 *)walk->src.virt.addr; - u128 *dst = (u128 *)walk->dst.virt.addr; - u128 ivs[2 - 1]; - u128 last_iv; + be128 ctrblks[2]; - /* Start of the last block. */ - src += nbytes / bsize - 1; - dst += nbytes / bsize - 1; - - last_iv = *src; - - /* Process two block batch */ - if (nbytes >= bsize * 2) { - do { - nbytes -= bsize * (2 - 1); - src -= 2 - 1; - dst -= 2 - 1; - - ivs[0] = src[0]; - - camellia_dec_blk_2way(ctx, (u8 *)dst, (u8 *)src); - - u128_xor(dst + 1, dst + 1, ivs + 0); - - nbytes -= bsize; - if (nbytes < bsize) - goto done; - - u128_xor(dst, dst, src - 1); - src -= 1; - dst -= 1; - } while (nbytes >= bsize * 2); - - if (nbytes < bsize) - goto done; + if (dst != src) { + dst[0] = src[0]; + dst[1] = src[1]; } - /* Handle leftovers */ - for (;;) { - camellia_dec_blk(ctx, (u8 *)dst, (u8 *)src); - - nbytes -= bsize; - if (nbytes < bsize) - break; + u128_to_be128(&ctrblks[0], iv); + u128_inc(iv); + u128_to_be128(&ctrblks[1], iv); + u128_inc(iv); - u128_xor(dst, dst, src - 1); - src -= 1; - dst -= 1; - } - -done: - u128_xor(dst, dst, (u128 *)walk->iv); - *(u128 *)walk->iv = last_iv; - - return nbytes; + camellia_enc_blk_xor_2way(ctx, (u8 *)dst, (u8 *)ctrblks); } -static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) -{ - struct blkcipher_walk walk; - int err; - - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); +static const struct common_glue_ctx camellia_enc = { + .num_funcs = 2, + .fpu_blocks_limit = -1, + + .funcs = { { + .num_blocks = 2, + .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk_2way) } + }, { + .num_blocks = 1, + .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk) } + } } +}; - while ((nbytes = walk.nbytes)) { - nbytes = __cbc_decrypt(desc, &walk); - err = blkcipher_walk_done(desc, &walk, nbytes); - } +static const struct common_glue_ctx camellia_ctr = { + .num_funcs = 2, + .fpu_blocks_limit = -1, + + .funcs = { { + .num_blocks = 2, + .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr_2way) } + }, { + .num_blocks = 1, + .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr) } + } } +}; - return err; -} +static const struct common_glue_ctx camellia_dec = { + .num_funcs = 2, + .fpu_blocks_limit = -1, + + .funcs = { { + .num_blocks = 2, + .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk_2way) } + }, { + .num_blocks = 1, + .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk) } + } } +}; -static inline void u128_to_be128(be128 *dst, const u128 *src) -{ - dst->a = cpu_to_be64(src->a); - dst->b = cpu_to_be64(src->b); -} +static const struct common_glue_ctx camellia_dec_cbc = { + .num_funcs = 2, + .fpu_blocks_limit = -1, + + .funcs = { { + .num_blocks = 2, + .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_decrypt_cbc_2way) } + }, { + .num_blocks = 1, + .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_dec_blk) } + } } +}; -static inline void be128_to_u128(u128 *dst, const be128 *src) +static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) { - dst->a = be64_to_cpu(src->a); - dst->b = be64_to_cpu(src->b); + return glue_ecb_crypt_128bit(&camellia_enc, desc, dst, src, nbytes); } -static inline void u128_inc(u128 *i) +static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) { - i->b++; - if (!i->b) - i->a++; + return glue_ecb_crypt_128bit(&camellia_dec, desc, dst, src, nbytes); } -static void ctr_crypt_final(struct blkcipher_desc *desc, - struct blkcipher_walk *walk) +static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) { - struct camellia_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - u8 keystream[CAMELLIA_BLOCK_SIZE]; - u8 *src = walk->src.virt.addr; - u8 *dst = walk->dst.virt.addr; - unsigned int nbytes = walk->nbytes; - u128 ctrblk; - - memcpy(keystream, src, nbytes); - camellia_enc_blk_xor(ctx, keystream, walk->iv); - memcpy(dst, keystream, nbytes); - - be128_to_u128(&ctrblk, (be128 *)walk->iv); - u128_inc(&ctrblk); - u128_to_be128((be128 *)walk->iv, &ctrblk); + return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(camellia_enc_blk), desc, + dst, src, nbytes); } -static unsigned int __ctr_crypt(struct blkcipher_desc *desc, - struct blkcipher_walk *walk) +static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) { - struct camellia_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - unsigned int bsize = CAMELLIA_BLOCK_SIZE; - unsigned int nbytes = walk->nbytes; - u128 *src = (u128 *)walk->src.virt.addr; - u128 *dst = (u128 *)walk->dst.virt.addr; - u128 ctrblk; - be128 ctrblocks[2]; - - be128_to_u128(&ctrblk, (be128 *)walk->iv); - - /* Process two block batch */ - if (nbytes >= bsize * 2) { - do { - if (dst != src) { - dst[0] = src[0]; - dst[1] = src[1]; - } - - /* create ctrblks for parallel encrypt */ - u128_to_be128(&ctrblocks[0], &ctrblk); - u128_inc(&ctrblk); - u128_to_be128(&ctrblocks[1], &ctrblk); - u128_inc(&ctrblk); - - camellia_enc_blk_xor_2way(ctx, (u8 *)dst, - (u8 *)ctrblocks); - - src += 2; - dst += 2; - nbytes -= bsize * 2; - } while (nbytes >= bsize * 2); - - if (nbytes < bsize) - goto done; - } - - /* Handle leftovers */ - do { - if (dst != src) - *dst = *src; - - u128_to_be128(&ctrblocks[0], &ctrblk); - u128_inc(&ctrblk); - - camellia_enc_blk_xor(ctx, (u8 *)dst, (u8 *)ctrblocks); - - src += 1; - dst += 1; - nbytes -= bsize; - } while (nbytes >= bsize); - -done: - u128_to_be128((be128 *)walk->iv, &ctrblk); - return nbytes; + return glue_cbc_decrypt_128bit(&camellia_dec_cbc, desc, dst, src, + nbytes); } static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { - struct blkcipher_walk walk; - int err; - - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt_block(desc, &walk, CAMELLIA_BLOCK_SIZE); - - while ((nbytes = walk.nbytes) >= CAMELLIA_BLOCK_SIZE) { - nbytes = __ctr_crypt(desc, &walk); - err = blkcipher_walk_done(desc, &walk, nbytes); - } - - if (walk.nbytes) { - ctr_crypt_final(desc, &walk); - err = blkcipher_walk_done(desc, &walk, 0); - } - - return err; + return glue_ctr_crypt_128bit(&camellia_ctr, desc, dst, src, nbytes); } static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) diff --git a/arch/x86/crypto/glue_helper.c b/arch/x86/crypto/glue_helper.c new file mode 100644 index 000000000000..4854f0f31e4f --- /dev/null +++ b/arch/x86/crypto/glue_helper.c @@ -0,0 +1,307 @@ +/* + * Shared glue code for 128bit block ciphers + * + * Copyright (c) 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> + * + * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: + * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> + * CTR part based on code (crypto/ctr.c) by: + * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA + * + */ + +#include <linux/module.h> +#include <crypto/b128ops.h> +#include <crypto/lrw.h> +#include <crypto/xts.h> +#include <asm/crypto/glue_helper.h> +#include <crypto/scatterwalk.h> + +static int __glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx, + struct blkcipher_desc *desc, + struct blkcipher_walk *walk) +{ + void *ctx = crypto_blkcipher_ctx(desc->tfm); + const unsigned int bsize = 128 / 8; + unsigned int nbytes, i, func_bytes; + bool fpu_enabled = false; + int err; + + err = blkcipher_walk_virt(desc, walk); + + while ((nbytes = walk->nbytes)) { + u8 *wsrc = walk->src.virt.addr; + u8 *wdst = walk->dst.virt.addr; + + fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, + desc, fpu_enabled, nbytes); + + for (i = 0; i < gctx->num_funcs; i++) { + func_bytes = bsize * gctx->funcs[i].num_blocks; + + /* Process multi-block batch */ + if (nbytes >= func_bytes) { + do { + gctx->funcs[i].fn_u.ecb(ctx, wdst, + wsrc); + + wsrc += func_bytes; + wdst += func_bytes; + nbytes -= func_bytes; + } while (nbytes >= func_bytes); + + if (nbytes < bsize) + goto done; + } + } + +done: + err = blkcipher_walk_done(desc, walk, nbytes); + } + + glue_fpu_end(fpu_enabled); + return err; +} + +int glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx, + struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct blkcipher_walk walk; + + blkcipher_walk_init(&walk, dst, src, nbytes); + return __glue_ecb_crypt_128bit(gctx, desc, &walk); +} +EXPORT_SYMBOL_GPL(glue_ecb_crypt_128bit); + +static unsigned int __glue_cbc_encrypt_128bit(const common_glue_func_t fn, + struct blkcipher_desc *desc, + struct blkcipher_walk *walk) +{ + void *ctx = crypto_blkcipher_ctx(desc->tfm); + const unsigned int bsize = 128 / 8; + unsigned int nbytes = walk->nbytes; + u128 *src = (u128 *)walk->src.virt.addr; + u128 *dst = (u128 *)walk->dst.virt.addr; + u128 *iv = (u128 *)walk->iv; + + do { + u128_xor(dst, src, iv); + fn(ctx, (u8 *)dst, (u8 *)dst); + iv = dst; + + src += 1; + dst += 1; + nbytes -= bsize; + } while (nbytes >= bsize); + + u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv); + return nbytes; +} + +int glue_cbc_encrypt_128bit(const common_glue_func_t fn, + struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + + while ((nbytes = walk.nbytes)) { + nbytes = __glue_cbc_encrypt_128bit(fn, desc, &walk); + err = blkcipher_walk_done(desc, &walk, nbytes); + } + + return err; +} +EXPORT_SYMBOL_GPL(glue_cbc_encrypt_128bit); + +static unsigned int +__glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx, + struct blkcipher_desc *desc, + struct blkcipher_walk *walk) +{ + void *ctx = crypto_blkcipher_ctx(desc->tfm); + const unsigned int bsize = 128 / 8; + unsigned int nbytes = walk->nbytes; + u128 *src = (u128 *)walk->src.virt.addr; + u128 *dst = (u128 *)walk->dst.virt.addr; + u128 last_iv; + unsigned int num_blocks, func_bytes; + unsigned int i; + + /* Start of the last block. */ + src += nbytes / bsize - 1; + dst += nbytes / bsize - 1; + + last_iv = *src; + + for (i = 0; i < gctx->num_funcs; i++) { + num_blocks = gctx->funcs[i].num_blocks; + func_bytes = bsize * num_blocks; + + /* Process multi-block batch */ + if (nbytes >= func_bytes) { + do { + nbytes -= func_bytes - bsize; + src -= num_blocks - 1; + dst -= num_blocks - 1; + + gctx->funcs[i].fn_u.cbc(ctx, dst, src); + + nbytes -= bsize; + if (nbytes < bsize) + goto done; + + u128_xor(dst, dst, src - 1); + src -= 1; + dst -= 1; + } while (nbytes >= func_bytes); + + if (nbytes < bsize) + goto done; + } + } + +done: + u128_xor(dst, dst, (u128 *)walk->iv); + *(u128 *)walk->iv = last_iv; + + return nbytes; +} + +int glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx, + struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + const unsigned int bsize = 128 / 8; + bool fpu_enabled = false; + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt(desc, &walk); + + while ((nbytes = walk.nbytes)) { + fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, + desc, fpu_enabled, nbytes); + nbytes = __glue_cbc_decrypt_128bit(gctx, desc, &walk); + err = blkcipher_walk_done(desc, &walk, nbytes); + } + + glue_fpu_end(fpu_enabled); + return err; +} +EXPORT_SYMBOL_GPL(glue_cbc_decrypt_128bit); + +static void glue_ctr_crypt_final_128bit(const common_glue_ctr_func_t fn_ctr, + struct blkcipher_desc *desc, + struct blkcipher_walk *walk) +{ + void *ctx = crypto_blkcipher_ctx(desc->tfm); + u8 *src = (u8 *)walk->src.virt.addr; + u8 *dst = (u8 *)walk->dst.virt.addr; + unsigned int nbytes = walk->nbytes; + u128 ctrblk; + u128 tmp; + + be128_to_u128(&ctrblk, (be128 *)walk->iv); + + memcpy(&tmp, src, nbytes); + fn_ctr(ctx, &tmp, &tmp, &ctrblk); + memcpy(dst, &tmp, nbytes); + + u128_to_be128((be128 *)walk->iv, &ctrblk); +} +EXPORT_SYMBOL_GPL(glue_ctr_crypt_final_128bit); + +static unsigned int __glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx, + struct blkcipher_desc *desc, + struct blkcipher_walk *walk) +{ + const unsigned int bsize = 128 / 8; + void *ctx = crypto_blkcipher_ctx(desc->tfm); + unsigned int nbytes = walk->nbytes; + u128 *src = (u128 *)walk->src.virt.addr; + u128 *dst = (u128 *)walk->dst.virt.addr; + u128 ctrblk; + unsigned int num_blocks, func_bytes; + unsigned int i; + + be128_to_u128(&ctrblk, (be128 *)walk->iv); + + /* Process multi-block batch */ + for (i = 0; i < gctx->num_funcs; i++) { + num_blocks = gctx->funcs[i].num_blocks; + func_bytes = bsize * num_blocks; + + if (nbytes >= func_bytes) { + do { + gctx->funcs[i].fn_u.ctr(ctx, dst, src, &ctrblk); + + src += num_blocks; + dst += num_blocks; + nbytes -= func_bytes; + } while (nbytes >= func_bytes); + + if (nbytes < bsize) + goto done; + } + } + +done: + u128_to_be128((be128 *)walk->iv, &ctrblk); + return nbytes; +} + +int glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx, + struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + const unsigned int bsize = 128 / 8; + bool fpu_enabled = false; + struct blkcipher_walk walk; + int err; + + blkcipher_walk_init(&walk, dst, src, nbytes); + err = blkcipher_walk_virt_block(desc, &walk, bsize); + + while ((nbytes = walk.nbytes) >= bsize) { + fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit, + desc, fpu_enabled, nbytes); + nbytes = __glue_ctr_crypt_128bit(gctx, desc, &walk); + err = blkcipher_walk_done(desc, &walk, nbytes); + } + + glue_fpu_end(fpu_enabled); + + if (walk.nbytes) { + glue_ctr_crypt_final_128bit( + gctx->funcs[gctx->num_funcs - 1].fn_u.ctr, desc, &walk); + err = blkcipher_walk_done(desc, &walk, 0); + } + + return err; +} +EXPORT_SYMBOL_GPL(glue_ctr_crypt_128bit); + +MODULE_LICENSE("GPL"); diff --git a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S new file mode 100644 index 000000000000..504106bf04a2 --- /dev/null +++ b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S @@ -0,0 +1,704 @@ +/* + * Serpent Cipher 8-way parallel algorithm (x86_64/AVX) + * + * Copyright (C) 2012 Johannes Goetzfried + * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> + * + * Based on arch/x86/crypto/serpent-sse2-x86_64-asm_64.S by + * Copyright (C) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA + * + */ + +.file "serpent-avx-x86_64-asm_64.S" +.text + +#define CTX %rdi + +/********************************************************************** + 8-way AVX serpent + **********************************************************************/ +#define RA1 %xmm0 +#define RB1 %xmm1 +#define RC1 %xmm2 +#define RD1 %xmm3 +#define RE1 %xmm4 + +#define tp %xmm5 + +#define RA2 %xmm6 +#define RB2 %xmm7 +#define RC2 %xmm8 +#define RD2 %xmm9 +#define RE2 %xmm10 + +#define RNOT %xmm11 + +#define RK0 %xmm12 +#define RK1 %xmm13 +#define RK2 %xmm14 +#define RK3 %xmm15 + + +#define S0_1(x0, x1, x2, x3, x4) \ + vpor x0, x3, tp; \ + vpxor x3, x0, x0; \ + vpxor x2, x3, x4; \ + vpxor RNOT, x4, x4; \ + vpxor x1, tp, x3; \ + vpand x0, x1, x1; \ + vpxor x4, x1, x1; \ + vpxor x0, x2, x2; +#define S0_2(x0, x1, x2, x3, x4) \ + vpxor x3, x0, x0; \ + vpor x0, x4, x4; \ + vpxor x2, x0, x0; \ + vpand x1, x2, x2; \ + vpxor x2, x3, x3; \ + vpxor RNOT, x1, x1; \ + vpxor x4, x2, x2; \ + vpxor x2, x1, x1; + +#define S1_1(x0, x1, x2, x3, x4) \ + vpxor x0, x1, tp; \ + vpxor x3, x0, x0; \ + vpxor RNOT, x3, x3; \ + vpand tp, x1, x4; \ + vpor tp, x0, x0; \ + vpxor x2, x3, x3; \ + vpxor x3, x0, x0; \ + vpxor x3, tp, x1; +#define S1_2(x0, x1, x2, x3, x4) \ + vpxor x4, x3, x3; \ + vpor x4, x1, x1; \ + vpxor x2, x4, x4; \ + vpand x0, x2, x2; \ + vpxor x1, x2, x2; \ + vpor x0, x1, x1; \ + vpxor RNOT, x0, x0; \ + vpxor x2, x0, x0; \ + vpxor x1, x4, x4; + +#define S2_1(x0, x1, x2, x3, x4) \ + vpxor RNOT, x3, x3; \ + vpxor x0, x1, x1; \ + vpand x2, x0, tp; \ + vpxor x3, tp, tp; \ + vpor x0, x3, x3; \ + vpxor x1, x2, x2; \ + vpxor x1, x3, x3; \ + vpand tp, x1, x1; +#define S2_2(x0, x1, x2, x3, x4) \ + vpxor x2, tp, tp; \ + vpand x3, x2, x2; \ + vpor x1, x3, x3; \ + vpxor RNOT, tp, tp; \ + vpxor tp, x3, x3; \ + vpxor tp, x0, x4; \ + vpxor x2, tp, x0; \ + vpor x2, x1, x1; + +#define S3_1(x0, x1, x2, x3, x4) \ + vpxor x3, x1, tp; \ + vpor x0, x3, x3; \ + vpand x0, x1, x4; \ + vpxor x2, x0, x0; \ + vpxor tp, x2, x2; \ + vpand x3, tp, x1; \ + vpxor x3, x2, x2; \ + vpor x4, x0, x0; \ + vpxor x3, x4, x4; +#define S3_2(x0, x1, x2, x3, x4) \ + vpxor x0, x1, x1; \ + vpand x3, x0, x0; \ + vpand x4, x3, x3; \ + vpxor x2, x3, x3; \ + vpor x1, x4, x4; \ + vpand x1, x2, x2; \ + vpxor x3, x4, x4; \ + vpxor x3, x0, x0; \ + vpxor x2, x3, x3; + +#define S4_1(x0, x1, x2, x3, x4) \ + vpand x0, x3, tp; \ + vpxor x3, x0, x0; \ + vpxor x2, tp, tp; \ + vpor x3, x2, x2; \ + vpxor x1, x0, x0; \ + vpxor tp, x3, x4; \ + vpor x0, x2, x2; \ + vpxor x1, x2, x2; +#define S4_2(x0, x1, x2, x3, x4) \ + vpand x0, x1, x1; \ + vpxor x4, x1, x1; \ + vpand x2, x4, x4; \ + vpxor tp, x2, x2; \ + vpxor x0, x4, x4; \ + vpor x1, tp, x3; \ + vpxor RNOT, x1, x1; \ + vpxor x0, x3, x3; + +#define S5_1(x0, x1, x2, x3, x4) \ + vpor x0, x1, tp; \ + vpxor tp, x2, x2; \ + vpxor RNOT, x3, x3; \ + vpxor x0, x1, x4; \ + vpxor x2, x0, x0; \ + vpand x4, tp, x1; \ + vpor x3, x4, x4; \ + vpxor x0, x4, x4; +#define S5_2(x0, x1, x2, x3, x4) \ + vpand x3, x0, x0; \ + vpxor x3, x1, x1; \ + vpxor x2, x3, x3; \ + vpxor x1, x0, x0; \ + vpand x4, x2, x2; \ + vpxor x2, x1, x1; \ + vpand x0, x2, x2; \ + vpxor x2, x3, x3; + +#define S6_1(x0, x1, x2, x3, x4) \ + vpxor x0, x3, x3; \ + vpxor x2, x1, tp; \ + vpxor x0, x2, x2; \ + vpand x3, x0, x0; \ + vpor x3, tp, tp; \ + vpxor RNOT, x1, x4; \ + vpxor tp, x0, x0; \ + vpxor x2, tp, x1; +#define S6_2(x0, x1, x2, x3, x4) \ + vpxor x4, x3, x3; \ + vpxor x0, x4, x4; \ + vpand x0, x2, x2; \ + vpxor x1, x4, x4; \ + vpxor x3, x2, x2; \ + vpand x1, x3, x3; \ + vpxor x0, x3, x3; \ + vpxor x2, x1, x1; + +#define S7_1(x0, x1, x2, x3, x4) \ + vpxor RNOT, x1, tp; \ + vpxor RNOT, x0, x0; \ + vpand x2, tp, x1; \ + vpxor x3, x1, x1; \ + vpor tp, x3, x3; \ + vpxor x2, tp, x4; \ + vpxor x3, x2, x2; \ + vpxor x0, x3, x3; \ + vpor x1, x0, x0; +#define S7_2(x0, x1, x2, x3, x4) \ + vpand x0, x2, x2; \ + vpxor x4, x0, x0; \ + vpxor x3, x4, x4; \ + vpand x0, x3, x3; \ + vpxor x1, x4, x4; \ + vpxor x4, x2, x2; \ + vpxor x1, x3, x3; \ + vpor x0, x4, x4; \ + vpxor x1, x4, x4; + +#define SI0_1(x0, x1, x2, x3, x4) \ + vpxor x0, x1, x1; \ + vpor x1, x3, tp; \ + vpxor x1, x3, x4; \ + vpxor RNOT, x0, x0; \ + vpxor tp, x2, x2; \ + vpxor x0, tp, x3; \ + vpand x1, x0, x0; \ + vpxor x2, x0, x0; +#define SI0_2(x0, x1, x2, x3, x4) \ + vpand x3, x2, x2; \ + vpxor x4, x3, x3; \ + vpxor x3, x2, x2; \ + vpxor x3, x1, x1; \ + vpand x0, x3, x3; \ + vpxor x0, x1, x1; \ + vpxor x2, x0, x0; \ + vpxor x3, x4, x4; + +#define SI1_1(x0, x1, x2, x3, x4) \ + vpxor x3, x1, x1; \ + vpxor x2, x0, tp; \ + vpxor RNOT, x2, x2; \ + vpor x1, x0, x4; \ + vpxor x3, x4, x4; \ + vpand x1, x3, x3; \ + vpxor x2, x1, x1; \ + vpand x4, x2, x2; +#define SI1_2(x0, x1, x2, x3, x4) \ + vpxor x1, x4, x4; \ + vpor x3, x1, x1; \ + vpxor tp, x3, x3; \ + vpxor tp, x2, x2; \ + vpor x4, tp, x0; \ + vpxor x4, x2, x2; \ + vpxor x0, x1, x1; \ + vpxor x1, x4, x4; + +#define SI2_1(x0, x1, x2, x3, x4) \ + vpxor x1, x2, x2; \ + vpxor RNOT, x3, tp; \ + vpor x2, tp, tp; \ + vpxor x3, x2, x2; \ + vpxor x0, x3, x4; \ + vpxor x1, tp, x3; \ + vpor x2, x1, x1; \ + vpxor x0, x2, x2; +#define SI2_2(x0, x1, x2, x3, x4) \ + vpxor x4, x1, x1; \ + vpor x3, x4, x4; \ + vpxor x3, x2, x2; \ + vpxor x2, x4, x4; \ + vpand x1, x2, x2; \ + vpxor x3, x2, x2; \ + vpxor x4, x3, x3; \ + vpxor x0, x4, x4; + +#define SI3_1(x0, x1, x2, x3, x4) \ + vpxor x1, x2, x2; \ + vpand x2, x1, tp; \ + vpxor x0, tp, tp; \ + vpor x1, x0, x0; \ + vpxor x3, x1, x4; \ + vpxor x3, x0, x0; \ + vpor tp, x3, x3; \ + vpxor x2, tp, x1; +#define SI3_2(x0, x1, x2, x3, x4) \ + vpxor x3, x1, x1; \ + vpxor x2, x0, x0; \ + vpxor x3, x2, x2; \ + vpand x1, x3, x3; \ + vpxor x0, x1, x1; \ + vpand x2, x0, x0; \ + vpxor x3, x4, x4; \ + vpxor x0, x3, x3; \ + vpxor x1, x0, x0; + +#define SI4_1(x0, x1, x2, x3, x4) \ + vpxor x3, x2, x2; \ + vpand x1, x0, tp; \ + vpxor x2, tp, tp; \ + vpor x3, x2, x2; \ + vpxor RNOT, x0, x4; \ + vpxor tp, x1, x1; \ + vpxor x2, tp, x0; \ + vpand x4, x2, x2; +#define SI4_2(x0, x1, x2, x3, x4) \ + vpxor x0, x2, x2; \ + vpor x4, x0, x0; \ + vpxor x3, x0, x0; \ + vpand x2, x3, x3; \ + vpxor x3, x4, x4; \ + vpxor x1, x3, x3; \ + vpand x0, x1, x1; \ + vpxor x1, x4, x4; \ + vpxor x3, x0, x0; + +#define SI5_1(x0, x1, x2, x3, x4) \ + vpor x2, x1, tp; \ + vpxor x1, x2, x2; \ + vpxor x3, tp, tp; \ + vpand x1, x3, x3; \ + vpxor x3, x2, x2; \ + vpor x0, x3, x3; \ + vpxor RNOT, x0, x0; \ + vpxor x2, x3, x3; \ + vpor x0, x2, x2; +#define SI5_2(x0, x1, x2, x3, x4) \ + vpxor tp, x1, x4; \ + vpxor x4, x2, x2; \ + vpand x0, x4, x4; \ + vpxor tp, x0, x0; \ + vpxor x3, tp, x1; \ + vpand x2, x0, x0; \ + vpxor x3, x2, x2; \ + vpxor x2, x0, x0; \ + vpxor x4, x2, x2; \ + vpxor x3, x4, x4; + +#define SI6_1(x0, x1, x2, x3, x4) \ + vpxor x2, x0, x0; \ + vpand x3, x0, tp; \ + vpxor x3, x2, x2; \ + vpxor x2, tp, tp; \ + vpxor x1, x3, x3; \ + vpor x0, x2, x2; \ + vpxor x3, x2, x2; \ + vpand tp, x3, x3; +#define SI6_2(x0, x1, x2, x3, x4) \ + vpxor RNOT, tp, tp; \ + vpxor x1, x3, x3; \ + vpand x2, x1, x1; \ + vpxor tp, x0, x4; \ + vpxor x4, x3, x3; \ + vpxor x2, x4, x4; \ + vpxor x1, tp, x0; \ + vpxor x0, x2, x2; + +#define SI7_1(x0, x1, x2, x3, x4) \ + vpand x0, x3, tp; \ + vpxor x2, x0, x0; \ + vpor x3, x2, x2; \ + vpxor x1, x3, x4; \ + vpxor RNOT, x0, x0; \ + vpor tp, x1, x1; \ + vpxor x0, x4, x4; \ + vpand x2, x0, x0; \ + vpxor x1, x0, x0; +#define SI7_2(x0, x1, x2, x3, x4) \ + vpand x2, x1, x1; \ + vpxor x2, tp, x3; \ + vpxor x3, x4, x4; \ + vpand x3, x2, x2; \ + vpor x0, x3, x3; \ + vpxor x4, x1, x1; \ + vpxor x4, x3, x3; \ + vpand x0, x4, x4; \ + vpxor x2, x4, x4; + +#define get_key(i, j, t) \ + vbroadcastss (4*(i)+(j))*4(CTX), t; + +#define K2(x0, x1, x2, x3, x4, i) \ + get_key(i, 0, RK0); \ + get_key(i, 1, RK1); \ + get_key(i, 2, RK2); \ + get_key(i, 3, RK3); \ + vpxor RK0, x0 ## 1, x0 ## 1; \ + vpxor RK1, x1 ## 1, x1 ## 1; \ + vpxor RK2, x2 ## 1, x2 ## 1; \ + vpxor RK3, x3 ## 1, x3 ## 1; \ + vpxor RK0, x0 ## 2, x0 ## 2; \ + vpxor RK1, x1 ## 2, x1 ## 2; \ + vpxor RK2, x2 ## 2, x2 ## 2; \ + vpxor RK3, x3 ## 2, x3 ## 2; + +#define LK2(x0, x1, x2, x3, x4, i) \ + vpslld $13, x0 ## 1, x4 ## 1; \ + vpsrld $(32 - 13), x0 ## 1, x0 ## 1; \ + vpor x4 ## 1, x0 ## 1, x0 ## 1; \ + vpxor x0 ## 1, x1 ## 1, x1 ## 1; \ + vpslld $3, x2 ## 1, x4 ## 1; \ + vpsrld $(32 - 3), x2 ## 1, x2 ## 1; \ + vpor x4 ## 1, x2 ## 1, x2 ## 1; \ + vpxor x2 ## 1, x1 ## 1, x1 ## 1; \ + vpslld $13, x0 ## 2, x4 ## 2; \ + vpsrld $(32 - 13), x0 ## 2, x0 ## 2; \ + vpor x4 ## 2, x0 ## 2, x0 ## 2; \ + vpxor x0 ## 2, x1 ## 2, x1 ## 2; \ + vpslld $3, x2 ## 2, x4 ## 2; \ + vpsrld $(32 - 3), x2 ## 2, x2 ## 2; \ + vpor x4 ## 2, x2 ## 2, x2 ## 2; \ + vpxor x2 ## 2, x1 ## 2, x1 ## 2; \ + vpslld $1, x1 ## 1, x4 ## 1; \ + vpsrld $(32 - 1), x1 ## 1, x1 ## 1; \ + vpor x4 ## 1, x1 ## 1, x1 ## 1; \ + vpslld $3, x0 ## 1, x4 ## 1; \ + vpxor x2 ## 1, x3 ## 1, x3 ## 1; \ + vpxor x4 ## 1, x3 ## 1, x3 ## 1; \ + get_key(i, 1, RK1); \ + vpslld $1, x1 ## 2, x4 ## 2; \ + vpsrld $(32 - 1), x1 ## 2, x1 ## 2; \ + vpor x4 ## 2, x1 ## 2, x1 ## 2; \ + vpslld $3, x0 ## 2, x4 ## 2; \ + vpxor x2 ## 2, x3 ## 2, x3 ## 2; \ + vpxor x4 ## 2, x3 ## 2, x3 ## 2; \ + get_key(i, 3, RK3); \ + vpslld $7, x3 ## 1, x4 ## 1; \ + vpsrld $(32 - 7), x3 ## 1, x3 ## 1; \ + vpor x4 ## 1, x3 ## 1, x3 ## 1; \ + vpslld $7, x1 ## 1, x4 ## 1; \ + vpxor x1 ## 1, x0 ## 1, x0 ## 1; \ + vpxor x3 ## 1, x0 ## 1, x0 ## 1; \ + vpxor x3 ## 1, x2 ## 1, x2 ## 1; \ + vpxor x4 ## 1, x2 ## 1, x2 ## 1; \ + get_key(i, 0, RK0); \ + vpslld $7, x3 ## 2, x4 ## 2; \ + vpsrld $(32 - 7), x3 ## 2, x3 ## 2; \ + vpor x4 ## 2, x3 ## 2, x3 ## 2; \ + vpslld $7, x1 ## 2, x4 ## 2; \ + vpxor x1 ## 2, x0 ## 2, x0 ## 2; \ + vpxor x3 ## 2, x0 ## 2, x0 ## 2; \ + vpxor x3 ## 2, x2 ## 2, x2 ## 2; \ + vpxor x4 ## 2, x2 ## 2, x2 ## 2; \ + get_key(i, 2, RK2); \ + vpxor RK1, x1 ## 1, x1 ## 1; \ + vpxor RK3, x3 ## 1, x3 ## 1; \ + vpslld $5, x0 ## 1, x4 ## 1; \ + vpsrld $(32 - 5), x0 ## 1, x0 ## 1; \ + vpor x4 ## 1, x0 ## 1, x0 ## 1; \ + vpslld $22, x2 ## 1, x4 ## 1; \ + vpsrld $(32 - 22), x2 ## 1, x2 ## 1; \ + vpor x4 ## 1, x2 ## 1, x2 ## 1; \ + vpxor RK0, x0 ## 1, x0 ## 1; \ + vpxor RK2, x2 ## 1, x2 ## 1; \ + vpxor RK1, x1 ## 2, x1 ## 2; \ + vpxor RK3, x3 ## 2, x3 ## 2; \ + vpslld $5, x0 ## 2, x4 ## 2; \ + vpsrld $(32 - 5), x0 ## 2, x0 ## 2; \ + vpor x4 ## 2, x0 ## 2, x0 ## 2; \ + vpslld $22, x2 ## 2, x4 ## 2; \ + vpsrld $(32 - 22), x2 ## 2, x2 ## 2; \ + vpor x4 ## 2, x2 ## 2, x2 ## 2; \ + vpxor RK0, x0 ## 2, x0 ## 2; \ + vpxor RK2, x2 ## 2, x2 ## 2; + +#define KL2(x0, x1, x2, x3, x4, i) \ + vpxor RK0, x0 ## 1, x0 ## 1; \ + vpxor RK2, x2 ## 1, x2 ## 1; \ + vpsrld $5, x0 ## 1, x4 ## 1; \ + vpslld $(32 - 5), x0 ## 1, x0 ## 1; \ + vpor x4 ## 1, x0 ## 1, x0 ## 1; \ + vpxor RK3, x3 ## 1, x3 ## 1; \ + vpxor RK1, x1 ## 1, x1 ## 1; \ + vpsrld $22, x2 ## 1, x4 ## 1; \ + vpslld $(32 - 22), x2 ## 1, x2 ## 1; \ + vpor x4 ## 1, x2 ## 1, x2 ## 1; \ + vpxor x3 ## 1, x2 ## 1, x2 ## 1; \ + vpxor RK0, x0 ## 2, x0 ## 2; \ + vpxor RK2, x2 ## 2, x2 ## 2; \ + vpsrld $5, x0 ## 2, x4 ## 2; \ + vpslld $(32 - 5), x0 ## 2, x0 ## 2; \ + vpor x4 ## 2, x0 ## 2, x0 ## 2; \ + vpxor RK3, x3 ## 2, x3 ## 2; \ + vpxor RK1, x1 ## 2, x1 ## 2; \ + vpsrld $22, x2 ## 2, x4 ## 2; \ + vpslld $(32 - 22), x2 ## 2, x2 ## 2; \ + vpor x4 ## 2, x2 ## 2, x2 ## 2; \ + vpxor x3 ## 2, x2 ## 2, x2 ## 2; \ + vpxor x3 ## 1, x0 ## 1, x0 ## 1; \ + vpslld $7, x1 ## 1, x4 ## 1; \ + vpxor x1 ## 1, x0 ## 1, x0 ## 1; \ + vpxor x4 ## 1, x2 ## 1, x2 ## 1; \ + vpsrld $1, x1 ## 1, x4 ## 1; \ + vpslld $(32 - 1), x1 ## 1, x1 ## 1; \ + vpor x4 ## 1, x1 ## 1, x1 ## 1; \ + vpxor x3 ## 2, x0 ## 2, x0 ## 2; \ + vpslld $7, x1 ## 2, x4 ## 2; \ + vpxor x1 ## 2, x0 ## 2, x0 ## 2; \ + vpxor x4 ## 2, x2 ## 2, x2 ## 2; \ + vpsrld $1, x1 ## 2, x4 ## 2; \ + vpslld $(32 - 1), x1 ## 2, x1 ## 2; \ + vpor x4 ## 2, x1 ## 2, x1 ## 2; \ + vpsrld $7, x3 ## 1, x4 ## 1; \ + vpslld $(32 - 7), x3 ## 1, x3 ## 1; \ + vpor x4 ## 1, x3 ## 1, x3 ## 1; \ + vpxor x0 ## 1, x1 ## 1, x1 ## 1; \ + vpslld $3, x0 ## 1, x4 ## 1; \ + vpxor x4 ## 1, x3 ## 1, x3 ## 1; \ + vpsrld $7, x3 ## 2, x4 ## 2; \ + vpslld $(32 - 7), x3 ## 2, x3 ## 2; \ + vpor x4 ## 2, x3 ## 2, x3 ## 2; \ + vpxor x0 ## 2, x1 ## 2, x1 ## 2; \ + vpslld $3, x0 ## 2, x4 ## 2; \ + vpxor x4 ## 2, x3 ## 2, x3 ## 2; \ + vpsrld $13, x0 ## 1, x4 ## 1; \ + vpslld $(32 - 13), x0 ## 1, x0 ## 1; \ + vpor x4 ## 1, x0 ## 1, x0 ## 1; \ + vpxor x2 ## 1, x1 ## 1, x1 ## 1; \ + vpxor x2 ## 1, x3 ## 1, x3 ## 1; \ + vpsrld $3, x2 ## 1, x4 ## 1; \ + vpslld $(32 - 3), x2 ## 1, x2 ## 1; \ + vpor x4 ## 1, x2 ## 1, x2 ## 1; \ + vpsrld $13, x0 ## 2, x4 ## 2; \ + vpslld $(32 - 13), x0 ## 2, x0 ## 2; \ + vpor x4 ## 2, x0 ## 2, x0 ## 2; \ + vpxor x2 ## 2, x1 ## 2, x1 ## 2; \ + vpxor x2 ## 2, x3 ## 2, x3 ## 2; \ + vpsrld $3, x2 ## 2, x4 ## 2; \ + vpslld $(32 - 3), x2 ## 2, x2 ## 2; \ + vpor x4 ## 2, x2 ## 2, x2 ## 2; + +#define S(SBOX, x0, x1, x2, x3, x4) \ + SBOX ## _1(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \ + SBOX ## _2(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \ + SBOX ## _1(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2); \ + SBOX ## _2(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2); + +#define SP(SBOX, x0, x1, x2, x3, x4, i) \ + get_key(i, 0, RK0); \ + SBOX ## _1(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \ + get_key(i, 2, RK2); \ + SBOX ## _2(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \ + get_key(i, 3, RK3); \ + SBOX ## _1(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2); \ + get_key(i, 1, RK1); \ + SBOX ## _2(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2); \ + +#define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \ + vpunpckldq x1, x0, t0; \ + vpunpckhdq x1, x0, t2; \ + vpunpckldq x3, x2, t1; \ + vpunpckhdq x3, x2, x3; \ + \ + vpunpcklqdq t1, t0, x0; \ + vpunpckhqdq t1, t0, x1; \ + vpunpcklqdq x3, t2, x2; \ + vpunpckhqdq x3, t2, x3; + +#define read_blocks(in, x0, x1, x2, x3, t0, t1, t2) \ + vmovdqu (0*4*4)(in), x0; \ + vmovdqu (1*4*4)(in), x1; \ + vmovdqu (2*4*4)(in), x2; \ + vmovdqu (3*4*4)(in), x3; \ + \ + transpose_4x4(x0, x1, x2, x3, t0, t1, t2) + +#define write_blocks(out, x0, x1, x2, x3, t0, t1, t2) \ + transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \ + \ + vmovdqu x0, (0*4*4)(out); \ + vmovdqu x1, (1*4*4)(out); \ + vmovdqu x2, (2*4*4)(out); \ + vmovdqu x3, (3*4*4)(out); + +#define xor_blocks(out, x0, x1, x2, x3, t0, t1, t2) \ + transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \ + \ + vpxor (0*4*4)(out), x0, x0; \ + vmovdqu x0, (0*4*4)(out); \ + vpxor (1*4*4)(out), x1, x1; \ + vmovdqu x1, (1*4*4)(out); \ + vpxor (2*4*4)(out), x2, x2; \ + vmovdqu x2, (2*4*4)(out); \ + vpxor (3*4*4)(out), x3, x3; \ + vmovdqu x3, (3*4*4)(out); + +.align 8 +.global __serpent_enc_blk_8way_avx +.type __serpent_enc_blk_8way_avx,@function; + +__serpent_enc_blk_8way_avx: + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + * %rcx: bool, if true: xor output + */ + + vpcmpeqd RNOT, RNOT, RNOT; + + leaq (4*4*4)(%rdx), %rax; + read_blocks(%rdx, RA1, RB1, RC1, RD1, RK0, RK1, RK2); + read_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2); + + K2(RA, RB, RC, RD, RE, 0); + S(S0, RA, RB, RC, RD, RE); LK2(RC, RB, RD, RA, RE, 1); + S(S1, RC, RB, RD, RA, RE); LK2(RE, RD, RA, RC, RB, 2); + S(S2, RE, RD, RA, RC, RB); LK2(RB, RD, RE, RC, RA, 3); + S(S3, RB, RD, RE, RC, RA); LK2(RC, RA, RD, RB, RE, 4); + S(S4, RC, RA, RD, RB, RE); LK2(RA, RD, RB, RE, RC, 5); + S(S5, RA, RD, RB, RE, RC); LK2(RC, RA, RD, RE, RB, 6); + S(S6, RC, RA, RD, RE, RB); LK2(RD, RB, RA, RE, RC, 7); + S(S7, RD, RB, RA, RE, RC); LK2(RC, RA, RE, RD, RB, 8); + S(S0, RC, RA, RE, RD, RB); LK2(RE, RA, RD, RC, RB, 9); + S(S1, RE, RA, RD, RC, RB); LK2(RB, RD, RC, RE, RA, 10); + S(S2, RB, RD, RC, RE, RA); LK2(RA, RD, RB, RE, RC, 11); + S(S3, RA, RD, RB, RE, RC); LK2(RE, RC, RD, RA, RB, 12); + S(S4, RE, RC, RD, RA, RB); LK2(RC, RD, RA, RB, RE, 13); + S(S5, RC, RD, RA, RB, RE); LK2(RE, RC, RD, RB, RA, 14); + S(S6, RE, RC, RD, RB, RA); LK2(RD, RA, RC, RB, RE, 15); + S(S7, RD, RA, RC, RB, RE); LK2(RE, RC, RB, RD, RA, 16); + S(S0, RE, RC, RB, RD, RA); LK2(RB, RC, RD, RE, RA, 17); + S(S1, RB, RC, RD, RE, RA); LK2(RA, RD, RE, RB, RC, 18); + S(S2, RA, RD, RE, RB, RC); LK2(RC, RD, RA, RB, RE, 19); + S(S3, RC, RD, RA, RB, RE); LK2(RB, RE, RD, RC, RA, 20); + S(S4, RB, RE, RD, RC, RA); LK2(RE, RD, RC, RA, RB, 21); + S(S5, RE, RD, RC, RA, RB); LK2(RB, RE, RD, RA, RC, 22); + S(S6, RB, RE, RD, RA, RC); LK2(RD, RC, RE, RA, RB, 23); + S(S7, RD, RC, RE, RA, RB); LK2(RB, RE, RA, RD, RC, 24); + S(S0, RB, RE, RA, RD, RC); LK2(RA, RE, RD, RB, RC, 25); + S(S1, RA, RE, RD, RB, RC); LK2(RC, RD, RB, RA, RE, 26); + S(S2, RC, RD, RB, RA, RE); LK2(RE, RD, RC, RA, RB, 27); + S(S3, RE, RD, RC, RA, RB); LK2(RA, RB, RD, RE, RC, 28); + S(S4, RA, RB, RD, RE, RC); LK2(RB, RD, RE, RC, RA, 29); + S(S5, RB, RD, RE, RC, RA); LK2(RA, RB, RD, RC, RE, 30); + S(S6, RA, RB, RD, RC, RE); LK2(RD, RE, RB, RC, RA, 31); + S(S7, RD, RE, RB, RC, RA); K2(RA, RB, RC, RD, RE, 32); + + leaq (4*4*4)(%rsi), %rax; + + testb %cl, %cl; + jnz __enc_xor8; + + write_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2); + write_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2); + + ret; + +__enc_xor8: + xor_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2); + xor_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2); + + ret; + +.align 8 +.global serpent_dec_blk_8way_avx +.type serpent_dec_blk_8way_avx,@function; + +serpent_dec_blk_8way_avx: + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + + vpcmpeqd RNOT, RNOT, RNOT; + + leaq (4*4*4)(%rdx), %rax; + read_blocks(%rdx, RA1, RB1, RC1, RD1, RK0, RK1, RK2); + read_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2); + + K2(RA, RB, RC, RD, RE, 32); + SP(SI7, RA, RB, RC, RD, RE, 31); KL2(RB, RD, RA, RE, RC, 31); + SP(SI6, RB, RD, RA, RE, RC, 30); KL2(RA, RC, RE, RB, RD, 30); + SP(SI5, RA, RC, RE, RB, RD, 29); KL2(RC, RD, RA, RE, RB, 29); + SP(SI4, RC, RD, RA, RE, RB, 28); KL2(RC, RA, RB, RE, RD, 28); + SP(SI3, RC, RA, RB, RE, RD, 27); KL2(RB, RC, RD, RE, RA, 27); + SP(SI2, RB, RC, RD, RE, RA, 26); KL2(RC, RA, RE, RD, RB, 26); + SP(SI1, RC, RA, RE, RD, RB, 25); KL2(RB, RA, RE, RD, RC, 25); + SP(SI0, RB, RA, RE, RD, RC, 24); KL2(RE, RC, RA, RB, RD, 24); + SP(SI7, RE, RC, RA, RB, RD, 23); KL2(RC, RB, RE, RD, RA, 23); + SP(SI6, RC, RB, RE, RD, RA, 22); KL2(RE, RA, RD, RC, RB, 22); + SP(SI5, RE, RA, RD, RC, RB, 21); KL2(RA, RB, RE, RD, RC, 21); + SP(SI4, RA, RB, RE, RD, RC, 20); KL2(RA, RE, RC, RD, RB, 20); + SP(SI3, RA, RE, RC, RD, RB, 19); KL2(RC, RA, RB, RD, RE, 19); + SP(SI2, RC, RA, RB, RD, RE, 18); KL2(RA, RE, RD, RB, RC, 18); + SP(SI1, RA, RE, RD, RB, RC, 17); KL2(RC, RE, RD, RB, RA, 17); + SP(SI0, RC, RE, RD, RB, RA, 16); KL2(RD, RA, RE, RC, RB, 16); + SP(SI7, RD, RA, RE, RC, RB, 15); KL2(RA, RC, RD, RB, RE, 15); + SP(SI6, RA, RC, RD, RB, RE, 14); KL2(RD, RE, RB, RA, RC, 14); + SP(SI5, RD, RE, RB, RA, RC, 13); KL2(RE, RC, RD, RB, RA, 13); + SP(SI4, RE, RC, RD, RB, RA, 12); KL2(RE, RD, RA, RB, RC, 12); + SP(SI3, RE, RD, RA, RB, RC, 11); KL2(RA, RE, RC, RB, RD, 11); + SP(SI2, RA, RE, RC, RB, RD, 10); KL2(RE, RD, RB, RC, RA, 10); + SP(SI1, RE, RD, RB, RC, RA, 9); KL2(RA, RD, RB, RC, RE, 9); + SP(SI0, RA, RD, RB, RC, RE, 8); KL2(RB, RE, RD, RA, RC, 8); + SP(SI7, RB, RE, RD, RA, RC, 7); KL2(RE, RA, RB, RC, RD, 7); + SP(SI6, RE, RA, RB, RC, RD, 6); KL2(RB, RD, RC, RE, RA, 6); + SP(SI5, RB, RD, RC, RE, RA, 5); KL2(RD, RA, RB, RC, RE, 5); + SP(SI4, RD, RA, RB, RC, RE, 4); KL2(RD, RB, RE, RC, RA, 4); + SP(SI3, RD, RB, RE, RC, RA, 3); KL2(RE, RD, RA, RC, RB, 3); + SP(SI2, RE, RD, RA, RC, RB, 2); KL2(RD, RB, RC, RA, RE, 2); + SP(SI1, RD, RB, RC, RA, RE, 1); KL2(RE, RB, RC, RA, RD, 1); + S(SI0, RE, RB, RC, RA, RD); K2(RC, RD, RB, RE, RA, 0); + + leaq (4*4*4)(%rsi), %rax; + write_blocks(%rsi, RC1, RD1, RB1, RE1, RK0, RK1, RK2); + write_blocks(%rax, RC2, RD2, RB2, RE2, RK0, RK1, RK2); + + ret; diff --git a/arch/x86/crypto/serpent_avx_glue.c b/arch/x86/crypto/serpent_avx_glue.c new file mode 100644 index 000000000000..b36bdac237eb --- /dev/null +++ b/arch/x86/crypto/serpent_avx_glue.c @@ -0,0 +1,636 @@ +/* + * Glue Code for AVX assembler versions of Serpent Cipher + * + * Copyright (C) 2012 Johannes Goetzfried + * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> + * + * Glue code based on serpent_sse2_glue.c by: + * Copyright (C) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA + * + */ + +#include <linux/module.h> +#include <linux/hardirq.h> +#include <linux/types.h> +#include <linux/crypto.h> +#include <linux/err.h> +#include <crypto/algapi.h> +#include <crypto/serpent.h> +#include <crypto/cryptd.h> +#include <crypto/b128ops.h> +#include <crypto/ctr.h> +#include <crypto/lrw.h> +#include <crypto/xts.h> +#include <asm/xcr.h> +#include <asm/xsave.h> +#include <asm/crypto/serpent-avx.h> +#include <asm/crypto/ablk_helper.h> +#include <asm/crypto/glue_helper.h> + +static void serpent_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src) +{ + u128 ivs[SERPENT_PARALLEL_BLOCKS - 1]; + unsigned int j; + + for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++) + ivs[j] = src[j]; + + serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src); + + for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++) + u128_xor(dst + (j + 1), dst + (j + 1), ivs + j); +} + +static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv) +{ + be128 ctrblk; + + u128_to_be128(&ctrblk, iv); + u128_inc(iv); + + __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk); + u128_xor(dst, src, (u128 *)&ctrblk); +} + +static void serpent_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src, + u128 *iv) +{ + be128 ctrblks[SERPENT_PARALLEL_BLOCKS]; + unsigned int i; + + for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) { + if (dst != src) + dst[i] = src[i]; + + u128_to_be128(&ctrblks[i], iv); + u128_inc(iv); + } + + serpent_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks); +} + +static const struct common_glue_ctx serpent_enc = { + .num_funcs = 2, + .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = SERPENT_PARALLEL_BLOCKS, + .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_enc_blk_xway) } + }, { + .num_blocks = 1, + .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) } + } } +}; + +static const struct common_glue_ctx serpent_ctr = { + .num_funcs = 2, + .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = SERPENT_PARALLEL_BLOCKS, + .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr_xway) } + }, { + .num_blocks = 1, + .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr) } + } } +}; + +static const struct common_glue_ctx serpent_dec = { + .num_funcs = 2, + .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = SERPENT_PARALLEL_BLOCKS, + .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_dec_blk_xway) } + }, { + .num_blocks = 1, + .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) } + } } +}; + +static const struct common_glue_ctx serpent_dec_cbc = { + .num_funcs = 2, + .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = SERPENT_PARALLEL_BLOCKS, + .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_decrypt_cbc_xway) } + }, { + .num_blocks = 1, + .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) } + } } +}; + +static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_ecb_crypt_128bit(&serpent_enc, desc, dst, src, nbytes); +} + +static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_ecb_crypt_128bit(&serpent_dec, desc, dst, src, nbytes); +} + +static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__serpent_encrypt), desc, + dst, src, nbytes); +} + +static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_cbc_decrypt_128bit(&serpent_dec_cbc, desc, dst, src, + nbytes); +} + +static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_ctr_crypt_128bit(&serpent_ctr, desc, dst, src, nbytes); +} + +static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes) +{ + return glue_fpu_begin(SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS, + NULL, fpu_enabled, nbytes); +} + +static inline void serpent_fpu_end(bool fpu_enabled) +{ + glue_fpu_end(fpu_enabled); +} + +struct crypt_priv { + struct serpent_ctx *ctx; + bool fpu_enabled; +}; + +static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) +{ + const unsigned int bsize = SERPENT_BLOCK_SIZE; + struct crypt_priv *ctx = priv; + int i; + + ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes); + + if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) { + serpent_enc_blk_xway(ctx->ctx, srcdst, srcdst); + return; + } + + for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) + __serpent_encrypt(ctx->ctx, srcdst, srcdst); +} + +static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) +{ + const unsigned int bsize = SERPENT_BLOCK_SIZE; + struct crypt_priv *ctx = priv; + int i; + + ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes); + + if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) { + serpent_dec_blk_xway(ctx->ctx, srcdst, srcdst); + return; + } + + for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) + __serpent_decrypt(ctx->ctx, srcdst, srcdst); +} + +struct serpent_lrw_ctx { + struct lrw_table_ctx lrw_table; + struct serpent_ctx serpent_ctx; +}; + +static int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key, + unsigned int keylen) +{ + struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm); + int err; + + err = __serpent_setkey(&ctx->serpent_ctx, key, keylen - + SERPENT_BLOCK_SIZE); + if (err) + return err; + + return lrw_init_table(&ctx->lrw_table, key + keylen - + SERPENT_BLOCK_SIZE); +} + +static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + be128 buf[SERPENT_PARALLEL_BLOCKS]; + struct crypt_priv crypt_ctx = { + .ctx = &ctx->serpent_ctx, + .fpu_enabled = false, + }; + struct lrw_crypt_req req = { + .tbuf = buf, + .tbuflen = sizeof(buf), + + .table_ctx = &ctx->lrw_table, + .crypt_ctx = &crypt_ctx, + .crypt_fn = encrypt_callback, + }; + int ret; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + ret = lrw_crypt(desc, dst, src, nbytes, &req); + serpent_fpu_end(crypt_ctx.fpu_enabled); + + return ret; +} + +static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + be128 buf[SERPENT_PARALLEL_BLOCKS]; + struct crypt_priv crypt_ctx = { + .ctx = &ctx->serpent_ctx, + .fpu_enabled = false, + }; + struct lrw_crypt_req req = { + .tbuf = buf, + .tbuflen = sizeof(buf), + + .table_ctx = &ctx->lrw_table, + .crypt_ctx = &crypt_ctx, + .crypt_fn = decrypt_callback, + }; + int ret; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + ret = lrw_crypt(desc, dst, src, nbytes, &req); + serpent_fpu_end(crypt_ctx.fpu_enabled); + + return ret; +} + +static void lrw_exit_tfm(struct crypto_tfm *tfm) +{ + struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm); + + lrw_free_table(&ctx->lrw_table); +} + +struct serpent_xts_ctx { + struct serpent_ctx tweak_ctx; + struct serpent_ctx crypt_ctx; +}; + +static int xts_serpent_setkey(struct crypto_tfm *tfm, const u8 *key, + unsigned int keylen) +{ + struct serpent_xts_ctx *ctx = crypto_tfm_ctx(tfm); + u32 *flags = &tfm->crt_flags; + int err; + + /* key consists of keys of equal size concatenated, therefore + * the length must be even + */ + if (keylen % 2) { + *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN; + return -EINVAL; + } + + /* first half of xts-key is for crypt */ + err = __serpent_setkey(&ctx->crypt_ctx, key, keylen / 2); + if (err) + return err; + + /* second half of xts-key is for tweak */ + return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2); +} + +static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + be128 buf[SERPENT_PARALLEL_BLOCKS]; + struct crypt_priv crypt_ctx = { + .ctx = &ctx->crypt_ctx, + .fpu_enabled = false, + }; + struct xts_crypt_req req = { + .tbuf = buf, + .tbuflen = sizeof(buf), + + .tweak_ctx = &ctx->tweak_ctx, + .tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt), + .crypt_ctx = &crypt_ctx, + .crypt_fn = encrypt_callback, + }; + int ret; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + ret = xts_crypt(desc, dst, src, nbytes, &req); + serpent_fpu_end(crypt_ctx.fpu_enabled); + + return ret; +} + +static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + be128 buf[SERPENT_PARALLEL_BLOCKS]; + struct crypt_priv crypt_ctx = { + .ctx = &ctx->crypt_ctx, + .fpu_enabled = false, + }; + struct xts_crypt_req req = { + .tbuf = buf, + .tbuflen = sizeof(buf), + + .tweak_ctx = &ctx->tweak_ctx, + .tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt), + .crypt_ctx = &crypt_ctx, + .crypt_fn = decrypt_callback, + }; + int ret; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + ret = xts_crypt(desc, dst, src, nbytes, &req); + serpent_fpu_end(crypt_ctx.fpu_enabled); + + return ret; +} + +static struct crypto_alg serpent_algs[10] = { { + .cra_name = "__ecb-serpent-avx", + .cra_driver_name = "__driver-ecb-serpent-avx", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = SERPENT_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct serpent_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(serpent_algs[0].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE, + .max_keysize = SERPENT_MAX_KEY_SIZE, + .setkey = serpent_setkey, + .encrypt = ecb_encrypt, + .decrypt = ecb_decrypt, + }, + }, +}, { + .cra_name = "__cbc-serpent-avx", + .cra_driver_name = "__driver-cbc-serpent-avx", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = SERPENT_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct serpent_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(serpent_algs[1].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE, + .max_keysize = SERPENT_MAX_KEY_SIZE, + .setkey = serpent_setkey, + .encrypt = cbc_encrypt, + .decrypt = cbc_decrypt, + }, + }, +}, { + .cra_name = "__ctr-serpent-avx", + .cra_driver_name = "__driver-ctr-serpent-avx", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct serpent_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(serpent_algs[2].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE, + .max_keysize = SERPENT_MAX_KEY_SIZE, + .ivsize = SERPENT_BLOCK_SIZE, + .setkey = serpent_setkey, + .encrypt = ctr_crypt, + .decrypt = ctr_crypt, + }, + }, +}, { + .cra_name = "__lrw-serpent-avx", + .cra_driver_name = "__driver-lrw-serpent-avx", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = SERPENT_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct serpent_lrw_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(serpent_algs[3].cra_list), + .cra_exit = lrw_exit_tfm, + .cra_u = { + .blkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE + + SERPENT_BLOCK_SIZE, + .max_keysize = SERPENT_MAX_KEY_SIZE + + SERPENT_BLOCK_SIZE, + .ivsize = SERPENT_BLOCK_SIZE, + .setkey = lrw_serpent_setkey, + .encrypt = lrw_encrypt, + .decrypt = lrw_decrypt, + }, + }, +}, { + .cra_name = "__xts-serpent-avx", + .cra_driver_name = "__driver-xts-serpent-avx", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = SERPENT_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct serpent_xts_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(serpent_algs[4].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE * 2, + .max_keysize = SERPENT_MAX_KEY_SIZE * 2, + .ivsize = SERPENT_BLOCK_SIZE, + .setkey = xts_serpent_setkey, + .encrypt = xts_encrypt, + .decrypt = xts_decrypt, + }, + }, +}, { + .cra_name = "ecb(serpent)", + .cra_driver_name = "ecb-serpent-avx", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = SERPENT_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(serpent_algs[5].cra_list), + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE, + .max_keysize = SERPENT_MAX_KEY_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +}, { + .cra_name = "cbc(serpent)", + .cra_driver_name = "cbc-serpent-avx", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = SERPENT_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(serpent_algs[6].cra_list), + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE, + .max_keysize = SERPENT_MAX_KEY_SIZE, + .ivsize = SERPENT_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = __ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +}, { + .cra_name = "ctr(serpent)", + .cra_driver_name = "ctr-serpent-avx", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(serpent_algs[7].cra_list), + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE, + .max_keysize = SERPENT_MAX_KEY_SIZE, + .ivsize = SERPENT_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_encrypt, + .geniv = "chainiv", + }, + }, +}, { + .cra_name = "lrw(serpent)", + .cra_driver_name = "lrw-serpent-avx", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = SERPENT_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(serpent_algs[8].cra_list), + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE + + SERPENT_BLOCK_SIZE, + .max_keysize = SERPENT_MAX_KEY_SIZE + + SERPENT_BLOCK_SIZE, + .ivsize = SERPENT_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +}, { + .cra_name = "xts(serpent)", + .cra_driver_name = "xts-serpent-avx", + .cra_priority = 500, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = SERPENT_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(serpent_algs[9].cra_list), + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = SERPENT_MIN_KEY_SIZE * 2, + .max_keysize = SERPENT_MAX_KEY_SIZE * 2, + .ivsize = SERPENT_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +} }; + +static int __init serpent_init(void) +{ + u64 xcr0; + + if (!cpu_has_avx || !cpu_has_osxsave) { + printk(KERN_INFO "AVX instructions are not detected.\n"); + return -ENODEV; + } + + xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); + if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) { + printk(KERN_INFO "AVX detected but unusable.\n"); + return -ENODEV; + } + + return crypto_register_algs(serpent_algs, ARRAY_SIZE(serpent_algs)); +} + +static void __exit serpent_exit(void) +{ + crypto_unregister_algs(serpent_algs, ARRAY_SIZE(serpent_algs)); +} + +module_init(serpent_init); +module_exit(serpent_exit); + +MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX optimized"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("serpent"); diff --git a/arch/x86/crypto/serpent_sse2_glue.c b/arch/x86/crypto/serpent_sse2_glue.c index 4b21be85e0a1..d679c8675f4a 100644 --- a/arch/x86/crypto/serpent_sse2_glue.c +++ b/arch/x86/crypto/serpent_sse2_glue.c @@ -41,358 +41,145 @@ #include <crypto/ctr.h> #include <crypto/lrw.h> #include <crypto/xts.h> -#include <asm/i387.h> -#include <asm/serpent.h> -#include <crypto/scatterwalk.h> -#include <linux/workqueue.h> -#include <linux/spinlock.h> - -struct async_serpent_ctx { - struct cryptd_ablkcipher *cryptd_tfm; -}; +#include <asm/crypto/serpent-sse2.h> +#include <asm/crypto/ablk_helper.h> +#include <asm/crypto/glue_helper.h> -static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes) -{ - if (fpu_enabled) - return true; - - /* SSE2 is only used when chunk to be processed is large enough, so - * do not enable FPU until it is necessary. - */ - if (nbytes < SERPENT_BLOCK_SIZE * SERPENT_PARALLEL_BLOCKS) - return false; - - kernel_fpu_begin(); - return true; -} - -static inline void serpent_fpu_end(bool fpu_enabled) +static void serpent_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src) { - if (fpu_enabled) - kernel_fpu_end(); -} - -static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk, - bool enc) -{ - bool fpu_enabled = false; - struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - const unsigned int bsize = SERPENT_BLOCK_SIZE; - unsigned int nbytes; - int err; - - err = blkcipher_walk_virt(desc, walk); - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; - - while ((nbytes = walk->nbytes)) { - u8 *wsrc = walk->src.virt.addr; - u8 *wdst = walk->dst.virt.addr; - - fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes); - - /* Process multi-block batch */ - if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) { - do { - if (enc) - serpent_enc_blk_xway(ctx, wdst, wsrc); - else - serpent_dec_blk_xway(ctx, wdst, wsrc); - - wsrc += bsize * SERPENT_PARALLEL_BLOCKS; - wdst += bsize * SERPENT_PARALLEL_BLOCKS; - nbytes -= bsize * SERPENT_PARALLEL_BLOCKS; - } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS); - - if (nbytes < bsize) - goto done; - } - - /* Handle leftovers */ - do { - if (enc) - __serpent_encrypt(ctx, wdst, wsrc); - else - __serpent_decrypt(ctx, wdst, wsrc); - - wsrc += bsize; - wdst += bsize; - nbytes -= bsize; - } while (nbytes >= bsize); - -done: - err = blkcipher_walk_done(desc, walk, nbytes); - } + u128 ivs[SERPENT_PARALLEL_BLOCKS - 1]; + unsigned int j; - serpent_fpu_end(fpu_enabled); - return err; -} + for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++) + ivs[j] = src[j]; -static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) -{ - struct blkcipher_walk walk; + serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src); - blkcipher_walk_init(&walk, dst, src, nbytes); - return ecb_crypt(desc, &walk, true); + for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++) + u128_xor(dst + (j + 1), dst + (j + 1), ivs + j); } -static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) +static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv) { - struct blkcipher_walk walk; + be128 ctrblk; - blkcipher_walk_init(&walk, dst, src, nbytes); - return ecb_crypt(desc, &walk, false); -} + u128_to_be128(&ctrblk, iv); + u128_inc(iv); -static unsigned int __cbc_encrypt(struct blkcipher_desc *desc, - struct blkcipher_walk *walk) -{ - struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - const unsigned int bsize = SERPENT_BLOCK_SIZE; - unsigned int nbytes = walk->nbytes; - u128 *src = (u128 *)walk->src.virt.addr; - u128 *dst = (u128 *)walk->dst.virt.addr; - u128 *iv = (u128 *)walk->iv; - - do { - u128_xor(dst, src, iv); - __serpent_encrypt(ctx, (u8 *)dst, (u8 *)dst); - iv = dst; - - src += 1; - dst += 1; - nbytes -= bsize; - } while (nbytes >= bsize); - - u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv); - return nbytes; + __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk); + u128_xor(dst, src, (u128 *)&ctrblk); } -static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) +static void serpent_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src, + u128 *iv) { - struct blkcipher_walk walk; - int err; + be128 ctrblks[SERPENT_PARALLEL_BLOCKS]; + unsigned int i; - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); + for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) { + if (dst != src) + dst[i] = src[i]; - while ((nbytes = walk.nbytes)) { - nbytes = __cbc_encrypt(desc, &walk); - err = blkcipher_walk_done(desc, &walk, nbytes); + u128_to_be128(&ctrblks[i], iv); + u128_inc(iv); } - return err; + serpent_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks); } -static unsigned int __cbc_decrypt(struct blkcipher_desc *desc, - struct blkcipher_walk *walk) -{ - struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - const unsigned int bsize = SERPENT_BLOCK_SIZE; - unsigned int nbytes = walk->nbytes; - u128 *src = (u128 *)walk->src.virt.addr; - u128 *dst = (u128 *)walk->dst.virt.addr; - u128 ivs[SERPENT_PARALLEL_BLOCKS - 1]; - u128 last_iv; - int i; - - /* Start of the last block. */ - src += nbytes / bsize - 1; - dst += nbytes / bsize - 1; - - last_iv = *src; - - /* Process multi-block batch */ - if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) { - do { - nbytes -= bsize * (SERPENT_PARALLEL_BLOCKS - 1); - src -= SERPENT_PARALLEL_BLOCKS - 1; - dst -= SERPENT_PARALLEL_BLOCKS - 1; - - for (i = 0; i < SERPENT_PARALLEL_BLOCKS - 1; i++) - ivs[i] = src[i]; - - serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src); - - for (i = 0; i < SERPENT_PARALLEL_BLOCKS - 1; i++) - u128_xor(dst + (i + 1), dst + (i + 1), ivs + i); - - nbytes -= bsize; - if (nbytes < bsize) - goto done; +static const struct common_glue_ctx serpent_enc = { + .num_funcs = 2, + .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, - u128_xor(dst, dst, src - 1); - src -= 1; - dst -= 1; - } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS); - - if (nbytes < bsize) - goto done; - } - - /* Handle leftovers */ - for (;;) { - __serpent_decrypt(ctx, (u8 *)dst, (u8 *)src); - - nbytes -= bsize; - if (nbytes < bsize) - break; + .funcs = { { + .num_blocks = SERPENT_PARALLEL_BLOCKS, + .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_enc_blk_xway) } + }, { + .num_blocks = 1, + .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) } + } } +}; - u128_xor(dst, dst, src - 1); - src -= 1; - dst -= 1; - } +static const struct common_glue_ctx serpent_ctr = { + .num_funcs = 2, + .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = SERPENT_PARALLEL_BLOCKS, + .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr_xway) } + }, { + .num_blocks = 1, + .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr) } + } } +}; -done: - u128_xor(dst, dst, (u128 *)walk->iv); - *(u128 *)walk->iv = last_iv; +static const struct common_glue_ctx serpent_dec = { + .num_funcs = 2, + .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = SERPENT_PARALLEL_BLOCKS, + .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_dec_blk_xway) } + }, { + .num_blocks = 1, + .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) } + } } +}; - return nbytes; -} +static const struct common_glue_ctx serpent_dec_cbc = { + .num_funcs = 2, + .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = SERPENT_PARALLEL_BLOCKS, + .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_decrypt_cbc_xway) } + }, { + .num_blocks = 1, + .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) } + } } +}; -static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, +static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { - bool fpu_enabled = false; - struct blkcipher_walk walk; - int err; - - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; - - while ((nbytes = walk.nbytes)) { - fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes); - nbytes = __cbc_decrypt(desc, &walk); - err = blkcipher_walk_done(desc, &walk, nbytes); - } - - serpent_fpu_end(fpu_enabled); - return err; + return glue_ecb_crypt_128bit(&serpent_enc, desc, dst, src, nbytes); } -static inline void u128_to_be128(be128 *dst, const u128 *src) +static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) { - dst->a = cpu_to_be64(src->a); - dst->b = cpu_to_be64(src->b); + return glue_ecb_crypt_128bit(&serpent_dec, desc, dst, src, nbytes); } -static inline void be128_to_u128(u128 *dst, const be128 *src) +static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) { - dst->a = be64_to_cpu(src->a); - dst->b = be64_to_cpu(src->b); + return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__serpent_encrypt), desc, + dst, src, nbytes); } -static inline void u128_inc(u128 *i) +static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) { - i->b++; - if (!i->b) - i->a++; + return glue_cbc_decrypt_128bit(&serpent_dec_cbc, desc, dst, src, + nbytes); } -static void ctr_crypt_final(struct blkcipher_desc *desc, - struct blkcipher_walk *walk) +static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) { - struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - u8 *ctrblk = walk->iv; - u8 keystream[SERPENT_BLOCK_SIZE]; - u8 *src = walk->src.virt.addr; - u8 *dst = walk->dst.virt.addr; - unsigned int nbytes = walk->nbytes; - - __serpent_encrypt(ctx, keystream, ctrblk); - crypto_xor(keystream, src, nbytes); - memcpy(dst, keystream, nbytes); - - crypto_inc(ctrblk, SERPENT_BLOCK_SIZE); + return glue_ctr_crypt_128bit(&serpent_ctr, desc, dst, src, nbytes); } -static unsigned int __ctr_crypt(struct blkcipher_desc *desc, - struct blkcipher_walk *walk) +static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes) { - struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - const unsigned int bsize = SERPENT_BLOCK_SIZE; - unsigned int nbytes = walk->nbytes; - u128 *src = (u128 *)walk->src.virt.addr; - u128 *dst = (u128 *)walk->dst.virt.addr; - u128 ctrblk; - be128 ctrblocks[SERPENT_PARALLEL_BLOCKS]; - int i; - - be128_to_u128(&ctrblk, (be128 *)walk->iv); - - /* Process multi-block batch */ - if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) { - do { - /* create ctrblks for parallel encrypt */ - for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) { - if (dst != src) - dst[i] = src[i]; - - u128_to_be128(&ctrblocks[i], &ctrblk); - u128_inc(&ctrblk); - } - - serpent_enc_blk_xway_xor(ctx, (u8 *)dst, - (u8 *)ctrblocks); - - src += SERPENT_PARALLEL_BLOCKS; - dst += SERPENT_PARALLEL_BLOCKS; - nbytes -= bsize * SERPENT_PARALLEL_BLOCKS; - } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS); - - if (nbytes < bsize) - goto done; - } - - /* Handle leftovers */ - do { - if (dst != src) - *dst = *src; - - u128_to_be128(&ctrblocks[0], &ctrblk); - u128_inc(&ctrblk); - - __serpent_encrypt(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks); - u128_xor(dst, dst, (u128 *)ctrblocks); - - src += 1; - dst += 1; - nbytes -= bsize; - } while (nbytes >= bsize); - -done: - u128_to_be128((be128 *)walk->iv, &ctrblk); - return nbytes; + return glue_fpu_begin(SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS, + NULL, fpu_enabled, nbytes); } -static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) +static inline void serpent_fpu_end(bool fpu_enabled) { - bool fpu_enabled = false; - struct blkcipher_walk walk; - int err; - - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt_block(desc, &walk, SERPENT_BLOCK_SIZE); - desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; - - while ((nbytes = walk.nbytes) >= SERPENT_BLOCK_SIZE) { - fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes); - nbytes = __ctr_crypt(desc, &walk); - err = blkcipher_walk_done(desc, &walk, nbytes); - } - - serpent_fpu_end(fpu_enabled); - - if (walk.nbytes) { - ctr_crypt_final(desc, &walk); - err = blkcipher_walk_done(desc, &walk, 0); - } - - return err; + glue_fpu_end(fpu_enabled); } struct crypt_priv { @@ -596,106 +383,6 @@ static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, return ret; } -static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key, - unsigned int key_len) -{ - struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm); - struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base; - int err; - - crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); - crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm) - & CRYPTO_TFM_REQ_MASK); - err = crypto_ablkcipher_setkey(child, key, key_len); - crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child) - & CRYPTO_TFM_RES_MASK); - return err; -} - -static int __ablk_encrypt(struct ablkcipher_request *req) -{ - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm); - struct blkcipher_desc desc; - - desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); - desc.info = req->info; - desc.flags = 0; - - return crypto_blkcipher_crt(desc.tfm)->encrypt( - &desc, req->dst, req->src, req->nbytes); -} - -static int ablk_encrypt(struct ablkcipher_request *req) -{ - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm); - - if (!irq_fpu_usable()) { - struct ablkcipher_request *cryptd_req = - ablkcipher_request_ctx(req); - - memcpy(cryptd_req, req, sizeof(*req)); - ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); - - return crypto_ablkcipher_encrypt(cryptd_req); - } else { - return __ablk_encrypt(req); - } -} - -static int ablk_decrypt(struct ablkcipher_request *req) -{ - struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); - struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm); - - if (!irq_fpu_usable()) { - struct ablkcipher_request *cryptd_req = - ablkcipher_request_ctx(req); - - memcpy(cryptd_req, req, sizeof(*req)); - ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); - - return crypto_ablkcipher_decrypt(cryptd_req); - } else { - struct blkcipher_desc desc; - - desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); - desc.info = req->info; - desc.flags = 0; - - return crypto_blkcipher_crt(desc.tfm)->decrypt( - &desc, req->dst, req->src, req->nbytes); - } -} - -static void ablk_exit(struct crypto_tfm *tfm) -{ - struct async_serpent_ctx *ctx = crypto_tfm_ctx(tfm); - - cryptd_free_ablkcipher(ctx->cryptd_tfm); -} - -static int ablk_init(struct crypto_tfm *tfm) -{ - struct async_serpent_ctx *ctx = crypto_tfm_ctx(tfm); - struct cryptd_ablkcipher *cryptd_tfm; - char drv_name[CRYPTO_MAX_ALG_NAME]; - - snprintf(drv_name, sizeof(drv_name), "__driver-%s", - crypto_tfm_alg_driver_name(tfm)); - - cryptd_tfm = cryptd_alloc_ablkcipher(drv_name, 0, 0); - if (IS_ERR(cryptd_tfm)) - return PTR_ERR(cryptd_tfm); - - ctx->cryptd_tfm = cryptd_tfm; - tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) + - crypto_ablkcipher_reqsize(&cryptd_tfm->base); - - return 0; -} - static struct crypto_alg serpent_algs[10] = { { .cra_name = "__ecb-serpent-sse2", .cra_driver_name = "__driver-ecb-serpent-sse2", @@ -808,7 +495,7 @@ static struct crypto_alg serpent_algs[10] = { { .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = SERPENT_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct async_serpent_ctx), + .cra_ctxsize = sizeof(struct async_helper_ctx), .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, @@ -830,7 +517,7 @@ static struct crypto_alg serpent_algs[10] = { { .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = SERPENT_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct async_serpent_ctx), + .cra_ctxsize = sizeof(struct async_helper_ctx), .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, @@ -853,7 +540,7 @@ static struct crypto_alg serpent_algs[10] = { { .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = 1, - .cra_ctxsize = sizeof(struct async_serpent_ctx), + .cra_ctxsize = sizeof(struct async_helper_ctx), .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, @@ -877,7 +564,7 @@ static struct crypto_alg serpent_algs[10] = { { .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = SERPENT_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct async_serpent_ctx), + .cra_ctxsize = sizeof(struct async_helper_ctx), .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, @@ -902,7 +589,7 @@ static struct crypto_alg serpent_algs[10] = { { .cra_priority = 400, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = SERPENT_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct async_serpent_ctx), + .cra_ctxsize = sizeof(struct async_helper_ctx), .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = THIS_MODULE, diff --git a/arch/x86/crypto/sha1_ssse3_asm.S b/arch/x86/crypto/sha1_ssse3_asm.S index b2c2f57d70e8..49d6987a73d9 100644 --- a/arch/x86/crypto/sha1_ssse3_asm.S +++ b/arch/x86/crypto/sha1_ssse3_asm.S @@ -468,7 +468,7 @@ W_PRECALC_SSSE3 */ SHA1_VECTOR_ASM sha1_transform_ssse3 -#ifdef SHA1_ENABLE_AVX_SUPPORT +#ifdef CONFIG_AS_AVX .macro W_PRECALC_AVX diff --git a/arch/x86/crypto/sha1_ssse3_glue.c b/arch/x86/crypto/sha1_ssse3_glue.c index f916499d0abe..4a11a9d72451 100644 --- a/arch/x86/crypto/sha1_ssse3_glue.c +++ b/arch/x86/crypto/sha1_ssse3_glue.c @@ -35,7 +35,7 @@ asmlinkage void sha1_transform_ssse3(u32 *digest, const char *data, unsigned int rounds); -#ifdef SHA1_ENABLE_AVX_SUPPORT +#ifdef CONFIG_AS_AVX asmlinkage void sha1_transform_avx(u32 *digest, const char *data, unsigned int rounds); #endif @@ -184,7 +184,7 @@ static struct shash_alg alg = { } }; -#ifdef SHA1_ENABLE_AVX_SUPPORT +#ifdef CONFIG_AS_AVX static bool __init avx_usable(void) { u64 xcr0; @@ -209,7 +209,7 @@ static int __init sha1_ssse3_mod_init(void) if (cpu_has_ssse3) sha1_transform_asm = sha1_transform_ssse3; -#ifdef SHA1_ENABLE_AVX_SUPPORT +#ifdef CONFIG_AS_AVX /* allow AVX to override SSSE3, it's a little faster */ if (avx_usable()) sha1_transform_asm = sha1_transform_avx; diff --git a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S new file mode 100644 index 000000000000..35f45574390d --- /dev/null +++ b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S @@ -0,0 +1,300 @@ +/* + * Twofish Cipher 8-way parallel algorithm (AVX/x86_64) + * + * Copyright (C) 2012 Johannes Goetzfried + * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA + * + */ + +.file "twofish-avx-x86_64-asm_64.S" +.text + +/* structure of crypto context */ +#define s0 0 +#define s1 1024 +#define s2 2048 +#define s3 3072 +#define w 4096 +#define k 4128 + +/********************************************************************** + 8-way AVX twofish + **********************************************************************/ +#define CTX %rdi + +#define RA1 %xmm0 +#define RB1 %xmm1 +#define RC1 %xmm2 +#define RD1 %xmm3 + +#define RA2 %xmm4 +#define RB2 %xmm5 +#define RC2 %xmm6 +#define RD2 %xmm7 + +#define RX %xmm8 +#define RY %xmm9 + +#define RK1 %xmm10 +#define RK2 %xmm11 + +#define RID1 %rax +#define RID1b %al +#define RID2 %rbx +#define RID2b %bl + +#define RGI1 %rdx +#define RGI1bl %dl +#define RGI1bh %dh +#define RGI2 %rcx +#define RGI2bl %cl +#define RGI2bh %ch + +#define RGS1 %r8 +#define RGS1d %r8d +#define RGS2 %r9 +#define RGS2d %r9d +#define RGS3 %r10 +#define RGS3d %r10d + + +#define lookup_32bit(t0, t1, t2, t3, src, dst) \ + movb src ## bl, RID1b; \ + movb src ## bh, RID2b; \ + movl t0(CTX, RID1, 4), dst ## d; \ + xorl t1(CTX, RID2, 4), dst ## d; \ + shrq $16, src; \ + movb src ## bl, RID1b; \ + movb src ## bh, RID2b; \ + xorl t2(CTX, RID1, 4), dst ## d; \ + xorl t3(CTX, RID2, 4), dst ## d; + +#define G(a, x, t0, t1, t2, t3) \ + vmovq a, RGI1; \ + vpsrldq $8, a, x; \ + vmovq x, RGI2; \ + \ + lookup_32bit(t0, t1, t2, t3, RGI1, RGS1); \ + shrq $16, RGI1; \ + lookup_32bit(t0, t1, t2, t3, RGI1, RGS2); \ + shlq $32, RGS2; \ + orq RGS1, RGS2; \ + \ + lookup_32bit(t0, t1, t2, t3, RGI2, RGS1); \ + shrq $16, RGI2; \ + lookup_32bit(t0, t1, t2, t3, RGI2, RGS3); \ + shlq $32, RGS3; \ + orq RGS1, RGS3; \ + \ + vmovq RGS2, x; \ + vpinsrq $1, RGS3, x, x; + +#define encround(a, b, c, d, x, y) \ + G(a, x, s0, s1, s2, s3); \ + G(b, y, s1, s2, s3, s0); \ + vpaddd x, y, x; \ + vpaddd y, x, y; \ + vpaddd x, RK1, x; \ + vpaddd y, RK2, y; \ + vpxor x, c, c; \ + vpsrld $1, c, x; \ + vpslld $(32 - 1), c, c; \ + vpor c, x, c; \ + vpslld $1, d, x; \ + vpsrld $(32 - 1), d, d; \ + vpor d, x, d; \ + vpxor d, y, d; + +#define decround(a, b, c, d, x, y) \ + G(a, x, s0, s1, s2, s3); \ + G(b, y, s1, s2, s3, s0); \ + vpaddd x, y, x; \ + vpaddd y, x, y; \ + vpaddd y, RK2, y; \ + vpxor d, y, d; \ + vpsrld $1, d, y; \ + vpslld $(32 - 1), d, d; \ + vpor d, y, d; \ + vpslld $1, c, y; \ + vpsrld $(32 - 1), c, c; \ + vpor c, y, c; \ + vpaddd x, RK1, x; \ + vpxor x, c, c; + +#define encrypt_round(n, a, b, c, d) \ + vbroadcastss (k+4*(2*(n)))(CTX), RK1; \ + vbroadcastss (k+4*(2*(n)+1))(CTX), RK2; \ + encround(a ## 1, b ## 1, c ## 1, d ## 1, RX, RY); \ + encround(a ## 2, b ## 2, c ## 2, d ## 2, RX, RY); + +#define decrypt_round(n, a, b, c, d) \ + vbroadcastss (k+4*(2*(n)))(CTX), RK1; \ + vbroadcastss (k+4*(2*(n)+1))(CTX), RK2; \ + decround(a ## 1, b ## 1, c ## 1, d ## 1, RX, RY); \ + decround(a ## 2, b ## 2, c ## 2, d ## 2, RX, RY); + +#define encrypt_cycle(n) \ + encrypt_round((2*n), RA, RB, RC, RD); \ + encrypt_round(((2*n) + 1), RC, RD, RA, RB); + +#define decrypt_cycle(n) \ + decrypt_round(((2*n) + 1), RC, RD, RA, RB); \ + decrypt_round((2*n), RA, RB, RC, RD); + + +#define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \ + vpunpckldq x1, x0, t0; \ + vpunpckhdq x1, x0, t2; \ + vpunpckldq x3, x2, t1; \ + vpunpckhdq x3, x2, x3; \ + \ + vpunpcklqdq t1, t0, x0; \ + vpunpckhqdq t1, t0, x1; \ + vpunpcklqdq x3, t2, x2; \ + vpunpckhqdq x3, t2, x3; + +#define inpack_blocks(in, x0, x1, x2, x3, wkey, t0, t1, t2) \ + vpxor (0*4*4)(in), wkey, x0; \ + vpxor (1*4*4)(in), wkey, x1; \ + vpxor (2*4*4)(in), wkey, x2; \ + vpxor (3*4*4)(in), wkey, x3; \ + \ + transpose_4x4(x0, x1, x2, x3, t0, t1, t2) + +#define outunpack_blocks(out, x0, x1, x2, x3, wkey, t0, t1, t2) \ + transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \ + \ + vpxor x0, wkey, x0; \ + vmovdqu x0, (0*4*4)(out); \ + vpxor x1, wkey, x1; \ + vmovdqu x1, (1*4*4)(out); \ + vpxor x2, wkey, x2; \ + vmovdqu x2, (2*4*4)(out); \ + vpxor x3, wkey, x3; \ + vmovdqu x3, (3*4*4)(out); + +#define outunpack_xor_blocks(out, x0, x1, x2, x3, wkey, t0, t1, t2) \ + transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \ + \ + vpxor x0, wkey, x0; \ + vpxor (0*4*4)(out), x0, x0; \ + vmovdqu x0, (0*4*4)(out); \ + vpxor x1, wkey, x1; \ + vpxor (1*4*4)(out), x1, x1; \ + vmovdqu x1, (1*4*4)(out); \ + vpxor x2, wkey, x2; \ + vpxor (2*4*4)(out), x2, x2; \ + vmovdqu x2, (2*4*4)(out); \ + vpxor x3, wkey, x3; \ + vpxor (3*4*4)(out), x3, x3; \ + vmovdqu x3, (3*4*4)(out); + +.align 8 +.global __twofish_enc_blk_8way +.type __twofish_enc_blk_8way,@function; + +__twofish_enc_blk_8way: + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + * %rcx: bool, if true: xor output + */ + + pushq %rbx; + pushq %rcx; + + vmovdqu w(CTX), RK1; + + leaq (4*4*4)(%rdx), %rax; + inpack_blocks(%rdx, RA1, RB1, RC1, RD1, RK1, RX, RY, RK2); + inpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX, RY, RK2); + + xorq RID1, RID1; + xorq RID2, RID2; + + encrypt_cycle(0); + encrypt_cycle(1); + encrypt_cycle(2); + encrypt_cycle(3); + encrypt_cycle(4); + encrypt_cycle(5); + encrypt_cycle(6); + encrypt_cycle(7); + + vmovdqu (w+4*4)(CTX), RK1; + + popq %rcx; + popq %rbx; + + leaq (4*4*4)(%rsi), %rax; + + testb %cl, %cl; + jnz __enc_xor8; + + outunpack_blocks(%rsi, RC1, RD1, RA1, RB1, RK1, RX, RY, RK2); + outunpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX, RY, RK2); + + ret; + +__enc_xor8: + outunpack_xor_blocks(%rsi, RC1, RD1, RA1, RB1, RK1, RX, RY, RK2); + outunpack_xor_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX, RY, RK2); + + ret; + +.align 8 +.global twofish_dec_blk_8way +.type twofish_dec_blk_8way,@function; + +twofish_dec_blk_8way: + /* input: + * %rdi: ctx, CTX + * %rsi: dst + * %rdx: src + */ + + pushq %rbx; + + vmovdqu (w+4*4)(CTX), RK1; + + leaq (4*4*4)(%rdx), %rax; + inpack_blocks(%rdx, RC1, RD1, RA1, RB1, RK1, RX, RY, RK2); + inpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX, RY, RK2); + + xorq RID1, RID1; + xorq RID2, RID2; + + decrypt_cycle(7); + decrypt_cycle(6); + decrypt_cycle(5); + decrypt_cycle(4); + decrypt_cycle(3); + decrypt_cycle(2); + decrypt_cycle(1); + decrypt_cycle(0); + + vmovdqu (w)(CTX), RK1; + + popq %rbx; + + leaq (4*4*4)(%rsi), %rax; + outunpack_blocks(%rsi, RA1, RB1, RC1, RD1, RK1, RX, RY, RK2); + outunpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX, RY, RK2); + + ret; diff --git a/arch/x86/crypto/twofish_avx_glue.c b/arch/x86/crypto/twofish_avx_glue.c new file mode 100644 index 000000000000..782b67ddaf6a --- /dev/null +++ b/arch/x86/crypto/twofish_avx_glue.c @@ -0,0 +1,624 @@ +/* + * Glue Code for AVX assembler version of Twofish Cipher + * + * Copyright (C) 2012 Johannes Goetzfried + * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA + * + */ + +#include <linux/module.h> +#include <linux/hardirq.h> +#include <linux/types.h> +#include <linux/crypto.h> +#include <linux/err.h> +#include <crypto/algapi.h> +#include <crypto/twofish.h> +#include <crypto/cryptd.h> +#include <crypto/b128ops.h> +#include <crypto/ctr.h> +#include <crypto/lrw.h> +#include <crypto/xts.h> +#include <asm/i387.h> +#include <asm/xcr.h> +#include <asm/xsave.h> +#include <asm/crypto/twofish.h> +#include <asm/crypto/ablk_helper.h> +#include <asm/crypto/glue_helper.h> +#include <crypto/scatterwalk.h> +#include <linux/workqueue.h> +#include <linux/spinlock.h> + +#define TWOFISH_PARALLEL_BLOCKS 8 + +static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst, + const u8 *src) +{ + __twofish_enc_blk_3way(ctx, dst, src, false); +} + +/* 8-way parallel cipher functions */ +asmlinkage void __twofish_enc_blk_8way(struct twofish_ctx *ctx, u8 *dst, + const u8 *src, bool xor); +asmlinkage void twofish_dec_blk_8way(struct twofish_ctx *ctx, u8 *dst, + const u8 *src); + +static inline void twofish_enc_blk_xway(struct twofish_ctx *ctx, u8 *dst, + const u8 *src) +{ + __twofish_enc_blk_8way(ctx, dst, src, false); +} + +static inline void twofish_enc_blk_xway_xor(struct twofish_ctx *ctx, u8 *dst, + const u8 *src) +{ + __twofish_enc_blk_8way(ctx, dst, src, true); +} + +static inline void twofish_dec_blk_xway(struct twofish_ctx *ctx, u8 *dst, + const u8 *src) +{ + twofish_dec_blk_8way(ctx, dst, src); +} + +static void twofish_dec_blk_cbc_xway(void *ctx, u128 *dst, const u128 *src) +{ + u128 ivs[TWOFISH_PARALLEL_BLOCKS - 1]; + unsigned int j; + + for (j = 0; j < TWOFISH_PARALLEL_BLOCKS - 1; j++) + ivs[j] = src[j]; + + twofish_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src); + + for (j = 0; j < TWOFISH_PARALLEL_BLOCKS - 1; j++) + u128_xor(dst + (j + 1), dst + (j + 1), ivs + j); +} + +static void twofish_enc_blk_ctr_xway(void *ctx, u128 *dst, const u128 *src, + u128 *iv) +{ + be128 ctrblks[TWOFISH_PARALLEL_BLOCKS]; + unsigned int i; + + for (i = 0; i < TWOFISH_PARALLEL_BLOCKS; i++) { + if (dst != src) + dst[i] = src[i]; + + u128_to_be128(&ctrblks[i], iv); + u128_inc(iv); + } + + twofish_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks); +} + +static const struct common_glue_ctx twofish_enc = { + .num_funcs = 3, + .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = TWOFISH_PARALLEL_BLOCKS, + .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_xway) } + }, { + .num_blocks = 3, + .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) } + }, { + .num_blocks = 1, + .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) } + } } +}; + +static const struct common_glue_ctx twofish_ctr = { + .num_funcs = 3, + .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = TWOFISH_PARALLEL_BLOCKS, + .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_xway) } + }, { + .num_blocks = 3, + .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) } + }, { + .num_blocks = 1, + .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) } + } } +}; + +static const struct common_glue_ctx twofish_dec = { + .num_funcs = 3, + .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = TWOFISH_PARALLEL_BLOCKS, + .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_xway) } + }, { + .num_blocks = 3, + .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) } + }, { + .num_blocks = 1, + .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) } + } } +}; + +static const struct common_glue_ctx twofish_dec_cbc = { + .num_funcs = 3, + .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS, + + .funcs = { { + .num_blocks = TWOFISH_PARALLEL_BLOCKS, + .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_xway) } + }, { + .num_blocks = 3, + .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) } + }, { + .num_blocks = 1, + .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) } + } } +}; + +static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes); +} + +static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes); +} + +static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc, + dst, src, nbytes); +} + +static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src, + nbytes); +} + +static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes); +} + +static inline bool twofish_fpu_begin(bool fpu_enabled, unsigned int nbytes) +{ + return glue_fpu_begin(TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS, NULL, + fpu_enabled, nbytes); +} + +static inline void twofish_fpu_end(bool fpu_enabled) +{ + glue_fpu_end(fpu_enabled); +} + +struct crypt_priv { + struct twofish_ctx *ctx; + bool fpu_enabled; +}; + +static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) +{ + const unsigned int bsize = TF_BLOCK_SIZE; + struct crypt_priv *ctx = priv; + int i; + + ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes); + + if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) { + twofish_enc_blk_xway(ctx->ctx, srcdst, srcdst); + return; + } + + for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3) + twofish_enc_blk_3way(ctx->ctx, srcdst, srcdst); + + nbytes %= bsize * 3; + + for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) + twofish_enc_blk(ctx->ctx, srcdst, srcdst); +} + +static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) +{ + const unsigned int bsize = TF_BLOCK_SIZE; + struct crypt_priv *ctx = priv; + int i; + + ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes); + + if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) { + twofish_dec_blk_xway(ctx->ctx, srcdst, srcdst); + return; + } + + for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3) + twofish_dec_blk_3way(ctx->ctx, srcdst, srcdst); + + nbytes %= bsize * 3; + + for (i = 0; i < nbytes / bsize; i++, srcdst += bsize) + twofish_dec_blk(ctx->ctx, srcdst, srcdst); +} + +static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + be128 buf[TWOFISH_PARALLEL_BLOCKS]; + struct crypt_priv crypt_ctx = { + .ctx = &ctx->twofish_ctx, + .fpu_enabled = false, + }; + struct lrw_crypt_req req = { + .tbuf = buf, + .tbuflen = sizeof(buf), + + .table_ctx = &ctx->lrw_table, + .crypt_ctx = &crypt_ctx, + .crypt_fn = encrypt_callback, + }; + int ret; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + ret = lrw_crypt(desc, dst, src, nbytes, &req); + twofish_fpu_end(crypt_ctx.fpu_enabled); + + return ret; +} + +static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + be128 buf[TWOFISH_PARALLEL_BLOCKS]; + struct crypt_priv crypt_ctx = { + .ctx = &ctx->twofish_ctx, + .fpu_enabled = false, + }; + struct lrw_crypt_req req = { + .tbuf = buf, + .tbuflen = sizeof(buf), + + .table_ctx = &ctx->lrw_table, + .crypt_ctx = &crypt_ctx, + .crypt_fn = decrypt_callback, + }; + int ret; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + ret = lrw_crypt(desc, dst, src, nbytes, &req); + twofish_fpu_end(crypt_ctx.fpu_enabled); + + return ret; +} + +static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + be128 buf[TWOFISH_PARALLEL_BLOCKS]; + struct crypt_priv crypt_ctx = { + .ctx = &ctx->crypt_ctx, + .fpu_enabled = false, + }; + struct xts_crypt_req req = { + .tbuf = buf, + .tbuflen = sizeof(buf), + + .tweak_ctx = &ctx->tweak_ctx, + .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk), + .crypt_ctx = &crypt_ctx, + .crypt_fn = encrypt_callback, + }; + int ret; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + ret = xts_crypt(desc, dst, src, nbytes, &req); + twofish_fpu_end(crypt_ctx.fpu_enabled); + + return ret; +} + +static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) +{ + struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); + be128 buf[TWOFISH_PARALLEL_BLOCKS]; + struct crypt_priv crypt_ctx = { + .ctx = &ctx->crypt_ctx, + .fpu_enabled = false, + }; + struct xts_crypt_req req = { + .tbuf = buf, + .tbuflen = sizeof(buf), + + .tweak_ctx = &ctx->tweak_ctx, + .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk), + .crypt_ctx = &crypt_ctx, + .crypt_fn = decrypt_callback, + }; + int ret; + + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + ret = xts_crypt(desc, dst, src, nbytes, &req); + twofish_fpu_end(crypt_ctx.fpu_enabled); + + return ret; +} + +static struct crypto_alg twofish_algs[10] = { { + .cra_name = "__ecb-twofish-avx", + .cra_driver_name = "__driver-ecb-twofish-avx", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = TF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct twofish_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(twofish_algs[0].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = TF_MIN_KEY_SIZE, + .max_keysize = TF_MAX_KEY_SIZE, + .setkey = twofish_setkey, + .encrypt = ecb_encrypt, + .decrypt = ecb_decrypt, + }, + }, +}, { + .cra_name = "__cbc-twofish-avx", + .cra_driver_name = "__driver-cbc-twofish-avx", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = TF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct twofish_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(twofish_algs[1].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = TF_MIN_KEY_SIZE, + .max_keysize = TF_MAX_KEY_SIZE, + .setkey = twofish_setkey, + .encrypt = cbc_encrypt, + .decrypt = cbc_decrypt, + }, + }, +}, { + .cra_name = "__ctr-twofish-avx", + .cra_driver_name = "__driver-ctr-twofish-avx", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct twofish_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(twofish_algs[2].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = TF_MIN_KEY_SIZE, + .max_keysize = TF_MAX_KEY_SIZE, + .ivsize = TF_BLOCK_SIZE, + .setkey = twofish_setkey, + .encrypt = ctr_crypt, + .decrypt = ctr_crypt, + }, + }, +}, { + .cra_name = "__lrw-twofish-avx", + .cra_driver_name = "__driver-lrw-twofish-avx", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = TF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct twofish_lrw_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(twofish_algs[3].cra_list), + .cra_exit = lrw_twofish_exit_tfm, + .cra_u = { + .blkcipher = { + .min_keysize = TF_MIN_KEY_SIZE + + TF_BLOCK_SIZE, + .max_keysize = TF_MAX_KEY_SIZE + + TF_BLOCK_SIZE, + .ivsize = TF_BLOCK_SIZE, + .setkey = lrw_twofish_setkey, + .encrypt = lrw_encrypt, + .decrypt = lrw_decrypt, + }, + }, +}, { + .cra_name = "__xts-twofish-avx", + .cra_driver_name = "__driver-xts-twofish-avx", + .cra_priority = 0, + .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER, + .cra_blocksize = TF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct twofish_xts_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_blkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(twofish_algs[4].cra_list), + .cra_u = { + .blkcipher = { + .min_keysize = TF_MIN_KEY_SIZE * 2, + .max_keysize = TF_MAX_KEY_SIZE * 2, + .ivsize = TF_BLOCK_SIZE, + .setkey = xts_twofish_setkey, + .encrypt = xts_encrypt, + .decrypt = xts_decrypt, + }, + }, +}, { + .cra_name = "ecb(twofish)", + .cra_driver_name = "ecb-twofish-avx", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = TF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(twofish_algs[5].cra_list), + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = TF_MIN_KEY_SIZE, + .max_keysize = TF_MAX_KEY_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +}, { + .cra_name = "cbc(twofish)", + .cra_driver_name = "cbc-twofish-avx", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = TF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(twofish_algs[6].cra_list), + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = TF_MIN_KEY_SIZE, + .max_keysize = TF_MAX_KEY_SIZE, + .ivsize = TF_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = __ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +}, { + .cra_name = "ctr(twofish)", + .cra_driver_name = "ctr-twofish-avx", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = 1, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(twofish_algs[7].cra_list), + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = TF_MIN_KEY_SIZE, + .max_keysize = TF_MAX_KEY_SIZE, + .ivsize = TF_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_encrypt, + .geniv = "chainiv", + }, + }, +}, { + .cra_name = "lrw(twofish)", + .cra_driver_name = "lrw-twofish-avx", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = TF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(twofish_algs[8].cra_list), + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = TF_MIN_KEY_SIZE + + TF_BLOCK_SIZE, + .max_keysize = TF_MAX_KEY_SIZE + + TF_BLOCK_SIZE, + .ivsize = TF_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +}, { + .cra_name = "xts(twofish)", + .cra_driver_name = "xts-twofish-avx", + .cra_priority = 400, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = TF_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct async_helper_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(twofish_algs[9].cra_list), + .cra_init = ablk_init, + .cra_exit = ablk_exit, + .cra_u = { + .ablkcipher = { + .min_keysize = TF_MIN_KEY_SIZE * 2, + .max_keysize = TF_MAX_KEY_SIZE * 2, + .ivsize = TF_BLOCK_SIZE, + .setkey = ablk_set_key, + .encrypt = ablk_encrypt, + .decrypt = ablk_decrypt, + }, + }, +} }; + +static int __init twofish_init(void) +{ + u64 xcr0; + + if (!cpu_has_avx || !cpu_has_osxsave) { + printk(KERN_INFO "AVX instructions are not detected.\n"); + return -ENODEV; + } + + xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK); + if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) { + printk(KERN_INFO "AVX detected but unusable.\n"); + return -ENODEV; + } + + return crypto_register_algs(twofish_algs, ARRAY_SIZE(twofish_algs)); +} + +static void __exit twofish_exit(void) +{ + crypto_unregister_algs(twofish_algs, ARRAY_SIZE(twofish_algs)); +} + +module_init(twofish_init); +module_exit(twofish_exit); + +MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("twofish"); diff --git a/arch/x86/crypto/twofish_glue_3way.c b/arch/x86/crypto/twofish_glue_3way.c index 922ab24cce31..15f9347316c8 100644 --- a/arch/x86/crypto/twofish_glue_3way.c +++ b/arch/x86/crypto/twofish_glue_3way.c @@ -3,11 +3,6 @@ * * Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi> * - * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by: - * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au> - * CTR part based on code (crypto/ctr.c) by: - * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com> - * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License, or @@ -33,20 +28,13 @@ #include <crypto/algapi.h> #include <crypto/twofish.h> #include <crypto/b128ops.h> +#include <asm/crypto/twofish.h> +#include <asm/crypto/glue_helper.h> #include <crypto/lrw.h> #include <crypto/xts.h> -/* regular block cipher functions from twofish_x86_64 module */ -asmlinkage void twofish_enc_blk(struct twofish_ctx *ctx, u8 *dst, - const u8 *src); -asmlinkage void twofish_dec_blk(struct twofish_ctx *ctx, u8 *dst, - const u8 *src); - -/* 3-way parallel cipher functions */ -asmlinkage void __twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst, - const u8 *src, bool xor); -asmlinkage void twofish_dec_blk_3way(struct twofish_ctx *ctx, u8 *dst, - const u8 *src); +EXPORT_SYMBOL_GPL(__twofish_enc_blk_3way); +EXPORT_SYMBOL_GPL(twofish_dec_blk_3way); static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst, const u8 *src) @@ -60,311 +48,139 @@ static inline void twofish_enc_blk_xor_3way(struct twofish_ctx *ctx, u8 *dst, __twofish_enc_blk_3way(ctx, dst, src, true); } -static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk, - void (*fn)(struct twofish_ctx *, u8 *, const u8 *), - void (*fn_3way)(struct twofish_ctx *, u8 *, const u8 *)) -{ - struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - unsigned int bsize = TF_BLOCK_SIZE; - unsigned int nbytes; - int err; - - err = blkcipher_walk_virt(desc, walk); - - while ((nbytes = walk->nbytes)) { - u8 *wsrc = walk->src.virt.addr; - u8 *wdst = walk->dst.virt.addr; - - /* Process three block batch */ - if (nbytes >= bsize * 3) { - do { - fn_3way(ctx, wdst, wsrc); - - wsrc += bsize * 3; - wdst += bsize * 3; - nbytes -= bsize * 3; - } while (nbytes >= bsize * 3); - - if (nbytes < bsize) - goto done; - } - - /* Handle leftovers */ - do { - fn(ctx, wdst, wsrc); - - wsrc += bsize; - wdst += bsize; - nbytes -= bsize; - } while (nbytes >= bsize); - -done: - err = blkcipher_walk_done(desc, walk, nbytes); - } - - return err; -} - -static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) +void twofish_dec_blk_cbc_3way(void *ctx, u128 *dst, const u128 *src) { - struct blkcipher_walk walk; + u128 ivs[2]; - blkcipher_walk_init(&walk, dst, src, nbytes); - return ecb_crypt(desc, &walk, twofish_enc_blk, twofish_enc_blk_3way); -} + ivs[0] = src[0]; + ivs[1] = src[1]; -static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) -{ - struct blkcipher_walk walk; + twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src); - blkcipher_walk_init(&walk, dst, src, nbytes); - return ecb_crypt(desc, &walk, twofish_dec_blk, twofish_dec_blk_3way); + u128_xor(&dst[1], &dst[1], &ivs[0]); + u128_xor(&dst[2], &dst[2], &ivs[1]); } +EXPORT_SYMBOL_GPL(twofish_dec_blk_cbc_3way); -static unsigned int __cbc_encrypt(struct blkcipher_desc *desc, - struct blkcipher_walk *walk) -{ - struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - unsigned int bsize = TF_BLOCK_SIZE; - unsigned int nbytes = walk->nbytes; - u128 *src = (u128 *)walk->src.virt.addr; - u128 *dst = (u128 *)walk->dst.virt.addr; - u128 *iv = (u128 *)walk->iv; - - do { - u128_xor(dst, src, iv); - twofish_enc_blk(ctx, (u8 *)dst, (u8 *)dst); - iv = dst; - - src += 1; - dst += 1; - nbytes -= bsize; - } while (nbytes >= bsize); - - u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv); - return nbytes; -} - -static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) +void twofish_enc_blk_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv) { - struct blkcipher_walk walk; - int err; + be128 ctrblk; - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); + if (dst != src) + *dst = *src; - while ((nbytes = walk.nbytes)) { - nbytes = __cbc_encrypt(desc, &walk); - err = blkcipher_walk_done(desc, &walk, nbytes); - } + u128_to_be128(&ctrblk, iv); + u128_inc(iv); - return err; + twofish_enc_blk(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk); + u128_xor(dst, dst, (u128 *)&ctrblk); } +EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr); -static unsigned int __cbc_decrypt(struct blkcipher_desc *desc, - struct blkcipher_walk *walk) +void twofish_enc_blk_ctr_3way(void *ctx, u128 *dst, const u128 *src, + u128 *iv) { - struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - unsigned int bsize = TF_BLOCK_SIZE; - unsigned int nbytes = walk->nbytes; - u128 *src = (u128 *)walk->src.virt.addr; - u128 *dst = (u128 *)walk->dst.virt.addr; - u128 ivs[3 - 1]; - u128 last_iv; - - /* Start of the last block. */ - src += nbytes / bsize - 1; - dst += nbytes / bsize - 1; - - last_iv = *src; - - /* Process three block batch */ - if (nbytes >= bsize * 3) { - do { - nbytes -= bsize * (3 - 1); - src -= 3 - 1; - dst -= 3 - 1; - - ivs[0] = src[0]; - ivs[1] = src[1]; - - twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src); - - u128_xor(dst + 1, dst + 1, ivs + 0); - u128_xor(dst + 2, dst + 2, ivs + 1); - - nbytes -= bsize; - if (nbytes < bsize) - goto done; - - u128_xor(dst, dst, src - 1); - src -= 1; - dst -= 1; - } while (nbytes >= bsize * 3); - - if (nbytes < bsize) - goto done; - } - - /* Handle leftovers */ - for (;;) { - twofish_dec_blk(ctx, (u8 *)dst, (u8 *)src); - - nbytes -= bsize; - if (nbytes < bsize) - break; + be128 ctrblks[3]; - u128_xor(dst, dst, src - 1); - src -= 1; - dst -= 1; + if (dst != src) { + dst[0] = src[0]; + dst[1] = src[1]; + dst[2] = src[2]; } -done: - u128_xor(dst, dst, (u128 *)walk->iv); - *(u128 *)walk->iv = last_iv; + u128_to_be128(&ctrblks[0], iv); + u128_inc(iv); + u128_to_be128(&ctrblks[1], iv); + u128_inc(iv); + u128_to_be128(&ctrblks[2], iv); + u128_inc(iv); - return nbytes; + twofish_enc_blk_xor_3way(ctx, (u8 *)dst, (u8 *)ctrblks); } +EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr_3way); + +static const struct common_glue_ctx twofish_enc = { + .num_funcs = 2, + .fpu_blocks_limit = -1, + + .funcs = { { + .num_blocks = 3, + .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) } + }, { + .num_blocks = 1, + .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) } + } } +}; -static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, - struct scatterlist *src, unsigned int nbytes) -{ - struct blkcipher_walk walk; - int err; - - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt(desc, &walk); +static const struct common_glue_ctx twofish_ctr = { + .num_funcs = 2, + .fpu_blocks_limit = -1, + + .funcs = { { + .num_blocks = 3, + .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_ctr_3way) } + }, { + .num_blocks = 1, + .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_ctr) } + } } +}; - while ((nbytes = walk.nbytes)) { - nbytes = __cbc_decrypt(desc, &walk); - err = blkcipher_walk_done(desc, &walk, nbytes); - } +static const struct common_glue_ctx twofish_dec = { + .num_funcs = 2, + .fpu_blocks_limit = -1, + + .funcs = { { + .num_blocks = 3, + .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) } + }, { + .num_blocks = 1, + .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) } + } } +}; - return err; -} +static const struct common_glue_ctx twofish_dec_cbc = { + .num_funcs = 2, + .fpu_blocks_limit = -1, + + .funcs = { { + .num_blocks = 3, + .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) } + }, { + .num_blocks = 1, + .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) } + } } +}; -static inline void u128_to_be128(be128 *dst, const u128 *src) +static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) { - dst->a = cpu_to_be64(src->a); - dst->b = cpu_to_be64(src->b); + return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes); } -static inline void be128_to_u128(u128 *dst, const be128 *src) +static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) { - dst->a = be64_to_cpu(src->a); - dst->b = be64_to_cpu(src->b); + return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes); } -static inline void u128_inc(u128 *i) +static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) { - i->b++; - if (!i->b) - i->a++; + return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc, + dst, src, nbytes); } -static void ctr_crypt_final(struct blkcipher_desc *desc, - struct blkcipher_walk *walk) +static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes) { - struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - u8 *ctrblk = walk->iv; - u8 keystream[TF_BLOCK_SIZE]; - u8 *src = walk->src.virt.addr; - u8 *dst = walk->dst.virt.addr; - unsigned int nbytes = walk->nbytes; - - twofish_enc_blk(ctx, keystream, ctrblk); - crypto_xor(keystream, src, nbytes); - memcpy(dst, keystream, nbytes); - - crypto_inc(ctrblk, TF_BLOCK_SIZE); -} - -static unsigned int __ctr_crypt(struct blkcipher_desc *desc, - struct blkcipher_walk *walk) -{ - struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm); - unsigned int bsize = TF_BLOCK_SIZE; - unsigned int nbytes = walk->nbytes; - u128 *src = (u128 *)walk->src.virt.addr; - u128 *dst = (u128 *)walk->dst.virt.addr; - u128 ctrblk; - be128 ctrblocks[3]; - - be128_to_u128(&ctrblk, (be128 *)walk->iv); - - /* Process three block batch */ - if (nbytes >= bsize * 3) { - do { - if (dst != src) { - dst[0] = src[0]; - dst[1] = src[1]; - dst[2] = src[2]; - } - - /* create ctrblks for parallel encrypt */ - u128_to_be128(&ctrblocks[0], &ctrblk); - u128_inc(&ctrblk); - u128_to_be128(&ctrblocks[1], &ctrblk); - u128_inc(&ctrblk); - u128_to_be128(&ctrblocks[2], &ctrblk); - u128_inc(&ctrblk); - - twofish_enc_blk_xor_3way(ctx, (u8 *)dst, - (u8 *)ctrblocks); - - src += 3; - dst += 3; - nbytes -= bsize * 3; - } while (nbytes >= bsize * 3); - - if (nbytes < bsize) - goto done; - } - - /* Handle leftovers */ - do { - if (dst != src) - *dst = *src; - - u128_to_be128(&ctrblocks[0], &ctrblk); - u128_inc(&ctrblk); - - twofish_enc_blk(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks); - u128_xor(dst, dst, (u128 *)ctrblocks); - - src += 1; - dst += 1; - nbytes -= bsize; - } while (nbytes >= bsize); - -done: - u128_to_be128((be128 *)walk->iv, &ctrblk); - return nbytes; + return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src, + nbytes); } static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { - struct blkcipher_walk walk; - int err; - - blkcipher_walk_init(&walk, dst, src, nbytes); - err = blkcipher_walk_virt_block(desc, &walk, TF_BLOCK_SIZE); - - while ((nbytes = walk.nbytes) >= TF_BLOCK_SIZE) { - nbytes = __ctr_crypt(desc, &walk); - err = blkcipher_walk_done(desc, &walk, nbytes); - } - - if (walk.nbytes) { - ctr_crypt_final(desc, &walk); - err = blkcipher_walk_done(desc, &walk, 0); - } - - return err; + return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes); } static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) @@ -397,13 +213,8 @@ static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes) twofish_dec_blk(ctx, srcdst, srcdst); } -struct twofish_lrw_ctx { - struct lrw_table_ctx lrw_table; - struct twofish_ctx twofish_ctx; -}; - -static int lrw_twofish_setkey(struct crypto_tfm *tfm, const u8 *key, - unsigned int keylen) +int lrw_twofish_setkey(struct crypto_tfm *tfm, const u8 *key, + unsigned int keylen) { struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm); int err; @@ -415,6 +226,7 @@ static int lrw_twofish_setkey(struct crypto_tfm *tfm, const u8 *key, return lrw_init_table(&ctx->lrw_table, key + keylen - TF_BLOCK_SIZE); } +EXPORT_SYMBOL_GPL(lrw_twofish_setkey); static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) @@ -450,20 +262,16 @@ static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, return lrw_crypt(desc, dst, src, nbytes, &req); } -static void lrw_exit_tfm(struct crypto_tfm *tfm) +void lrw_twofish_exit_tfm(struct crypto_tfm *tfm) { struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm); lrw_free_table(&ctx->lrw_table); } +EXPORT_SYMBOL_GPL(lrw_twofish_exit_tfm); -struct twofish_xts_ctx { - struct twofish_ctx tweak_ctx; - struct twofish_ctx crypt_ctx; -}; - -static int xts_twofish_setkey(struct crypto_tfm *tfm, const u8 *key, - unsigned int keylen) +int xts_twofish_setkey(struct crypto_tfm *tfm, const u8 *key, + unsigned int keylen) { struct twofish_xts_ctx *ctx = crypto_tfm_ctx(tfm); u32 *flags = &tfm->crt_flags; @@ -486,6 +294,7 @@ static int xts_twofish_setkey(struct crypto_tfm *tfm, const u8 *key, return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2, flags); } +EXPORT_SYMBOL_GPL(xts_twofish_setkey); static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) @@ -596,7 +405,7 @@ static struct crypto_alg tf_algs[5] = { { .cra_type = &crypto_blkcipher_type, .cra_module = THIS_MODULE, .cra_list = LIST_HEAD_INIT(tf_algs[3].cra_list), - .cra_exit = lrw_exit_tfm, + .cra_exit = lrw_twofish_exit_tfm, .cra_u = { .blkcipher = { .min_keysize = TF_MIN_KEY_SIZE + TF_BLOCK_SIZE, diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 3ea51a84a0e4..f34261296ffb 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -546,7 +546,7 @@ static inline const struct cpumask *online_target_cpus(void) return cpu_online_mask; } -DECLARE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid); +DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid); static inline unsigned int read_apic_id(void) diff --git a/arch/x86/include/asm/bootparam.h b/arch/x86/include/asm/bootparam.h index eb45aa6b1f27..2ad874cb661c 100644 --- a/arch/x86/include/asm/bootparam.h +++ b/arch/x86/include/asm/bootparam.h @@ -66,6 +66,7 @@ struct setup_header { __u64 setup_data; __u64 pref_address; __u32 init_size; + __u32 handover_offset; } __attribute__((packed)); struct sys_desc_table { diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h index f91e80f4f180..6b7ee5ff6820 100644 --- a/arch/x86/include/asm/cpufeature.h +++ b/arch/x86/include/asm/cpufeature.h @@ -207,6 +207,8 @@ #define X86_FEATURE_ERMS (9*32+ 9) /* Enhanced REP MOVSB/STOSB */ #define X86_FEATURE_INVPCID (9*32+10) /* Invalidate Processor Context ID */ #define X86_FEATURE_RTM (9*32+11) /* Restricted Transactional Memory */ +#define X86_FEATURE_RDSEED (9*32+18) /* The RDSEED instruction */ +#define X86_FEATURE_ADX (9*32+19) /* The ADCX and ADOX instructions */ #if defined(__KERNEL__) && !defined(__ASSEMBLY__) diff --git a/arch/x86/include/asm/crypto/ablk_helper.h b/arch/x86/include/asm/crypto/ablk_helper.h new file mode 100644 index 000000000000..4f93df50c23e --- /dev/null +++ b/arch/x86/include/asm/crypto/ablk_helper.h @@ -0,0 +1,31 @@ +/* + * Shared async block cipher helpers + */ + +#ifndef _CRYPTO_ABLK_HELPER_H +#define _CRYPTO_ABLK_HELPER_H + +#include <linux/crypto.h> +#include <linux/kernel.h> +#include <crypto/cryptd.h> + +struct async_helper_ctx { + struct cryptd_ablkcipher *cryptd_tfm; +}; + +extern int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key, + unsigned int key_len); + +extern int __ablk_encrypt(struct ablkcipher_request *req); + +extern int ablk_encrypt(struct ablkcipher_request *req); + +extern int ablk_decrypt(struct ablkcipher_request *req); + +extern void ablk_exit(struct crypto_tfm *tfm); + +extern int ablk_init_common(struct crypto_tfm *tfm, const char *drv_name); + +extern int ablk_init(struct crypto_tfm *tfm); + +#endif /* _CRYPTO_ABLK_HELPER_H */ diff --git a/arch/x86/include/asm/aes.h b/arch/x86/include/asm/crypto/aes.h index 80545a1cbe39..80545a1cbe39 100644 --- a/arch/x86/include/asm/aes.h +++ b/arch/x86/include/asm/crypto/aes.h diff --git a/arch/x86/include/asm/crypto/glue_helper.h b/arch/x86/include/asm/crypto/glue_helper.h new file mode 100644 index 000000000000..3e408bddc96f --- /dev/null +++ b/arch/x86/include/asm/crypto/glue_helper.h @@ -0,0 +1,115 @@ +/* + * Shared glue code for 128bit block ciphers + */ + +#ifndef _CRYPTO_GLUE_HELPER_H +#define _CRYPTO_GLUE_HELPER_H + +#include <linux/kernel.h> +#include <linux/crypto.h> +#include <asm/i387.h> +#include <crypto/b128ops.h> + +typedef void (*common_glue_func_t)(void *ctx, u8 *dst, const u8 *src); +typedef void (*common_glue_cbc_func_t)(void *ctx, u128 *dst, const u128 *src); +typedef void (*common_glue_ctr_func_t)(void *ctx, u128 *dst, const u128 *src, + u128 *iv); + +#define GLUE_FUNC_CAST(fn) ((common_glue_func_t)(fn)) +#define GLUE_CBC_FUNC_CAST(fn) ((common_glue_cbc_func_t)(fn)) +#define GLUE_CTR_FUNC_CAST(fn) ((common_glue_ctr_func_t)(fn)) + +struct common_glue_func_entry { + unsigned int num_blocks; /* number of blocks that @fn will process */ + union { + common_glue_func_t ecb; + common_glue_cbc_func_t cbc; + common_glue_ctr_func_t ctr; + } fn_u; +}; + +struct common_glue_ctx { + unsigned int num_funcs; + int fpu_blocks_limit; /* -1 means fpu not needed at all */ + + /* + * First funcs entry must have largest num_blocks and last funcs entry + * must have num_blocks == 1! + */ + struct common_glue_func_entry funcs[]; +}; + +static inline bool glue_fpu_begin(unsigned int bsize, int fpu_blocks_limit, + struct blkcipher_desc *desc, + bool fpu_enabled, unsigned int nbytes) +{ + if (likely(fpu_blocks_limit < 0)) + return false; + + if (fpu_enabled) + return true; + + /* + * Vector-registers are only used when chunk to be processed is large + * enough, so do not enable FPU until it is necessary. + */ + if (nbytes < bsize * (unsigned int)fpu_blocks_limit) + return false; + + if (desc) { + /* prevent sleeping if FPU is in use */ + desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + } + + kernel_fpu_begin(); + return true; +} + +static inline void glue_fpu_end(bool fpu_enabled) +{ + if (fpu_enabled) + kernel_fpu_end(); +} + +static inline void u128_to_be128(be128 *dst, const u128 *src) +{ + dst->a = cpu_to_be64(src->a); + dst->b = cpu_to_be64(src->b); +} + +static inline void be128_to_u128(u128 *dst, const be128 *src) +{ + dst->a = be64_to_cpu(src->a); + dst->b = be64_to_cpu(src->b); +} + +static inline void u128_inc(u128 *i) +{ + i->b++; + if (!i->b) + i->a++; +} + +extern int glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx, + struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes); + +extern int glue_cbc_encrypt_128bit(const common_glue_func_t fn, + struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, + unsigned int nbytes); + +extern int glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx, + struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, + unsigned int nbytes); + +extern int glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx, + struct blkcipher_desc *desc, + struct scatterlist *dst, + struct scatterlist *src, unsigned int nbytes); + +#endif /* _CRYPTO_GLUE_HELPER_H */ diff --git a/arch/x86/include/asm/crypto/serpent-avx.h b/arch/x86/include/asm/crypto/serpent-avx.h new file mode 100644 index 000000000000..432deedd2945 --- /dev/null +++ b/arch/x86/include/asm/crypto/serpent-avx.h @@ -0,0 +1,32 @@ +#ifndef ASM_X86_SERPENT_AVX_H +#define ASM_X86_SERPENT_AVX_H + +#include <linux/crypto.h> +#include <crypto/serpent.h> + +#define SERPENT_PARALLEL_BLOCKS 8 + +asmlinkage void __serpent_enc_blk_8way_avx(struct serpent_ctx *ctx, u8 *dst, + const u8 *src, bool xor); +asmlinkage void serpent_dec_blk_8way_avx(struct serpent_ctx *ctx, u8 *dst, + const u8 *src); + +static inline void serpent_enc_blk_xway(struct serpent_ctx *ctx, u8 *dst, + const u8 *src) +{ + __serpent_enc_blk_8way_avx(ctx, dst, src, false); +} + +static inline void serpent_enc_blk_xway_xor(struct serpent_ctx *ctx, u8 *dst, + const u8 *src) +{ + __serpent_enc_blk_8way_avx(ctx, dst, src, true); +} + +static inline void serpent_dec_blk_xway(struct serpent_ctx *ctx, u8 *dst, + const u8 *src) +{ + serpent_dec_blk_8way_avx(ctx, dst, src); +} + +#endif diff --git a/arch/x86/include/asm/serpent.h b/arch/x86/include/asm/crypto/serpent-sse2.h index d3ef63fe0c81..e6e77dffbdab 100644 --- a/arch/x86/include/asm/serpent.h +++ b/arch/x86/include/asm/crypto/serpent-sse2.h @@ -1,5 +1,5 @@ -#ifndef ASM_X86_SERPENT_H -#define ASM_X86_SERPENT_H +#ifndef ASM_X86_SERPENT_SSE2_H +#define ASM_X86_SERPENT_SSE2_H #include <linux/crypto.h> #include <crypto/serpent.h> diff --git a/arch/x86/include/asm/crypto/twofish.h b/arch/x86/include/asm/crypto/twofish.h new file mode 100644 index 000000000000..9d2c514bd5f9 --- /dev/null +++ b/arch/x86/include/asm/crypto/twofish.h @@ -0,0 +1,46 @@ +#ifndef ASM_X86_TWOFISH_H +#define ASM_X86_TWOFISH_H + +#include <linux/crypto.h> +#include <crypto/twofish.h> +#include <crypto/lrw.h> +#include <crypto/b128ops.h> + +struct twofish_lrw_ctx { + struct lrw_table_ctx lrw_table; + struct twofish_ctx twofish_ctx; +}; + +struct twofish_xts_ctx { + struct twofish_ctx tweak_ctx; + struct twofish_ctx crypt_ctx; +}; + +/* regular block cipher functions from twofish_x86_64 module */ +asmlinkage void twofish_enc_blk(struct twofish_ctx *ctx, u8 *dst, + const u8 *src); +asmlinkage void twofish_dec_blk(struct twofish_ctx *ctx, u8 *dst, + const u8 *src); + +/* 3-way parallel cipher functions */ +asmlinkage void __twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst, + const u8 *src, bool xor); +asmlinkage void twofish_dec_blk_3way(struct twofish_ctx *ctx, u8 *dst, + const u8 *src); + +/* helpers from twofish_x86_64-3way module */ +extern void twofish_dec_blk_cbc_3way(void *ctx, u128 *dst, const u128 *src); +extern void twofish_enc_blk_ctr(void *ctx, u128 *dst, const u128 *src, + u128 *iv); +extern void twofish_enc_blk_ctr_3way(void *ctx, u128 *dst, const u128 *src, + u128 *iv); + +extern int lrw_twofish_setkey(struct crypto_tfm *tfm, const u8 *key, + unsigned int keylen); + +extern void lrw_twofish_exit_tfm(struct crypto_tfm *tfm); + +extern int xts_twofish_setkey(struct crypto_tfm *tfm, const u8 *key, + unsigned int keylen); + +#endif /* ASM_X86_TWOFISH_H */ diff --git a/arch/x86/include/asm/entry_arch.h b/arch/x86/include/asm/entry_arch.h index 0baa628e330c..40afa0005c69 100644 --- a/arch/x86/include/asm/entry_arch.h +++ b/arch/x86/include/asm/entry_arch.h @@ -15,15 +15,6 @@ BUILD_INTERRUPT(call_function_interrupt,CALL_FUNCTION_VECTOR) BUILD_INTERRUPT(call_function_single_interrupt,CALL_FUNCTION_SINGLE_VECTOR) BUILD_INTERRUPT(irq_move_cleanup_interrupt,IRQ_MOVE_CLEANUP_VECTOR) BUILD_INTERRUPT(reboot_interrupt,REBOOT_VECTOR) - -.irp idx,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, \ - 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 -.if NUM_INVALIDATE_TLB_VECTORS > \idx -BUILD_INTERRUPT3(invalidate_interrupt\idx, - (INVALIDATE_TLB_VECTOR_START)+\idx, - smp_invalidate_interrupt) -.endif -.endr #endif BUILD_INTERRUPT(x86_platform_ipi, X86_PLATFORM_IPI_VECTOR) diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h index 4b4448761e88..1508e518c7e3 100644 --- a/arch/x86/include/asm/irq_vectors.h +++ b/arch/x86/include/asm/irq_vectors.h @@ -119,17 +119,6 @@ */ #define LOCAL_TIMER_VECTOR 0xef -/* up to 32 vectors used for spreading out TLB flushes: */ -#if NR_CPUS <= 32 -# define NUM_INVALIDATE_TLB_VECTORS (NR_CPUS) -#else -# define NUM_INVALIDATE_TLB_VECTORS (32) -#endif - -#define INVALIDATE_TLB_VECTOR_END (0xee) -#define INVALIDATE_TLB_VECTOR_START \ - (INVALIDATE_TLB_VECTOR_END-NUM_INVALIDATE_TLB_VECTORS+1) - #define NR_VECTORS 256 #define FPU_IRQ 13 diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h index 0b47ddb6f00b..a0facf3908d7 100644 --- a/arch/x86/include/asm/paravirt.h +++ b/arch/x86/include/asm/paravirt.h @@ -360,9 +360,10 @@ static inline void __flush_tlb_single(unsigned long addr) static inline void flush_tlb_others(const struct cpumask *cpumask, struct mm_struct *mm, - unsigned long va) + unsigned long start, + unsigned long end) { - PVOP_VCALL3(pv_mmu_ops.flush_tlb_others, cpumask, mm, va); + PVOP_VCALL4(pv_mmu_ops.flush_tlb_others, cpumask, mm, start, end); } static inline int paravirt_pgd_alloc(struct mm_struct *mm) diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h index 8613cbb7ba41..142236ed83af 100644 --- a/arch/x86/include/asm/paravirt_types.h +++ b/arch/x86/include/asm/paravirt_types.h @@ -248,7 +248,8 @@ struct pv_mmu_ops { void (*flush_tlb_single)(unsigned long addr); void (*flush_tlb_others)(const struct cpumask *cpus, struct mm_struct *mm, - unsigned long va); + unsigned long start, + unsigned long end); /* Hooks for allocating and freeing a pagetable top-level */ int (*pgd_alloc)(struct mm_struct *mm); diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h index d9b8e3f7f42a..1104afaba52b 100644 --- a/arch/x86/include/asm/percpu.h +++ b/arch/x86/include/asm/percpu.h @@ -551,6 +551,12 @@ DECLARE_PER_CPU(unsigned long, this_cpu_off); { [0 ... NR_CPUS-1] = _initvalue }; \ __typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map +#define DEFINE_EARLY_PER_CPU_READ_MOSTLY(_type, _name, _initvalue) \ + DEFINE_PER_CPU_READ_MOSTLY(_type, _name) = _initvalue; \ + __typeof__(_type) _name##_early_map[NR_CPUS] __initdata = \ + { [0 ... NR_CPUS-1] = _initvalue }; \ + __typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map + #define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \ EXPORT_PER_CPU_SYMBOL(_name) @@ -559,6 +565,11 @@ DECLARE_PER_CPU(unsigned long, this_cpu_off); extern __typeof__(_type) *_name##_early_ptr; \ extern __typeof__(_type) _name##_early_map[] +#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name) \ + DECLARE_PER_CPU_READ_MOSTLY(_type, _name); \ + extern __typeof__(_type) *_name##_early_ptr; \ + extern __typeof__(_type) _name##_early_map[] + #define early_per_cpu_ptr(_name) (_name##_early_ptr) #define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx]) #define early_per_cpu(_name, _cpu) \ @@ -570,12 +581,18 @@ DECLARE_PER_CPU(unsigned long, this_cpu_off); #define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \ DEFINE_PER_CPU(_type, _name) = _initvalue +#define DEFINE_EARLY_PER_CPU_READ_MOSTLY(_type, _name, _initvalue) \ + DEFINE_PER_CPU_READ_MOSTLY(_type, _name) = _initvalue + #define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \ EXPORT_PER_CPU_SYMBOL(_name) #define DECLARE_EARLY_PER_CPU(_type, _name) \ DECLARE_PER_CPU(_type, _name) +#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name) \ + DECLARE_PER_CPU_READ_MOSTLY(_type, _name) + #define early_per_cpu(_name, _cpu) per_cpu(_name, _cpu) #define early_per_cpu_ptr(_name) NULL /* no early_per_cpu_map() */ diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 39bc5777211a..d048cad9bcad 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -61,6 +61,19 @@ static inline void *current_text_addr(void) # define ARCH_MIN_MMSTRUCT_ALIGN 0 #endif +enum tlb_infos { + ENTRIES, + NR_INFO +}; + +extern u16 __read_mostly tlb_lli_4k[NR_INFO]; +extern u16 __read_mostly tlb_lli_2m[NR_INFO]; +extern u16 __read_mostly tlb_lli_4m[NR_INFO]; +extern u16 __read_mostly tlb_lld_4k[NR_INFO]; +extern u16 __read_mostly tlb_lld_2m[NR_INFO]; +extern u16 __read_mostly tlb_lld_4m[NR_INFO]; +extern s8 __read_mostly tlb_flushall_shift; + /* * CPU type and hardware bug flags. Kept separately for each CPU. * Members of this structure are referenced in head.S, so think twice diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h index 2ffa95dc2333..4f19a1526037 100644 --- a/arch/x86/include/asm/smp.h +++ b/arch/x86/include/asm/smp.h @@ -31,12 +31,12 @@ static inline bool cpu_has_ht_siblings(void) return has_siblings; } -DECLARE_PER_CPU(cpumask_var_t, cpu_sibling_map); -DECLARE_PER_CPU(cpumask_var_t, cpu_core_map); +DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map); +DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map); /* cpus sharing the last level cache: */ -DECLARE_PER_CPU(cpumask_var_t, cpu_llc_shared_map); -DECLARE_PER_CPU(u16, cpu_llc_id); -DECLARE_PER_CPU(int, cpu_number); +DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map); +DECLARE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id); +DECLARE_PER_CPU_READ_MOSTLY(int, cpu_number); static inline struct cpumask *cpu_sibling_mask(int cpu) { @@ -53,10 +53,10 @@ static inline struct cpumask *cpu_llc_shared_mask(int cpu) return per_cpu(cpu_llc_shared_map, cpu); } -DECLARE_EARLY_PER_CPU(u16, x86_cpu_to_apicid); -DECLARE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid); +DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid); +DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid); #if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86_32) -DECLARE_EARLY_PER_CPU(int, x86_cpu_to_logical_apicid); +DECLARE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid); #endif /* Static state in head.S used to set up a CPU */ diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h index 829215fef9ee..4fef20773b8f 100644 --- a/arch/x86/include/asm/tlb.h +++ b/arch/x86/include/asm/tlb.h @@ -4,7 +4,14 @@ #define tlb_start_vma(tlb, vma) do { } while (0) #define tlb_end_vma(tlb, vma) do { } while (0) #define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0) -#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm) + +#define tlb_flush(tlb) \ +{ \ + if (tlb->fullmm == 0) \ + flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end, 0UL); \ + else \ + flush_tlb_mm_range(tlb->mm, 0UL, TLB_FLUSH_ALL, 0UL); \ +} #include <asm-generic/tlb.h> diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h index 36a1a2ab87d2..74a44333545a 100644 --- a/arch/x86/include/asm/tlbflush.h +++ b/arch/x86/include/asm/tlbflush.h @@ -73,14 +73,10 @@ static inline void __flush_tlb_one(unsigned long addr) * - flush_tlb_page(vma, vmaddr) flushes one page * - flush_tlb_range(vma, start, end) flushes a range of pages * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages - * - flush_tlb_others(cpumask, mm, va) flushes TLBs on other cpus + * - flush_tlb_others(cpumask, mm, start, end) flushes TLBs on other cpus * * ..but the i386 has somewhat limited tlb flushing capabilities, * and page-granular flushes are available only on i486 and up. - * - * x86-64 can only flush individual pages or full VMs. For a range flush - * we always do the full VM. Might be worth trying if for a small - * range a few INVLPGs in a row are a win. */ #ifndef CONFIG_SMP @@ -109,9 +105,17 @@ static inline void flush_tlb_range(struct vm_area_struct *vma, __flush_tlb(); } +static inline void flush_tlb_mm_range(struct mm_struct *mm, + unsigned long start, unsigned long end, unsigned long vmflag) +{ + if (mm == current->active_mm) + __flush_tlb(); +} + static inline void native_flush_tlb_others(const struct cpumask *cpumask, struct mm_struct *mm, - unsigned long va) + unsigned long start, + unsigned long end) { } @@ -119,27 +123,35 @@ static inline void reset_lazy_tlbstate(void) { } +static inline void flush_tlb_kernel_range(unsigned long start, + unsigned long end) +{ + flush_tlb_all(); +} + #else /* SMP */ #include <asm/smp.h> #define local_flush_tlb() __flush_tlb() +#define flush_tlb_mm(mm) flush_tlb_mm_range(mm, 0UL, TLB_FLUSH_ALL, 0UL) + +#define flush_tlb_range(vma, start, end) \ + flush_tlb_mm_range(vma->vm_mm, start, end, vma->vm_flags) + extern void flush_tlb_all(void); extern void flush_tlb_current_task(void); -extern void flush_tlb_mm(struct mm_struct *); extern void flush_tlb_page(struct vm_area_struct *, unsigned long); +extern void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, + unsigned long end, unsigned long vmflag); +extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); #define flush_tlb() flush_tlb_current_task() -static inline void flush_tlb_range(struct vm_area_struct *vma, - unsigned long start, unsigned long end) -{ - flush_tlb_mm(vma->vm_mm); -} - void native_flush_tlb_others(const struct cpumask *cpumask, - struct mm_struct *mm, unsigned long va); + struct mm_struct *mm, + unsigned long start, unsigned long end); #define TLBSTATE_OK 1 #define TLBSTATE_LAZY 2 @@ -159,13 +171,8 @@ static inline void reset_lazy_tlbstate(void) #endif /* SMP */ #ifndef CONFIG_PARAVIRT -#define flush_tlb_others(mask, mm, va) native_flush_tlb_others(mask, mm, va) +#define flush_tlb_others(mask, mm, start, end) \ + native_flush_tlb_others(mask, mm, start, end) #endif -static inline void flush_tlb_kernel_range(unsigned long start, - unsigned long end) -{ - flush_tlb_all(); -} - #endif /* _ASM_X86_TLBFLUSH_H */ diff --git a/arch/x86/include/asm/uv/uv.h b/arch/x86/include/asm/uv/uv.h index 3bb9491b7659..b47c2a82ff15 100644 --- a/arch/x86/include/asm/uv/uv.h +++ b/arch/x86/include/asm/uv/uv.h @@ -15,7 +15,8 @@ extern void uv_nmi_init(void); extern void uv_system_init(void); extern const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask, struct mm_struct *mm, - unsigned long va, + unsigned long start, + unsigned end, unsigned int cpu); #else /* X86_UV */ @@ -26,7 +27,7 @@ static inline void uv_cpu_init(void) { } static inline void uv_system_init(void) { } static inline const struct cpumask * uv_flush_tlb_others(const struct cpumask *cpumask, struct mm_struct *mm, - unsigned long va, unsigned int cpu) + unsigned long start, unsigned long end, unsigned int cpu) { return cpumask; } #endif /* X86_UV */ diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 98e24131ff3a..24deb3082328 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -75,8 +75,8 @@ physid_mask_t phys_cpu_present_map; /* * Map cpu index to physical APIC ID */ -DEFINE_EARLY_PER_CPU(u16, x86_cpu_to_apicid, BAD_APICID); -DEFINE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid, BAD_APICID); +DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid, BAD_APICID); +DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid, BAD_APICID); EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid); EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid); @@ -88,7 +88,7 @@ EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid); * used for the mapping. This is where the behaviors of x86_64 and 32 * actually diverge. Let's keep it ugly for now. */ -DEFINE_EARLY_PER_CPU(int, x86_cpu_to_logical_apicid, BAD_APICID); +DEFINE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid, BAD_APICID); /* * Knob to control our willingness to enable the local APIC. diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile index bac4c3804cc7..d30a6a9a0121 100644 --- a/arch/x86/kernel/cpu/Makefile +++ b/arch/x86/kernel/cpu/Makefile @@ -14,7 +14,7 @@ CFLAGS_common.o := $(nostackp) obj-y := intel_cacheinfo.o scattered.o topology.o obj-y += proc.o capflags.o powerflags.o common.o -obj-y += vmware.o hypervisor.o sched.o mshyperv.o +obj-y += vmware.o hypervisor.o mshyperv.o obj-y += rdrand.o obj-y += match.o diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 5bbc082c47ad..46d8786d655e 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -452,6 +452,35 @@ void __cpuinit cpu_detect_cache_sizes(struct cpuinfo_x86 *c) c->x86_cache_size = l2size; } +u16 __read_mostly tlb_lli_4k[NR_INFO]; +u16 __read_mostly tlb_lli_2m[NR_INFO]; +u16 __read_mostly tlb_lli_4m[NR_INFO]; +u16 __read_mostly tlb_lld_4k[NR_INFO]; +u16 __read_mostly tlb_lld_2m[NR_INFO]; +u16 __read_mostly tlb_lld_4m[NR_INFO]; + +/* + * tlb_flushall_shift shows the balance point in replacing cr3 write + * with multiple 'invlpg'. It will do this replacement when + * flush_tlb_lines <= active_lines/2^tlb_flushall_shift. + * If tlb_flushall_shift is -1, means the replacement will be disabled. + */ +s8 __read_mostly tlb_flushall_shift = -1; + +void __cpuinit cpu_detect_tlb(struct cpuinfo_x86 *c) +{ + if (this_cpu->c_detect_tlb) + this_cpu->c_detect_tlb(c); + + printk(KERN_INFO "Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \ + "Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \ + "tlb_flushall_shift is 0x%x\n", + tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES], + tlb_lli_4m[ENTRIES], tlb_lld_4k[ENTRIES], + tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES], + tlb_flushall_shift); +} + void __cpuinit detect_ht(struct cpuinfo_x86 *c) { #ifdef CONFIG_X86_HT @@ -911,6 +940,8 @@ void __init identify_boot_cpu(void) #else vgetcpu_set_mode(); #endif + if (boot_cpu_data.cpuid_level >= 2) + cpu_detect_tlb(&boot_cpu_data); } void __cpuinit identify_secondary_cpu(struct cpuinfo_x86 *c) diff --git a/arch/x86/kernel/cpu/cpu.h b/arch/x86/kernel/cpu/cpu.h index 8bacc7826fb3..4041c24ae7db 100644 --- a/arch/x86/kernel/cpu/cpu.h +++ b/arch/x86/kernel/cpu/cpu.h @@ -20,10 +20,19 @@ struct cpu_dev { void (*c_bsp_init)(struct cpuinfo_x86 *); void (*c_init)(struct cpuinfo_x86 *); void (*c_identify)(struct cpuinfo_x86 *); + void (*c_detect_tlb)(struct cpuinfo_x86 *); unsigned int (*c_size_cache)(struct cpuinfo_x86 *, unsigned int); int c_x86_vendor; }; +struct _tlb_table { + unsigned char descriptor; + char tlb_type; + unsigned int entries; + /* unsigned int ways; */ + char info[128]; +}; + #define cpu_dev_register(cpu_devX) \ static const struct cpu_dev *const __cpu_dev_##cpu_devX __used \ __attribute__((__section__(".x86_cpu_dev.init"))) = \ diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index 3e6ff6cbf42a..0a4ce2980a5a 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -491,6 +491,181 @@ static unsigned int __cpuinit intel_size_cache(struct cpuinfo_x86 *c, unsigned i } #endif +#define TLB_INST_4K 0x01 +#define TLB_INST_4M 0x02 +#define TLB_INST_2M_4M 0x03 + +#define TLB_INST_ALL 0x05 +#define TLB_INST_1G 0x06 + +#define TLB_DATA_4K 0x11 +#define TLB_DATA_4M 0x12 +#define TLB_DATA_2M_4M 0x13 +#define TLB_DATA_4K_4M 0x14 + +#define TLB_DATA_1G 0x16 + +#define TLB_DATA0_4K 0x21 +#define TLB_DATA0_4M 0x22 +#define TLB_DATA0_2M_4M 0x23 + +#define STLB_4K 0x41 + +static const struct _tlb_table intel_tlb_table[] __cpuinitconst = { + { 0x01, TLB_INST_4K, 32, " TLB_INST 4 KByte pages, 4-way set associative" }, + { 0x02, TLB_INST_4M, 2, " TLB_INST 4 MByte pages, full associative" }, + { 0x03, TLB_DATA_4K, 64, " TLB_DATA 4 KByte pages, 4-way set associative" }, + { 0x04, TLB_DATA_4M, 8, " TLB_DATA 4 MByte pages, 4-way set associative" }, + { 0x05, TLB_DATA_4M, 32, " TLB_DATA 4 MByte pages, 4-way set associative" }, + { 0x0b, TLB_INST_4M, 4, " TLB_INST 4 MByte pages, 4-way set associative" }, + { 0x4f, TLB_INST_4K, 32, " TLB_INST 4 KByte pages */" }, + { 0x50, TLB_INST_ALL, 64, " TLB_INST 4 KByte and 2-MByte or 4-MByte pages" }, + { 0x51, TLB_INST_ALL, 128, " TLB_INST 4 KByte and 2-MByte or 4-MByte pages" }, + { 0x52, TLB_INST_ALL, 256, " TLB_INST 4 KByte and 2-MByte or 4-MByte pages" }, + { 0x55, TLB_INST_2M_4M, 7, " TLB_INST 2-MByte or 4-MByte pages, fully associative" }, + { 0x56, TLB_DATA0_4M, 16, " TLB_DATA0 4 MByte pages, 4-way set associative" }, + { 0x57, TLB_DATA0_4K, 16, " TLB_DATA0 4 KByte pages, 4-way associative" }, + { 0x59, TLB_DATA0_4K, 16, " TLB_DATA0 4 KByte pages, fully associative" }, + { 0x5a, TLB_DATA0_2M_4M, 32, " TLB_DATA0 2-MByte or 4 MByte pages, 4-way set associative" }, + { 0x5b, TLB_DATA_4K_4M, 64, " TLB_DATA 4 KByte and 4 MByte pages" }, + { 0x5c, TLB_DATA_4K_4M, 128, " TLB_DATA 4 KByte and 4 MByte pages" }, + { 0x5d, TLB_DATA_4K_4M, 256, " TLB_DATA 4 KByte and 4 MByte pages" }, + { 0xb0, TLB_INST_4K, 128, " TLB_INST 4 KByte pages, 4-way set associative" }, + { 0xb1, TLB_INST_2M_4M, 4, " TLB_INST 2M pages, 4-way, 8 entries or 4M pages, 4-way entries" }, + { 0xb2, TLB_INST_4K, 64, " TLB_INST 4KByte pages, 4-way set associative" }, + { 0xb3, TLB_DATA_4K, 128, " TLB_DATA 4 KByte pages, 4-way set associative" }, + { 0xb4, TLB_DATA_4K, 256, " TLB_DATA 4 KByte pages, 4-way associative" }, + { 0xba, TLB_DATA_4K, 64, " TLB_DATA 4 KByte pages, 4-way associative" }, + { 0xc0, TLB_DATA_4K_4M, 8, " TLB_DATA 4 KByte and 4 MByte pages, 4-way associative" }, + { 0xca, STLB_4K, 512, " STLB 4 KByte pages, 4-way associative" }, + { 0x00, 0, 0 } +}; + +static void __cpuinit intel_tlb_lookup(const unsigned char desc) +{ + unsigned char k; + if (desc == 0) + return; + + /* look up this descriptor in the table */ + for (k = 0; intel_tlb_table[k].descriptor != desc && \ + intel_tlb_table[k].descriptor != 0; k++) + ; + + if (intel_tlb_table[k].tlb_type == 0) + return; + + switch (intel_tlb_table[k].tlb_type) { + case STLB_4K: + if (tlb_lli_4k[ENTRIES] < intel_tlb_table[k].entries) + tlb_lli_4k[ENTRIES] = intel_tlb_table[k].entries; + if (tlb_lld_4k[ENTRIES] < intel_tlb_table[k].entries) + tlb_lld_4k[ENTRIES] = intel_tlb_table[k].entries; + break; + case TLB_INST_ALL: + if (tlb_lli_4k[ENTRIES] < intel_tlb_table[k].entries) + tlb_lli_4k[ENTRIES] = intel_tlb_table[k].entries; + if (tlb_lli_2m[ENTRIES] < intel_tlb_table[k].entries) + tlb_lli_2m[ENTRIES] = intel_tlb_table[k].entries; + if (tlb_lli_4m[ENTRIES] < intel_tlb_table[k].entries) + tlb_lli_4m[ENTRIES] = intel_tlb_table[k].entries; + break; + case TLB_INST_4K: + if (tlb_lli_4k[ENTRIES] < intel_tlb_table[k].entries) + tlb_lli_4k[ENTRIES] = intel_tlb_table[k].entries; + break; + case TLB_INST_4M: + if (tlb_lli_4m[ENTRIES] < intel_tlb_table[k].entries) + tlb_lli_4m[ENTRIES] = intel_tlb_table[k].entries; + break; + case TLB_INST_2M_4M: + if (tlb_lli_2m[ENTRIES] < intel_tlb_table[k].entries) + tlb_lli_2m[ENTRIES] = intel_tlb_table[k].entries; + if (tlb_lli_4m[ENTRIES] < intel_tlb_table[k].entries) + tlb_lli_4m[ENTRIES] = intel_tlb_table[k].entries; + break; + case TLB_DATA_4K: + case TLB_DATA0_4K: + if (tlb_lld_4k[ENTRIES] < intel_tlb_table[k].entries) + tlb_lld_4k[ENTRIES] = intel_tlb_table[k].entries; + break; + case TLB_DATA_4M: + case TLB_DATA0_4M: + if (tlb_lld_4m[ENTRIES] < intel_tlb_table[k].entries) + tlb_lld_4m[ENTRIES] = intel_tlb_table[k].entries; + break; + case TLB_DATA_2M_4M: + case TLB_DATA0_2M_4M: + if (tlb_lld_2m[ENTRIES] < intel_tlb_table[k].entries) + tlb_lld_2m[ENTRIES] = intel_tlb_table[k].entries; + if (tlb_lld_4m[ENTRIES] < intel_tlb_table[k].entries) + tlb_lld_4m[ENTRIES] = intel_tlb_table[k].entries; + break; + case TLB_DATA_4K_4M: + if (tlb_lld_4k[ENTRIES] < intel_tlb_table[k].entries) + tlb_lld_4k[ENTRIES] = intel_tlb_table[k].entries; + if (tlb_lld_4m[ENTRIES] < intel_tlb_table[k].entries) + tlb_lld_4m[ENTRIES] = intel_tlb_table[k].entries; + break; + } +} + +static void __cpuinit intel_tlb_flushall_shift_set(struct cpuinfo_x86 *c) +{ + if (!cpu_has_invlpg) { + tlb_flushall_shift = -1; + return; + } + switch ((c->x86 << 8) + c->x86_model) { + case 0x60f: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */ + case 0x616: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */ + case 0x617: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */ + case 0x61d: /* six-core 45 nm xeon "Dunnington" */ + tlb_flushall_shift = -1; + break; + case 0x61a: /* 45 nm nehalem, "Bloomfield" */ + case 0x61e: /* 45 nm nehalem, "Lynnfield" */ + case 0x625: /* 32 nm nehalem, "Clarkdale" */ + case 0x62c: /* 32 nm nehalem, "Gulftown" */ + case 0x62e: /* 45 nm nehalem-ex, "Beckton" */ + case 0x62f: /* 32 nm Xeon E7 */ + tlb_flushall_shift = 6; + break; + case 0x62a: /* SandyBridge */ + case 0x62d: /* SandyBridge, "Romely-EP" */ + tlb_flushall_shift = 5; + break; + case 0x63a: /* Ivybridge */ + tlb_flushall_shift = 1; + break; + default: + tlb_flushall_shift = 6; + } +} + +static void __cpuinit intel_detect_tlb(struct cpuinfo_x86 *c) +{ + int i, j, n; + unsigned int regs[4]; + unsigned char *desc = (unsigned char *)regs; + /* Number of times to iterate */ + n = cpuid_eax(2) & 0xFF; + + for (i = 0 ; i < n ; i++) { + cpuid(2, ®s[0], ®s[1], ®s[2], ®s[3]); + + /* If bit 31 is set, this is an unknown format */ + for (j = 0 ; j < 3 ; j++) + if (regs[j] & (1 << 31)) + regs[j] = 0; + + /* Byte 0 is level count, not a descriptor */ + for (j = 1 ; j < 16 ; j++) + intel_tlb_lookup(desc[j]); + } + intel_tlb_flushall_shift_set(c); +} + static const struct cpu_dev __cpuinitconst intel_cpu_dev = { .c_vendor = "Intel", .c_ident = { "GenuineIntel" }, @@ -546,6 +721,7 @@ static const struct cpu_dev __cpuinitconst intel_cpu_dev = { }, .c_size_cache = intel_size_cache, #endif + .c_detect_tlb = intel_detect_tlb, .c_early_init = early_init_intel, .c_init = init_intel, .c_x86_vendor = X86_VENDOR_INTEL, diff --git a/arch/x86/kernel/cpu/sched.c b/arch/x86/kernel/cpu/sched.c deleted file mode 100644 index a640ae5ad201..000000000000 --- a/arch/x86/kernel/cpu/sched.c +++ /dev/null @@ -1,55 +0,0 @@ -#include <linux/sched.h> -#include <linux/math64.h> -#include <linux/percpu.h> -#include <linux/irqflags.h> - -#include <asm/cpufeature.h> -#include <asm/processor.h> - -#ifdef CONFIG_SMP - -static DEFINE_PER_CPU(struct aperfmperf, old_perf_sched); - -static unsigned long scale_aperfmperf(void) -{ - struct aperfmperf val, *old = &__get_cpu_var(old_perf_sched); - unsigned long ratio, flags; - - local_irq_save(flags); - get_aperfmperf(&val); - local_irq_restore(flags); - - ratio = calc_aperfmperf_ratio(old, &val); - *old = val; - - return ratio; -} - -unsigned long arch_scale_freq_power(struct sched_domain *sd, int cpu) -{ - /* - * do aperf/mperf on the cpu level because it includes things - * like turbo mode, which are relevant to full cores. - */ - if (boot_cpu_has(X86_FEATURE_APERFMPERF)) - return scale_aperfmperf(); - - /* - * maybe have something cpufreq here - */ - - return default_scale_freq_power(sd, cpu); -} - -unsigned long arch_scale_smt_power(struct sched_domain *sd, int cpu) -{ - /* - * aperf/mperf already includes the smt gain - */ - if (boot_cpu_has(X86_FEATURE_APERFMPERF)) - return SCHED_LOAD_SCALE; - - return default_scale_smt_power(sd, cpu); -} - -#endif diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index 111f6bbd8b38..69babd8c834f 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -1048,24 +1048,6 @@ apicinterrupt LOCAL_TIMER_VECTOR \ apicinterrupt X86_PLATFORM_IPI_VECTOR \ x86_platform_ipi smp_x86_platform_ipi -#ifdef CONFIG_SMP - ALIGN - INTR_FRAME -.irp idx,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, \ - 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 -.if NUM_INVALIDATE_TLB_VECTORS > \idx -ENTRY(invalidate_interrupt\idx) - pushq_cfi $~(INVALIDATE_TLB_VECTOR_START+\idx) - jmp .Lcommon_invalidate_interrupt0 - CFI_ADJUST_CFA_OFFSET -8 -END(invalidate_interrupt\idx) -.endif -.endr - CFI_ENDPROC -apicinterrupt INVALIDATE_TLB_VECTOR_START, \ - invalidate_interrupt0, smp_invalidate_interrupt -#endif - apicinterrupt THRESHOLD_APIC_VECTOR \ threshold_interrupt smp_threshold_interrupt apicinterrupt THERMAL_APIC_VECTOR \ diff --git a/arch/x86/kernel/irqinit.c b/arch/x86/kernel/irqinit.c index 252981afd6c4..6e03b0d69138 100644 --- a/arch/x86/kernel/irqinit.c +++ b/arch/x86/kernel/irqinit.c @@ -171,79 +171,6 @@ static void __init smp_intr_init(void) */ alloc_intr_gate(RESCHEDULE_VECTOR, reschedule_interrupt); - /* IPIs for invalidation */ -#define ALLOC_INVTLB_VEC(NR) \ - alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+NR, \ - invalidate_interrupt##NR) - - switch (NUM_INVALIDATE_TLB_VECTORS) { - default: - ALLOC_INVTLB_VEC(31); - case 31: - ALLOC_INVTLB_VEC(30); - case 30: - ALLOC_INVTLB_VEC(29); - case 29: - ALLOC_INVTLB_VEC(28); - case 28: - ALLOC_INVTLB_VEC(27); - case 27: - ALLOC_INVTLB_VEC(26); - case 26: - ALLOC_INVTLB_VEC(25); - case 25: - ALLOC_INVTLB_VEC(24); - case 24: - ALLOC_INVTLB_VEC(23); - case 23: - ALLOC_INVTLB_VEC(22); - case 22: - ALLOC_INVTLB_VEC(21); - case 21: - ALLOC_INVTLB_VEC(20); - case 20: - ALLOC_INVTLB_VEC(19); - case 19: - ALLOC_INVTLB_VEC(18); - case 18: - ALLOC_INVTLB_VEC(17); - case 17: - ALLOC_INVTLB_VEC(16); - case 16: - ALLOC_INVTLB_VEC(15); - case 15: - ALLOC_INVTLB_VEC(14); - case 14: - ALLOC_INVTLB_VEC(13); - case 13: - ALLOC_INVTLB_VEC(12); - case 12: - ALLOC_INVTLB_VEC(11); - case 11: - ALLOC_INVTLB_VEC(10); - case 10: - ALLOC_INVTLB_VEC(9); - case 9: - ALLOC_INVTLB_VEC(8); - case 8: - ALLOC_INVTLB_VEC(7); - case 7: - ALLOC_INVTLB_VEC(6); - case 6: - ALLOC_INVTLB_VEC(5); - case 5: - ALLOC_INVTLB_VEC(4); - case 4: - ALLOC_INVTLB_VEC(3); - case 3: - ALLOC_INVTLB_VEC(2); - case 2: - ALLOC_INVTLB_VEC(1); - case 1: - ALLOC_INVTLB_VEC(0); - break; - } - /* IPI for generic function call */ alloc_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt); diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c index 5a98aa272184..5cdff0357746 100644 --- a/arch/x86/kernel/setup_percpu.c +++ b/arch/x86/kernel/setup_percpu.c @@ -21,7 +21,7 @@ #include <asm/cpu.h> #include <asm/stackprotector.h> -DEFINE_PER_CPU(int, cpu_number); +DEFINE_PER_CPU_READ_MOSTLY(int, cpu_number); EXPORT_PER_CPU_SYMBOL(cpu_number); #ifdef CONFIG_X86_64 diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index c1a310fb8309..7c5a8c314c02 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -106,17 +106,17 @@ int smp_num_siblings = 1; EXPORT_SYMBOL(smp_num_siblings); /* Last level cache ID of each logical CPU */ -DEFINE_PER_CPU(u16, cpu_llc_id) = BAD_APICID; +DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID; /* representing HT siblings of each logical CPU */ -DEFINE_PER_CPU(cpumask_var_t, cpu_sibling_map); +DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map); EXPORT_PER_CPU_SYMBOL(cpu_sibling_map); /* representing HT and core siblings of each logical CPU */ -DEFINE_PER_CPU(cpumask_var_t, cpu_core_map); +DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map); EXPORT_PER_CPU_SYMBOL(cpu_core_map); -DEFINE_PER_CPU(cpumask_var_t, cpu_llc_shared_map); +DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map); /* Per CPU bogomips and other parameters */ DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info); diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c index a718e0d23503..931930a96160 100644 --- a/arch/x86/mm/pageattr.c +++ b/arch/x86/mm/pageattr.c @@ -919,11 +919,13 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages, /* * On success we use clflush, when the CPU supports it to - * avoid the wbindv. If the CPU does not support it and in the - * error case we fall back to cpa_flush_all (which uses - * wbindv): + * avoid the wbindv. If the CPU does not support it, in the + * error case, and during early boot (for EFI) we fall back + * to cpa_flush_all (which uses wbinvd): */ - if (!ret && cpu_has_clflush) { + if (early_boot_irqs_disabled) + __cpa_flush_all((void *)(long)cache); + else if (!ret && cpu_has_clflush) { if (cpa.flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) { cpa_flush_array(addr, numpages, cache, cpa.flags, pages); diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c index 5e57e113b72c..613cd83e8c0c 100644 --- a/arch/x86/mm/tlb.c +++ b/arch/x86/mm/tlb.c @@ -12,6 +12,7 @@ #include <asm/cache.h> #include <asm/apic.h> #include <asm/uv/uv.h> +#include <linux/debugfs.h> DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate) = { &init_mm, 0, }; @@ -27,33 +28,14 @@ DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate) * * More scalable flush, from Andi Kleen * - * To avoid global state use 8 different call vectors. - * Each CPU uses a specific vector to trigger flushes on other - * CPUs. Depending on the received vector the target CPUs look into - * the right array slot for the flush data. - * - * With more than 8 CPUs they are hashed to the 8 available - * vectors. The limited global vector space forces us to this right now. - * In future when interrupts are split into per CPU domains this could be - * fixed, at the cost of triggering multiple IPIs in some cases. + * Implement flush IPI by CALL_FUNCTION_VECTOR, Alex Shi */ -union smp_flush_state { - struct { - struct mm_struct *flush_mm; - unsigned long flush_va; - raw_spinlock_t tlbstate_lock; - DECLARE_BITMAP(flush_cpumask, NR_CPUS); - }; - char pad[INTERNODE_CACHE_BYTES]; -} ____cacheline_internodealigned_in_smp; - -/* State is put into the per CPU data section, but padded - to a full cache line because other CPUs can access it and we don't - want false sharing in the per cpu data segment. */ -static union smp_flush_state flush_state[NUM_INVALIDATE_TLB_VECTORS]; - -static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset); +struct flush_tlb_info { + struct mm_struct *flush_mm; + unsigned long flush_start; + unsigned long flush_end; +}; /* * We cannot call mmdrop() because we are in interrupt context, @@ -72,28 +54,25 @@ void leave_mm(int cpu) EXPORT_SYMBOL_GPL(leave_mm); /* - * * The flush IPI assumes that a thread switch happens in this order: * [cpu0: the cpu that switches] * 1) switch_mm() either 1a) or 1b) * 1a) thread switch to a different mm - * 1a1) cpu_clear(cpu, old_mm->cpu_vm_mask); - * Stop ipi delivery for the old mm. This is not synchronized with - * the other cpus, but smp_invalidate_interrupt ignore flush ipis - * for the wrong mm, and in the worst case we perform a superfluous - * tlb flush. - * 1a2) set cpu mmu_state to TLBSTATE_OK - * Now the smp_invalidate_interrupt won't call leave_mm if cpu0 - * was in lazy tlb mode. - * 1a3) update cpu active_mm + * 1a1) set cpu_tlbstate to TLBSTATE_OK + * Now the tlb flush NMI handler flush_tlb_func won't call leave_mm + * if cpu0 was in lazy tlb mode. + * 1a2) update cpu active_mm * Now cpu0 accepts tlb flushes for the new mm. - * 1a4) cpu_set(cpu, new_mm->cpu_vm_mask); + * 1a3) cpu_set(cpu, new_mm->cpu_vm_mask); * Now the other cpus will send tlb flush ipis. * 1a4) change cr3. + * 1a5) cpu_clear(cpu, old_mm->cpu_vm_mask); + * Stop ipi delivery for the old mm. This is not synchronized with + * the other cpus, but flush_tlb_func ignore flush ipis for the wrong + * mm, and in the worst case we perform a superfluous tlb flush. * 1b) thread switch without mm change - * cpu active_mm is correct, cpu0 already handles - * flush ipis. - * 1b1) set cpu mmu_state to TLBSTATE_OK + * cpu active_mm is correct, cpu0 already handles flush ipis. + * 1b1) set cpu_tlbstate to TLBSTATE_OK * 1b2) test_and_set the cpu bit in cpu_vm_mask. * Atomically set the bit [other cpus will start sending flush ipis], * and test the bit. @@ -106,174 +85,62 @@ EXPORT_SYMBOL_GPL(leave_mm); * runs in kernel space, the cpu could load tlb entries for user space * pages. * - * The good news is that cpu mmu_state is local to each cpu, no + * The good news is that cpu_tlbstate is local to each cpu, no * write/read ordering problems. */ /* - * TLB flush IPI: - * + * TLB flush funcation: * 1) Flush the tlb entries if the cpu uses the mm that's being flushed. * 2) Leave the mm if we are in the lazy tlb mode. - * - * Interrupts are disabled. - */ - -/* - * FIXME: use of asmlinkage is not consistent. On x86_64 it's noop - * but still used for documentation purpose but the usage is slightly - * inconsistent. On x86_32, asmlinkage is regparm(0) but interrupt - * entry calls in with the first parameter in %eax. Maybe define - * intrlinkage? */ -#ifdef CONFIG_X86_64 -asmlinkage -#endif -void smp_invalidate_interrupt(struct pt_regs *regs) +static void flush_tlb_func(void *info) { - unsigned int cpu; - unsigned int sender; - union smp_flush_state *f; - - cpu = smp_processor_id(); - /* - * orig_rax contains the negated interrupt vector. - * Use that to determine where the sender put the data. - */ - sender = ~regs->orig_ax - INVALIDATE_TLB_VECTOR_START; - f = &flush_state[sender]; - - if (!cpumask_test_cpu(cpu, to_cpumask(f->flush_cpumask))) - goto out; - /* - * This was a BUG() but until someone can quote me the - * line from the intel manual that guarantees an IPI to - * multiple CPUs is retried _only_ on the erroring CPUs - * its staying as a return - * - * BUG(); - */ - - if (f->flush_mm == this_cpu_read(cpu_tlbstate.active_mm)) { - if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) { - if (f->flush_va == TLB_FLUSH_ALL) - local_flush_tlb(); - else - __flush_tlb_one(f->flush_va); - } else - leave_mm(cpu); - } -out: - ack_APIC_irq(); - smp_mb__before_clear_bit(); - cpumask_clear_cpu(cpu, to_cpumask(f->flush_cpumask)); - smp_mb__after_clear_bit(); - inc_irq_stat(irq_tlb_count); -} + struct flush_tlb_info *f = info; -static void flush_tlb_others_ipi(const struct cpumask *cpumask, - struct mm_struct *mm, unsigned long va) -{ - unsigned int sender; - union smp_flush_state *f; - - /* Caller has disabled preemption */ - sender = this_cpu_read(tlb_vector_offset); - f = &flush_state[sender]; - - if (nr_cpu_ids > NUM_INVALIDATE_TLB_VECTORS) - raw_spin_lock(&f->tlbstate_lock); - - f->flush_mm = mm; - f->flush_va = va; - if (cpumask_andnot(to_cpumask(f->flush_cpumask), cpumask, cpumask_of(smp_processor_id()))) { - /* - * We have to send the IPI only to - * CPUs affected. - */ - apic->send_IPI_mask(to_cpumask(f->flush_cpumask), - INVALIDATE_TLB_VECTOR_START + sender); - - while (!cpumask_empty(to_cpumask(f->flush_cpumask))) - cpu_relax(); - } + if (f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm)) + return; + + if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) { + if (f->flush_end == TLB_FLUSH_ALL || !cpu_has_invlpg) + local_flush_tlb(); + else if (!f->flush_end) + __flush_tlb_single(f->flush_start); + else { + unsigned long addr; + addr = f->flush_start; + while (addr < f->flush_end) { + __flush_tlb_single(addr); + addr += PAGE_SIZE; + } + } + } else + leave_mm(smp_processor_id()); - f->flush_mm = NULL; - f->flush_va = 0; - if (nr_cpu_ids > NUM_INVALIDATE_TLB_VECTORS) - raw_spin_unlock(&f->tlbstate_lock); } void native_flush_tlb_others(const struct cpumask *cpumask, - struct mm_struct *mm, unsigned long va) + struct mm_struct *mm, unsigned long start, + unsigned long end) { + struct flush_tlb_info info; + info.flush_mm = mm; + info.flush_start = start; + info.flush_end = end; + if (is_uv_system()) { unsigned int cpu; cpu = smp_processor_id(); - cpumask = uv_flush_tlb_others(cpumask, mm, va, cpu); + cpumask = uv_flush_tlb_others(cpumask, mm, start, end, cpu); if (cpumask) - flush_tlb_others_ipi(cpumask, mm, va); + smp_call_function_many(cpumask, flush_tlb_func, + &info, 1); return; } - flush_tlb_others_ipi(cpumask, mm, va); + smp_call_function_many(cpumask, flush_tlb_func, &info, 1); } -static void __cpuinit calculate_tlb_offset(void) -{ - int cpu, node, nr_node_vecs, idx = 0; - /* - * we are changing tlb_vector_offset for each CPU in runtime, but this - * will not cause inconsistency, as the write is atomic under X86. we - * might see more lock contentions in a short time, but after all CPU's - * tlb_vector_offset are changed, everything should go normal - * - * Note: if NUM_INVALIDATE_TLB_VECTORS % nr_online_nodes !=0, we might - * waste some vectors. - **/ - if (nr_online_nodes > NUM_INVALIDATE_TLB_VECTORS) - nr_node_vecs = 1; - else - nr_node_vecs = NUM_INVALIDATE_TLB_VECTORS/nr_online_nodes; - - for_each_online_node(node) { - int node_offset = (idx % NUM_INVALIDATE_TLB_VECTORS) * - nr_node_vecs; - int cpu_offset = 0; - for_each_cpu(cpu, cpumask_of_node(node)) { - per_cpu(tlb_vector_offset, cpu) = node_offset + - cpu_offset; - cpu_offset++; - cpu_offset = cpu_offset % nr_node_vecs; - } - idx++; - } -} - -static int __cpuinit tlb_cpuhp_notify(struct notifier_block *n, - unsigned long action, void *hcpu) -{ - switch (action & 0xf) { - case CPU_ONLINE: - case CPU_DEAD: - calculate_tlb_offset(); - } - return NOTIFY_OK; -} - -static int __cpuinit init_smp_flush(void) -{ - int i; - - for (i = 0; i < ARRAY_SIZE(flush_state); i++) - raw_spin_lock_init(&flush_state[i].tlbstate_lock); - - calculate_tlb_offset(); - hotcpu_notifier(tlb_cpuhp_notify, 0); - return 0; -} -core_initcall(init_smp_flush); - void flush_tlb_current_task(void) { struct mm_struct *mm = current->mm; @@ -282,27 +149,91 @@ void flush_tlb_current_task(void) local_flush_tlb(); if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids) - flush_tlb_others(mm_cpumask(mm), mm, TLB_FLUSH_ALL); + flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL); preempt_enable(); } -void flush_tlb_mm(struct mm_struct *mm) +/* + * It can find out the THP large page, or + * HUGETLB page in tlb_flush when THP disabled + */ +static inline unsigned long has_large_page(struct mm_struct *mm, + unsigned long start, unsigned long end) +{ + pgd_t *pgd; + pud_t *pud; + pmd_t *pmd; + unsigned long addr = ALIGN(start, HPAGE_SIZE); + for (; addr < end; addr += HPAGE_SIZE) { + pgd = pgd_offset(mm, addr); + if (likely(!pgd_none(*pgd))) { + pud = pud_offset(pgd, addr); + if (likely(!pud_none(*pud))) { + pmd = pmd_offset(pud, addr); + if (likely(!pmd_none(*pmd))) + if (pmd_large(*pmd)) + return addr; + } + } + } + return 0; +} + +void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, + unsigned long end, unsigned long vmflag) { + unsigned long addr; + unsigned act_entries, tlb_entries = 0; + preempt_disable(); + if (current->active_mm != mm) + goto flush_all; - if (current->active_mm == mm) { - if (current->mm) + if (!current->mm) { + leave_mm(smp_processor_id()); + goto flush_all; + } + + if (end == TLB_FLUSH_ALL || tlb_flushall_shift == -1 + || vmflag == VM_HUGETLB) { + local_flush_tlb(); + goto flush_all; + } + + /* In modern CPU, last level tlb used for both data/ins */ + if (vmflag & VM_EXEC) + tlb_entries = tlb_lli_4k[ENTRIES]; + else + tlb_entries = tlb_lld_4k[ENTRIES]; + /* Assume all of TLB entries was occupied by this task */ + act_entries = mm->total_vm > tlb_entries ? tlb_entries : mm->total_vm; + + /* tlb_flushall_shift is on balance point, details in commit log */ + if ((end - start) >> PAGE_SHIFT > act_entries >> tlb_flushall_shift) + local_flush_tlb(); + else { + if (has_large_page(mm, start, end)) { local_flush_tlb(); - else - leave_mm(smp_processor_id()); + goto flush_all; + } + /* flush range by one by one 'invlpg' */ + for (addr = start; addr < end; addr += PAGE_SIZE) + __flush_tlb_single(addr); + + if (cpumask_any_but(mm_cpumask(mm), + smp_processor_id()) < nr_cpu_ids) + flush_tlb_others(mm_cpumask(mm), mm, start, end); + preempt_enable(); + return; } - if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids) - flush_tlb_others(mm_cpumask(mm), mm, TLB_FLUSH_ALL); +flush_all: + if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids) + flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL); preempt_enable(); } -void flush_tlb_page(struct vm_area_struct *vma, unsigned long va) +void flush_tlb_page(struct vm_area_struct *vma, unsigned long start) { struct mm_struct *mm = vma->vm_mm; @@ -310,13 +241,13 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va) if (current->active_mm == mm) { if (current->mm) - __flush_tlb_one(va); + __flush_tlb_one(start); else leave_mm(smp_processor_id()); } if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids) - flush_tlb_others(mm_cpumask(mm), mm, va); + flush_tlb_others(mm_cpumask(mm), mm, start, 0UL); preempt_enable(); } @@ -332,3 +263,83 @@ void flush_tlb_all(void) { on_each_cpu(do_flush_tlb_all, NULL, 1); } + +static void do_kernel_range_flush(void *info) +{ + struct flush_tlb_info *f = info; + unsigned long addr; + + /* flush range by one by one 'invlpg' */ + for (addr = f->flush_start; addr < f->flush_end; addr += PAGE_SIZE) + __flush_tlb_single(addr); +} + +void flush_tlb_kernel_range(unsigned long start, unsigned long end) +{ + unsigned act_entries; + struct flush_tlb_info info; + + /* In modern CPU, last level tlb used for both data/ins */ + act_entries = tlb_lld_4k[ENTRIES]; + + /* Balance as user space task's flush, a bit conservative */ + if (end == TLB_FLUSH_ALL || tlb_flushall_shift == -1 || + (end - start) >> PAGE_SHIFT > act_entries >> tlb_flushall_shift) + + on_each_cpu(do_flush_tlb_all, NULL, 1); + else { + info.flush_start = start; + info.flush_end = end; + on_each_cpu(do_kernel_range_flush, &info, 1); + } +} + +#ifdef CONFIG_DEBUG_TLBFLUSH +static ssize_t tlbflush_read_file(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + char buf[32]; + unsigned int len; + + len = sprintf(buf, "%hd\n", tlb_flushall_shift); + return simple_read_from_buffer(user_buf, count, ppos, buf, len); +} + +static ssize_t tlbflush_write_file(struct file *file, + const char __user *user_buf, size_t count, loff_t *ppos) +{ + char buf[32]; + ssize_t len; + s8 shift; + + len = min(count, sizeof(buf) - 1); + if (copy_from_user(buf, user_buf, len)) + return -EFAULT; + + buf[len] = '\0'; + if (kstrtos8(buf, 0, &shift)) + return -EINVAL; + + if (shift > 64) + return -EINVAL; + + tlb_flushall_shift = shift; + return count; +} + +static const struct file_operations fops_tlbflush = { + .read = tlbflush_read_file, + .write = tlbflush_write_file, + .llseek = default_llseek, +}; + +static int __cpuinit create_tlb_flushall_shift(void) +{ + if (cpu_has_invlpg) { + debugfs_create_file("tlb_flushall_shift", S_IRUSR | S_IWUSR, + arch_debugfs_dir, NULL, &fops_tlbflush); + } + return 0; +} +late_initcall(create_tlb_flushall_shift); +#endif diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c index 92660edaa1e7..2dc29f51e75a 100644 --- a/arch/x86/platform/efi/efi.c +++ b/arch/x86/platform/efi/efi.c @@ -234,22 +234,7 @@ static efi_status_t __init phys_efi_set_virtual_address_map( return status; } -static efi_status_t __init phys_efi_get_time(efi_time_t *tm, - efi_time_cap_t *tc) -{ - unsigned long flags; - efi_status_t status; - - spin_lock_irqsave(&rtc_lock, flags); - efi_call_phys_prelog(); - status = efi_call_phys2(efi_phys.get_time, virt_to_phys(tm), - virt_to_phys(tc)); - efi_call_phys_epilog(); - spin_unlock_irqrestore(&rtc_lock, flags); - return status; -} - -int efi_set_rtc_mmss(unsigned long nowtime) +static int efi_set_rtc_mmss(unsigned long nowtime) { int real_seconds, real_minutes; efi_status_t status; @@ -278,7 +263,7 @@ int efi_set_rtc_mmss(unsigned long nowtime) return 0; } -unsigned long efi_get_time(void) +static unsigned long efi_get_time(void) { efi_status_t status; efi_time_t eft; @@ -621,18 +606,13 @@ static int __init efi_runtime_init(void) } /* * We will only need *early* access to the following - * two EFI runtime services before set_virtual_address_map + * EFI runtime service before set_virtual_address_map * is invoked. */ - efi_phys.get_time = (efi_get_time_t *)runtime->get_time; efi_phys.set_virtual_address_map = (efi_set_virtual_address_map_t *) runtime->set_virtual_address_map; - /* - * Make efi_get_time can be called before entering - * virtual mode. - */ - efi.get_time = phys_efi_get_time; + early_iounmap(runtime, sizeof(efi_runtime_services_t)); return 0; @@ -720,12 +700,10 @@ void __init efi_init(void) efi_enabled = 0; return; } -#ifdef CONFIG_X86_32 if (efi_native) { x86_platform.get_wallclock = efi_get_time; x86_platform.set_wallclock = efi_set_rtc_mmss; } -#endif #if EFI_DEBUG print_efi_memmap(); diff --git a/arch/x86/platform/uv/tlb_uv.c b/arch/x86/platform/uv/tlb_uv.c index 71b5d5a07d7b..b8b3a37c80cd 100644 --- a/arch/x86/platform/uv/tlb_uv.c +++ b/arch/x86/platform/uv/tlb_uv.c @@ -1055,8 +1055,8 @@ static int set_distrib_bits(struct cpumask *flush_mask, struct bau_control *bcp, * done. The returned pointer is valid till preemption is re-enabled. */ const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask, - struct mm_struct *mm, unsigned long va, - unsigned int cpu) + struct mm_struct *mm, unsigned long start, + unsigned end, unsigned int cpu) { int locals = 0; int remotes = 0; @@ -1113,7 +1113,7 @@ const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask, record_send_statistics(stat, locals, hubs, remotes, bau_desc); - bau_desc->payload.address = va; + bau_desc->payload.address = start; bau_desc->payload.sending_cpu = cpu; /* * uv_flush_send_and_wait returns 0 if all cpu's were messaged, diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c index 27336dfcda8e..b65a76133f4f 100644 --- a/arch/x86/xen/mmu.c +++ b/arch/x86/xen/mmu.c @@ -1256,7 +1256,8 @@ static void xen_flush_tlb_single(unsigned long addr) } static void xen_flush_tlb_others(const struct cpumask *cpus, - struct mm_struct *mm, unsigned long va) + struct mm_struct *mm, unsigned long start, + unsigned long end) { struct { struct mmuext_op op; @@ -1268,7 +1269,7 @@ static void xen_flush_tlb_others(const struct cpumask *cpus, } *args; struct multicall_space mcs; - trace_xen_mmu_flush_tlb_others(cpus, mm, va); + trace_xen_mmu_flush_tlb_others(cpus, mm, start, end); if (cpumask_empty(cpus)) return; /* nothing to do */ @@ -1281,11 +1282,10 @@ static void xen_flush_tlb_others(const struct cpumask *cpus, cpumask_and(to_cpumask(args->mask), cpus, cpu_online_mask); cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask)); - if (va == TLB_FLUSH_ALL) { - args->op.cmd = MMUEXT_TLB_FLUSH_MULTI; - } else { + args->op.cmd = MMUEXT_TLB_FLUSH_MULTI; + if (start != TLB_FLUSH_ALL && (end - start) <= PAGE_SIZE) { args->op.cmd = MMUEXT_INVLPG_MULTI; - args->op.arg1.linear_addr = va; + args->op.arg1.linear_addr = start; } MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF); |