summaryrefslogtreecommitdiff
path: root/arch
diff options
context:
space:
mode:
Diffstat (limited to 'arch')
-rw-r--r--arch/arm/boot/dts/imx51.dtsi8
-rw-r--r--arch/arm/boot/dts/imx53.dtsi14
-rw-r--r--arch/arm/boot/dts/imx6q.dtsi14
-rw-r--r--arch/arm/mach-at91/at91sam9g45.c13
-rw-r--r--arch/arm/mach-at91/at91sam9g45_devices.c128
-rw-r--r--arch/arm/mach-at91/include/mach/at91sam9g45.h2
-rw-r--r--arch/arm/mach-imx/mm-imx25.c10
-rw-r--r--arch/arm/mach-imx/mm-imx3.c7
-rw-r--r--arch/arm/mach-imx/mm-imx5.c40
-rw-r--r--arch/arm/mach-omap2/clock3xxx_data.c14
-rw-r--r--arch/arm/plat-spear/include/plat/keyboard.h2
-rw-r--r--arch/m68k/Kconfig.bus7
-rw-r--r--arch/m68k/Kconfig.cpu18
-rw-r--r--arch/m68k/Makefile2
-rw-r--r--arch/m68k/include/asm/cacheflush_mm.h41
-rw-r--r--arch/m68k/include/asm/dma.h8
-rw-r--r--arch/m68k/include/asm/gpio.h179
-rw-r--r--arch/m68k/include/asm/io_mm.h50
-rw-r--r--arch/m68k/include/asm/m520xsim.h14
-rw-r--r--arch/m68k/include/asm/m523xsim.h1
-rw-r--r--arch/m68k/include/asm/m525xsim.h194
-rw-r--r--arch/m68k/include/asm/m527xsim.h1
-rw-r--r--arch/m68k/include/asm/m528xsim.h2
-rw-r--r--arch/m68k/include/asm/m532xsim.h17
-rw-r--r--arch/m68k/include/asm/m5441xsim.h276
-rw-r--r--arch/m68k/include/asm/m54xxacr.h4
-rw-r--r--arch/m68k/include/asm/m54xxpci.h138
-rw-r--r--arch/m68k/include/asm/m54xxsim.h3
-rw-r--r--arch/m68k/include/asm/mcfclk.h43
-rw-r--r--arch/m68k/include/asm/mcfgpio.h343
-rw-r--r--arch/m68k/include/asm/mcfsim.h5
-rw-r--r--arch/m68k/include/asm/mcftimer.h2
-rw-r--r--arch/m68k/include/asm/mcfuart.h4
-rw-r--r--arch/m68k/include/asm/pci.h6
-rw-r--r--arch/m68k/include/asm/pinmux.h30
-rw-r--r--arch/m68k/kernel/Makefile1
-rw-r--r--arch/m68k/kernel/dma.c5
-rw-r--r--arch/m68k/kernel/entry.S452
-rw-r--r--arch/m68k/kernel/entry_mm.S419
-rw-r--r--arch/m68k/kernel/entry_no.S130
-rw-r--r--arch/m68k/kernel/pcibios.c109
-rw-r--r--arch/m68k/mm/memory.c2
-rw-r--r--arch/m68k/platform/coldfire/Makefile7
-rw-r--r--arch/m68k/platform/coldfire/clk.c108
-rw-r--r--arch/m68k/platform/coldfire/device.c57
-rw-r--r--arch/m68k/platform/coldfire/gpio.c172
-rw-r--r--arch/m68k/platform/coldfire/head.S6
-rw-r--r--arch/m68k/platform/coldfire/intc-525x.c91
-rw-r--r--arch/m68k/platform/coldfire/intc-simr.c26
-rw-r--r--arch/m68k/platform/coldfire/m5206.c9
-rw-r--r--arch/m68k/platform/coldfire/m520x.c103
-rw-r--r--arch/m68k/platform/coldfire/m523x.c22
-rw-r--r--arch/m68k/platform/coldfire/m5249.c10
-rw-r--r--arch/m68k/platform/coldfire/m525x.c66
-rw-r--r--arch/m68k/platform/coldfire/m5272.c11
-rw-r--r--arch/m68k/platform/coldfire/m527x.c43
-rw-r--r--arch/m68k/platform/coldfire/m528x.c33
-rw-r--r--arch/m68k/platform/coldfire/m5307.c9
-rw-r--r--arch/m68k/platform/coldfire/m532x.c154
-rw-r--r--arch/m68k/platform/coldfire/m5407.c9
-rw-r--r--arch/m68k/platform/coldfire/m5441x.c261
-rw-r--r--arch/m68k/platform/coldfire/m54xx.c7
-rw-r--r--arch/m68k/platform/coldfire/mcf8390.c38
-rw-r--r--arch/m68k/platform/coldfire/pci.c327
-rw-r--r--arch/m68k/platform/coldfire/pinmux.c28
-rw-r--r--arch/m68k/platform/coldfire/pit.c4
-rw-r--r--arch/m68k/platform/coldfire/timers.c2
-rw-r--r--arch/mips/kernel/kspd.c2
-rw-r--r--arch/powerpc/Makefile1
-rw-r--r--arch/s390/crypto/crypto_des.h18
-rw-r--r--arch/sh/include/asm/siu.h1
-rw-r--r--arch/sh/kernel/cpu/sh4a/setup-sh7722.c1
-rw-r--r--arch/sparc/include/asm/fixmap.h110
-rw-r--r--arch/sparc/include/asm/highmem.h3
-rw-r--r--arch/sparc/include/asm/leon.h1
-rw-r--r--arch/sparc/include/asm/mmu_context_32.h8
-rw-r--r--arch/sparc/include/asm/page_32.h3
-rw-r--r--arch/sparc/include/asm/pgalloc_32.h29
-rw-r--r--arch/sparc/include/asm/pgtable_32.h44
-rw-r--r--arch/sparc/include/asm/vaddrs.h22
-rw-r--r--arch/sparc/kernel/head_32.S2
-rw-r--r--arch/sparc/kernel/leon_kernel.c16
-rw-r--r--arch/sparc/kernel/process_32.c4
-rw-r--r--arch/sparc/kernel/setup_32.c1
-rw-r--r--arch/sparc/kernel/sys_sparc_64.c17
-rw-r--r--arch/sparc/lib/NG2memcpy.S72
-rw-r--r--arch/sparc/lib/U1memcpy.S4
-rw-r--r--arch/sparc/lib/copy_page.S56
-rw-r--r--arch/sparc/mm/fault_32.c18
-rw-r--r--arch/sparc/mm/highmem.c42
-rw-r--r--arch/sparc/mm/init_32.c58
-rw-r--r--arch/sparc/mm/srmmu.c332
-rw-r--r--arch/sparc/prom/init_32.c7
-rw-r--r--arch/sparc/prom/init_64.c4
-rw-r--r--arch/tile/kernel/pci.c2
-rw-r--r--arch/tile/kernel/pci_gx.c2
-rw-r--r--arch/x86/Kconfig.debug19
-rw-r--r--arch/x86/boot/compressed/cmdline.c4
-rw-r--r--arch/x86/boot/compressed/early_serial_console.c4
-rw-r--r--arch/x86/boot/compressed/eboot.c198
-rw-r--r--arch/x86/boot/compressed/head_32.S10
-rw-r--r--arch/x86/boot/compressed/head_64.S10
-rw-r--r--arch/x86/boot/compressed/misc.c31
-rw-r--r--arch/x86/boot/compressed/misc.h27
-rw-r--r--arch/x86/boot/header.S11
-rw-r--r--arch/x86/crypto/Makefile14
-rw-r--r--arch/x86/crypto/ablk_helper.c149
-rw-r--r--arch/x86/crypto/aes_glue.c2
-rw-r--r--arch/x86/crypto/aesni-intel_glue.c110
-rw-r--r--arch/x86/crypto/camellia_glue.c355
-rw-r--r--arch/x86/crypto/glue_helper.c307
-rw-r--r--arch/x86/crypto/serpent-avx-x86_64-asm_64.S704
-rw-r--r--arch/x86/crypto/serpent_avx_glue.c636
-rw-r--r--arch/x86/crypto/serpent_sse2_glue.c513
-rw-r--r--arch/x86/crypto/sha1_ssse3_asm.S2
-rw-r--r--arch/x86/crypto/sha1_ssse3_glue.c6
-rw-r--r--arch/x86/crypto/twofish-avx-x86_64-asm_64.S300
-rw-r--r--arch/x86/crypto/twofish_avx_glue.c624
-rw-r--r--arch/x86/crypto/twofish_glue_3way.c409
-rw-r--r--arch/x86/include/asm/apic.h2
-rw-r--r--arch/x86/include/asm/bootparam.h1
-rw-r--r--arch/x86/include/asm/cpufeature.h2
-rw-r--r--arch/x86/include/asm/crypto/ablk_helper.h31
-rw-r--r--arch/x86/include/asm/crypto/aes.h (renamed from arch/x86/include/asm/aes.h)0
-rw-r--r--arch/x86/include/asm/crypto/glue_helper.h115
-rw-r--r--arch/x86/include/asm/crypto/serpent-avx.h32
-rw-r--r--arch/x86/include/asm/crypto/serpent-sse2.h (renamed from arch/x86/include/asm/serpent.h)4
-rw-r--r--arch/x86/include/asm/crypto/twofish.h46
-rw-r--r--arch/x86/include/asm/entry_arch.h9
-rw-r--r--arch/x86/include/asm/irq_vectors.h11
-rw-r--r--arch/x86/include/asm/paravirt.h5
-rw-r--r--arch/x86/include/asm/paravirt_types.h3
-rw-r--r--arch/x86/include/asm/percpu.h17
-rw-r--r--arch/x86/include/asm/processor.h13
-rw-r--r--arch/x86/include/asm/smp.h16
-rw-r--r--arch/x86/include/asm/tlb.h9
-rw-r--r--arch/x86/include/asm/tlbflush.h49
-rw-r--r--arch/x86/include/asm/uv/uv.h5
-rw-r--r--arch/x86/kernel/apic/apic.c6
-rw-r--r--arch/x86/kernel/cpu/Makefile2
-rw-r--r--arch/x86/kernel/cpu/common.c31
-rw-r--r--arch/x86/kernel/cpu/cpu.h9
-rw-r--r--arch/x86/kernel/cpu/intel.c176
-rw-r--r--arch/x86/kernel/cpu/sched.c55
-rw-r--r--arch/x86/kernel/entry_64.S18
-rw-r--r--arch/x86/kernel/irqinit.c73
-rw-r--r--arch/x86/kernel/setup_percpu.c2
-rw-r--r--arch/x86/kernel/smpboot.c8
-rw-r--r--arch/x86/mm/pageattr.c10
-rw-r--r--arch/x86/mm/tlb.c401
-rw-r--r--arch/x86/platform/efi/efi.c30
-rw-r--r--arch/x86/platform/uv/tlb_uv.c6
-rw-r--r--arch/x86/xen/mmu.c12
153 files changed, 7519 insertions, 3376 deletions
diff --git a/arch/arm/boot/dts/imx51.dtsi b/arch/arm/boot/dts/imx51.dtsi
index 922adefdd291..53cbaa3d4f90 100644
--- a/arch/arm/boot/dts/imx51.dtsi
+++ b/arch/arm/boot/dts/imx51.dtsi
@@ -127,7 +127,7 @@
};
gpio1: gpio@73f84000 {
- compatible = "fsl,imx51-gpio", "fsl,imx31-gpio";
+ compatible = "fsl,imx51-gpio", "fsl,imx35-gpio";
reg = <0x73f84000 0x4000>;
interrupts = <50 51>;
gpio-controller;
@@ -137,7 +137,7 @@
};
gpio2: gpio@73f88000 {
- compatible = "fsl,imx51-gpio", "fsl,imx31-gpio";
+ compatible = "fsl,imx51-gpio", "fsl,imx35-gpio";
reg = <0x73f88000 0x4000>;
interrupts = <52 53>;
gpio-controller;
@@ -147,7 +147,7 @@
};
gpio3: gpio@73f8c000 {
- compatible = "fsl,imx51-gpio", "fsl,imx31-gpio";
+ compatible = "fsl,imx51-gpio", "fsl,imx35-gpio";
reg = <0x73f8c000 0x4000>;
interrupts = <54 55>;
gpio-controller;
@@ -157,7 +157,7 @@
};
gpio4: gpio@73f90000 {
- compatible = "fsl,imx51-gpio", "fsl,imx31-gpio";
+ compatible = "fsl,imx51-gpio", "fsl,imx35-gpio";
reg = <0x73f90000 0x4000>;
interrupts = <56 57>;
gpio-controller;
diff --git a/arch/arm/boot/dts/imx53.dtsi b/arch/arm/boot/dts/imx53.dtsi
index 4e735edc78ed..fc79cdc4b4e6 100644
--- a/arch/arm/boot/dts/imx53.dtsi
+++ b/arch/arm/boot/dts/imx53.dtsi
@@ -129,7 +129,7 @@
};
gpio1: gpio@53f84000 {
- compatible = "fsl,imx53-gpio", "fsl,imx31-gpio";
+ compatible = "fsl,imx53-gpio", "fsl,imx35-gpio";
reg = <0x53f84000 0x4000>;
interrupts = <50 51>;
gpio-controller;
@@ -139,7 +139,7 @@
};
gpio2: gpio@53f88000 {
- compatible = "fsl,imx53-gpio", "fsl,imx31-gpio";
+ compatible = "fsl,imx53-gpio", "fsl,imx35-gpio";
reg = <0x53f88000 0x4000>;
interrupts = <52 53>;
gpio-controller;
@@ -149,7 +149,7 @@
};
gpio3: gpio@53f8c000 {
- compatible = "fsl,imx53-gpio", "fsl,imx31-gpio";
+ compatible = "fsl,imx53-gpio", "fsl,imx35-gpio";
reg = <0x53f8c000 0x4000>;
interrupts = <54 55>;
gpio-controller;
@@ -159,7 +159,7 @@
};
gpio4: gpio@53f90000 {
- compatible = "fsl,imx53-gpio", "fsl,imx31-gpio";
+ compatible = "fsl,imx53-gpio", "fsl,imx35-gpio";
reg = <0x53f90000 0x4000>;
interrupts = <56 57>;
gpio-controller;
@@ -197,7 +197,7 @@
};
gpio5: gpio@53fdc000 {
- compatible = "fsl,imx53-gpio", "fsl,imx31-gpio";
+ compatible = "fsl,imx53-gpio", "fsl,imx35-gpio";
reg = <0x53fdc000 0x4000>;
interrupts = <103 104>;
gpio-controller;
@@ -207,7 +207,7 @@
};
gpio6: gpio@53fe0000 {
- compatible = "fsl,imx53-gpio", "fsl,imx31-gpio";
+ compatible = "fsl,imx53-gpio", "fsl,imx35-gpio";
reg = <0x53fe0000 0x4000>;
interrupts = <105 106>;
gpio-controller;
@@ -217,7 +217,7 @@
};
gpio7: gpio@53fe4000 {
- compatible = "fsl,imx53-gpio", "fsl,imx31-gpio";
+ compatible = "fsl,imx53-gpio", "fsl,imx35-gpio";
reg = <0x53fe4000 0x4000>;
interrupts = <107 108>;
gpio-controller;
diff --git a/arch/arm/boot/dts/imx6q.dtsi b/arch/arm/boot/dts/imx6q.dtsi
index c25d49584814..3d3c64b014e6 100644
--- a/arch/arm/boot/dts/imx6q.dtsi
+++ b/arch/arm/boot/dts/imx6q.dtsi
@@ -277,7 +277,7 @@
};
gpio1: gpio@0209c000 {
- compatible = "fsl,imx6q-gpio", "fsl,imx31-gpio";
+ compatible = "fsl,imx6q-gpio", "fsl,imx35-gpio";
reg = <0x0209c000 0x4000>;
interrupts = <0 66 0x04 0 67 0x04>;
gpio-controller;
@@ -287,7 +287,7 @@
};
gpio2: gpio@020a0000 {
- compatible = "fsl,imx6q-gpio", "fsl,imx31-gpio";
+ compatible = "fsl,imx6q-gpio", "fsl,imx35-gpio";
reg = <0x020a0000 0x4000>;
interrupts = <0 68 0x04 0 69 0x04>;
gpio-controller;
@@ -297,7 +297,7 @@
};
gpio3: gpio@020a4000 {
- compatible = "fsl,imx6q-gpio", "fsl,imx31-gpio";
+ compatible = "fsl,imx6q-gpio", "fsl,imx35-gpio";
reg = <0x020a4000 0x4000>;
interrupts = <0 70 0x04 0 71 0x04>;
gpio-controller;
@@ -307,7 +307,7 @@
};
gpio4: gpio@020a8000 {
- compatible = "fsl,imx6q-gpio", "fsl,imx31-gpio";
+ compatible = "fsl,imx6q-gpio", "fsl,imx35-gpio";
reg = <0x020a8000 0x4000>;
interrupts = <0 72 0x04 0 73 0x04>;
gpio-controller;
@@ -317,7 +317,7 @@
};
gpio5: gpio@020ac000 {
- compatible = "fsl,imx6q-gpio", "fsl,imx31-gpio";
+ compatible = "fsl,imx6q-gpio", "fsl,imx35-gpio";
reg = <0x020ac000 0x4000>;
interrupts = <0 74 0x04 0 75 0x04>;
gpio-controller;
@@ -327,7 +327,7 @@
};
gpio6: gpio@020b0000 {
- compatible = "fsl,imx6q-gpio", "fsl,imx31-gpio";
+ compatible = "fsl,imx6q-gpio", "fsl,imx35-gpio";
reg = <0x020b0000 0x4000>;
interrupts = <0 76 0x04 0 77 0x04>;
gpio-controller;
@@ -337,7 +337,7 @@
};
gpio7: gpio@020b4000 {
- compatible = "fsl,imx6q-gpio", "fsl,imx31-gpio";
+ compatible = "fsl,imx6q-gpio", "fsl,imx35-gpio";
reg = <0x020b4000 0x4000>;
interrupts = <0 78 0x04 0 79 0x04>;
gpio-controller;
diff --git a/arch/arm/mach-at91/at91sam9g45.c b/arch/arm/mach-at91/at91sam9g45.c
index 977127368a7d..ef6cedd52e3c 100644
--- a/arch/arm/mach-at91/at91sam9g45.c
+++ b/arch/arm/mach-at91/at91sam9g45.c
@@ -183,6 +183,13 @@ static struct clk adc_op_clk = {
.rate_hz = 13200000,
};
+/* AES/TDES/SHA clock - Only for sam9m11/sam9g56 */
+static struct clk aestdessha_clk = {
+ .name = "aestdessha_clk",
+ .pmc_mask = 1 << AT91SAM9G45_ID_AESTDESSHA,
+ .type = CLK_TYPE_PERIPHERAL,
+};
+
static struct clk *periph_clocks[] __initdata = {
&pioA_clk,
&pioB_clk,
@@ -212,6 +219,7 @@ static struct clk *periph_clocks[] __initdata = {
&udphs_clk,
&mmc1_clk,
&adc_op_clk,
+ &aestdessha_clk,
// irq0
};
@@ -232,6 +240,9 @@ static struct clk_lookup periph_clocks_lookups[] = {
CLKDEV_CON_DEV_ID("pclk", "ssc.0", &ssc0_clk),
CLKDEV_CON_DEV_ID("pclk", "ssc.1", &ssc1_clk),
CLKDEV_CON_DEV_ID(NULL, "atmel-trng", &trng_clk),
+ CLKDEV_CON_DEV_ID(NULL, "atmel_sha", &aestdessha_clk),
+ CLKDEV_CON_DEV_ID(NULL, "atmel_tdes", &aestdessha_clk),
+ CLKDEV_CON_DEV_ID(NULL, "atmel_aes", &aestdessha_clk),
/* more usart lookup table for DT entries */
CLKDEV_CON_DEV_ID("usart", "ffffee00.serial", &mck),
CLKDEV_CON_DEV_ID("usart", "fff8c000.serial", &usart0_clk),
@@ -388,7 +399,7 @@ static unsigned int at91sam9g45_default_irq_priority[NR_AIC_IRQS] __initdata = {
3, /* Ethernet */
0, /* Image Sensor Interface */
2, /* USB Device High speed port */
- 0,
+ 0, /* AESTDESSHA Crypto HW Accelerators */
0, /* Multimedia Card Interface 1 */
0,
0, /* Advanced Interrupt Controller (IRQ0) */
diff --git a/arch/arm/mach-at91/at91sam9g45_devices.c b/arch/arm/mach-at91/at91sam9g45_devices.c
index 40fb79df2de0..06073996a382 100644
--- a/arch/arm/mach-at91/at91sam9g45_devices.c
+++ b/arch/arm/mach-at91/at91sam9g45_devices.c
@@ -18,6 +18,7 @@
#include <linux/platform_device.h>
#include <linux/i2c-gpio.h>
#include <linux/atmel-mci.h>
+#include <linux/platform_data/atmel-aes.h>
#include <linux/platform_data/at91_adc.h>
@@ -1830,6 +1831,130 @@ void __init at91_register_uart(unsigned id, unsigned portnr, unsigned pins) {}
void __init at91_add_device_serial(void) {}
#endif
+/* --------------------------------------------------------------------
+ * SHA1/SHA256
+ * -------------------------------------------------------------------- */
+
+#if defined(CONFIG_CRYPTO_DEV_ATMEL_SHA) || defined(CONFIG_CRYPTO_DEV_ATMEL_SHA_MODULE)
+static struct resource sha_resources[] = {
+ {
+ .start = AT91SAM9G45_BASE_SHA,
+ .end = AT91SAM9G45_BASE_SHA + SZ_16K - 1,
+ .flags = IORESOURCE_MEM,
+ },
+ [1] = {
+ .start = AT91SAM9G45_ID_AESTDESSHA,
+ .end = AT91SAM9G45_ID_AESTDESSHA,
+ .flags = IORESOURCE_IRQ,
+ },
+};
+
+static struct platform_device at91sam9g45_sha_device = {
+ .name = "atmel_sha",
+ .id = -1,
+ .resource = sha_resources,
+ .num_resources = ARRAY_SIZE(sha_resources),
+};
+
+static void __init at91_add_device_sha(void)
+{
+ platform_device_register(&at91sam9g45_sha_device);
+}
+#else
+static void __init at91_add_device_sha(void) {}
+#endif
+
+/* --------------------------------------------------------------------
+ * DES/TDES
+ * -------------------------------------------------------------------- */
+
+#if defined(CONFIG_CRYPTO_DEV_ATMEL_TDES) || defined(CONFIG_CRYPTO_DEV_ATMEL_TDES_MODULE)
+static struct resource tdes_resources[] = {
+ [0] = {
+ .start = AT91SAM9G45_BASE_TDES,
+ .end = AT91SAM9G45_BASE_TDES + SZ_16K - 1,
+ .flags = IORESOURCE_MEM,
+ },
+ [1] = {
+ .start = AT91SAM9G45_ID_AESTDESSHA,
+ .end = AT91SAM9G45_ID_AESTDESSHA,
+ .flags = IORESOURCE_IRQ,
+ },
+};
+
+static struct platform_device at91sam9g45_tdes_device = {
+ .name = "atmel_tdes",
+ .id = -1,
+ .resource = tdes_resources,
+ .num_resources = ARRAY_SIZE(tdes_resources),
+};
+
+static void __init at91_add_device_tdes(void)
+{
+ platform_device_register(&at91sam9g45_tdes_device);
+}
+#else
+static void __init at91_add_device_tdes(void) {}
+#endif
+
+/* --------------------------------------------------------------------
+ * AES
+ * -------------------------------------------------------------------- */
+
+#if defined(CONFIG_CRYPTO_DEV_ATMEL_AES) || defined(CONFIG_CRYPTO_DEV_ATMEL_AES_MODULE)
+static struct aes_platform_data aes_data;
+static u64 aes_dmamask = DMA_BIT_MASK(32);
+
+static struct resource aes_resources[] = {
+ [0] = {
+ .start = AT91SAM9G45_BASE_AES,
+ .end = AT91SAM9G45_BASE_AES + SZ_16K - 1,
+ .flags = IORESOURCE_MEM,
+ },
+ [1] = {
+ .start = AT91SAM9G45_ID_AESTDESSHA,
+ .end = AT91SAM9G45_ID_AESTDESSHA,
+ .flags = IORESOURCE_IRQ,
+ },
+};
+
+static struct platform_device at91sam9g45_aes_device = {
+ .name = "atmel_aes",
+ .id = -1,
+ .dev = {
+ .dma_mask = &aes_dmamask,
+ .coherent_dma_mask = DMA_BIT_MASK(32),
+ .platform_data = &aes_data,
+ },
+ .resource = aes_resources,
+ .num_resources = ARRAY_SIZE(aes_resources),
+};
+
+static void __init at91_add_device_aes(void)
+{
+ struct at_dma_slave *atslave;
+ struct aes_dma_data *alt_atslave;
+
+ alt_atslave = kzalloc(sizeof(struct aes_dma_data), GFP_KERNEL);
+
+ /* DMA TX slave channel configuration */
+ atslave = &alt_atslave->txdata;
+ atslave->dma_dev = &at_hdmac_device.dev;
+ atslave->cfg = ATC_FIFOCFG_ENOUGHSPACE | ATC_SRC_H2SEL_HW |
+ ATC_SRC_PER(AT_DMA_ID_AES_RX);
+
+ /* DMA RX slave channel configuration */
+ atslave = &alt_atslave->rxdata;
+ atslave->dma_dev = &at_hdmac_device.dev;
+ atslave->cfg = ATC_FIFOCFG_ENOUGHSPACE | ATC_DST_H2SEL_HW |
+ ATC_DST_PER(AT_DMA_ID_AES_TX);
+
+ aes_data.dma_slave = alt_atslave;
+ platform_device_register(&at91sam9g45_aes_device);
+}
+#else
+static void __init at91_add_device_aes(void) {}
+#endif
/* -------------------------------------------------------------------- */
/*
@@ -1847,6 +1972,9 @@ static int __init at91_add_standard_devices(void)
at91_add_device_trng();
at91_add_device_watchdog();
at91_add_device_tc();
+ at91_add_device_sha();
+ at91_add_device_tdes();
+ at91_add_device_aes();
return 0;
}
diff --git a/arch/arm/mach-at91/include/mach/at91sam9g45.h b/arch/arm/mach-at91/include/mach/at91sam9g45.h
index 3a4da24d5911..8eba1021f533 100644
--- a/arch/arm/mach-at91/include/mach/at91sam9g45.h
+++ b/arch/arm/mach-at91/include/mach/at91sam9g45.h
@@ -136,6 +136,8 @@
#define AT_DMA_ID_SSC1_RX 8
#define AT_DMA_ID_AC97_TX 9
#define AT_DMA_ID_AC97_RX 10
+#define AT_DMA_ID_AES_TX 11
+#define AT_DMA_ID_AES_RX 12
#define AT_DMA_ID_MCI1 13
#endif
diff --git a/arch/arm/mach-imx/mm-imx25.c b/arch/arm/mach-imx/mm-imx25.c
index 388928fdb11a..f3f5c6542ab4 100644
--- a/arch/arm/mach-imx/mm-imx25.c
+++ b/arch/arm/mach-imx/mm-imx25.c
@@ -89,11 +89,11 @@ static const struct resource imx25_audmux_res[] __initconst = {
void __init imx25_soc_init(void)
{
- /* i.mx25 has the i.mx31 type gpio */
- mxc_register_gpio("imx31-gpio", 0, MX25_GPIO1_BASE_ADDR, SZ_16K, MX25_INT_GPIO1, 0);
- mxc_register_gpio("imx31-gpio", 1, MX25_GPIO2_BASE_ADDR, SZ_16K, MX25_INT_GPIO2, 0);
- mxc_register_gpio("imx31-gpio", 2, MX25_GPIO3_BASE_ADDR, SZ_16K, MX25_INT_GPIO3, 0);
- mxc_register_gpio("imx31-gpio", 3, MX25_GPIO4_BASE_ADDR, SZ_16K, MX25_INT_GPIO4, 0);
+ /* i.mx25 has the i.mx35 type gpio */
+ mxc_register_gpio("imx35-gpio", 0, MX25_GPIO1_BASE_ADDR, SZ_16K, MX25_INT_GPIO1, 0);
+ mxc_register_gpio("imx35-gpio", 1, MX25_GPIO2_BASE_ADDR, SZ_16K, MX25_INT_GPIO2, 0);
+ mxc_register_gpio("imx35-gpio", 2, MX25_GPIO3_BASE_ADDR, SZ_16K, MX25_INT_GPIO3, 0);
+ mxc_register_gpio("imx35-gpio", 3, MX25_GPIO4_BASE_ADDR, SZ_16K, MX25_INT_GPIO4, 0);
pinctrl_provide_dummies();
/* i.mx25 has the i.mx35 type sdma */
diff --git a/arch/arm/mach-imx/mm-imx3.c b/arch/arm/mach-imx/mm-imx3.c
index fe96105109b3..9d2c843bde02 100644
--- a/arch/arm/mach-imx/mm-imx3.c
+++ b/arch/arm/mach-imx/mm-imx3.c
@@ -272,10 +272,9 @@ void __init imx35_soc_init(void)
imx3_init_l2x0();
- /* i.mx35 has the i.mx31 type gpio */
- mxc_register_gpio("imx31-gpio", 0, MX35_GPIO1_BASE_ADDR, SZ_16K, MX35_INT_GPIO1, 0);
- mxc_register_gpio("imx31-gpio", 1, MX35_GPIO2_BASE_ADDR, SZ_16K, MX35_INT_GPIO2, 0);
- mxc_register_gpio("imx31-gpio", 2, MX35_GPIO3_BASE_ADDR, SZ_16K, MX35_INT_GPIO3, 0);
+ mxc_register_gpio("imx35-gpio", 0, MX35_GPIO1_BASE_ADDR, SZ_16K, MX35_INT_GPIO1, 0);
+ mxc_register_gpio("imx35-gpio", 1, MX35_GPIO2_BASE_ADDR, SZ_16K, MX35_INT_GPIO2, 0);
+ mxc_register_gpio("imx35-gpio", 2, MX35_GPIO3_BASE_ADDR, SZ_16K, MX35_INT_GPIO3, 0);
pinctrl_provide_dummies();
if (to_version == 1) {
diff --git a/arch/arm/mach-imx/mm-imx5.c b/arch/arm/mach-imx/mm-imx5.c
index f19d604e1b2a..52d8f534be10 100644
--- a/arch/arm/mach-imx/mm-imx5.c
+++ b/arch/arm/mach-imx/mm-imx5.c
@@ -161,13 +161,13 @@ static const struct resource imx53_audmux_res[] __initconst = {
void __init imx50_soc_init(void)
{
- /* i.mx50 has the i.mx31 type gpio */
- mxc_register_gpio("imx31-gpio", 0, MX50_GPIO1_BASE_ADDR, SZ_16K, MX50_INT_GPIO1_LOW, MX50_INT_GPIO1_HIGH);
- mxc_register_gpio("imx31-gpio", 1, MX50_GPIO2_BASE_ADDR, SZ_16K, MX50_INT_GPIO2_LOW, MX50_INT_GPIO2_HIGH);
- mxc_register_gpio("imx31-gpio", 2, MX50_GPIO3_BASE_ADDR, SZ_16K, MX50_INT_GPIO3_LOW, MX50_INT_GPIO3_HIGH);
- mxc_register_gpio("imx31-gpio", 3, MX50_GPIO4_BASE_ADDR, SZ_16K, MX50_INT_GPIO4_LOW, MX50_INT_GPIO4_HIGH);
- mxc_register_gpio("imx31-gpio", 4, MX50_GPIO5_BASE_ADDR, SZ_16K, MX50_INT_GPIO5_LOW, MX50_INT_GPIO5_HIGH);
- mxc_register_gpio("imx31-gpio", 5, MX50_GPIO6_BASE_ADDR, SZ_16K, MX50_INT_GPIO6_LOW, MX50_INT_GPIO6_HIGH);
+ /* i.mx50 has the i.mx35 type gpio */
+ mxc_register_gpio("imx35-gpio", 0, MX50_GPIO1_BASE_ADDR, SZ_16K, MX50_INT_GPIO1_LOW, MX50_INT_GPIO1_HIGH);
+ mxc_register_gpio("imx35-gpio", 1, MX50_GPIO2_BASE_ADDR, SZ_16K, MX50_INT_GPIO2_LOW, MX50_INT_GPIO2_HIGH);
+ mxc_register_gpio("imx35-gpio", 2, MX50_GPIO3_BASE_ADDR, SZ_16K, MX50_INT_GPIO3_LOW, MX50_INT_GPIO3_HIGH);
+ mxc_register_gpio("imx35-gpio", 3, MX50_GPIO4_BASE_ADDR, SZ_16K, MX50_INT_GPIO4_LOW, MX50_INT_GPIO4_HIGH);
+ mxc_register_gpio("imx35-gpio", 4, MX50_GPIO5_BASE_ADDR, SZ_16K, MX50_INT_GPIO5_LOW, MX50_INT_GPIO5_HIGH);
+ mxc_register_gpio("imx35-gpio", 5, MX50_GPIO6_BASE_ADDR, SZ_16K, MX50_INT_GPIO6_LOW, MX50_INT_GPIO6_HIGH);
/* i.mx50 has the i.mx31 type audmux */
platform_device_register_simple("imx31-audmux", 0, imx50_audmux_res,
@@ -176,11 +176,11 @@ void __init imx50_soc_init(void)
void __init imx51_soc_init(void)
{
- /* i.mx51 has the i.mx31 type gpio */
- mxc_register_gpio("imx31-gpio", 0, MX51_GPIO1_BASE_ADDR, SZ_16K, MX51_INT_GPIO1_LOW, MX51_INT_GPIO1_HIGH);
- mxc_register_gpio("imx31-gpio", 1, MX51_GPIO2_BASE_ADDR, SZ_16K, MX51_INT_GPIO2_LOW, MX51_INT_GPIO2_HIGH);
- mxc_register_gpio("imx31-gpio", 2, MX51_GPIO3_BASE_ADDR, SZ_16K, MX51_INT_GPIO3_LOW, MX51_INT_GPIO3_HIGH);
- mxc_register_gpio("imx31-gpio", 3, MX51_GPIO4_BASE_ADDR, SZ_16K, MX51_INT_GPIO4_LOW, MX51_INT_GPIO4_HIGH);
+ /* i.mx51 has the i.mx35 type gpio */
+ mxc_register_gpio("imx35-gpio", 0, MX51_GPIO1_BASE_ADDR, SZ_16K, MX51_INT_GPIO1_LOW, MX51_INT_GPIO1_HIGH);
+ mxc_register_gpio("imx35-gpio", 1, MX51_GPIO2_BASE_ADDR, SZ_16K, MX51_INT_GPIO2_LOW, MX51_INT_GPIO2_HIGH);
+ mxc_register_gpio("imx35-gpio", 2, MX51_GPIO3_BASE_ADDR, SZ_16K, MX51_INT_GPIO3_LOW, MX51_INT_GPIO3_HIGH);
+ mxc_register_gpio("imx35-gpio", 3, MX51_GPIO4_BASE_ADDR, SZ_16K, MX51_INT_GPIO4_LOW, MX51_INT_GPIO4_HIGH);
pinctrl_provide_dummies();
@@ -198,14 +198,14 @@ void __init imx51_soc_init(void)
void __init imx53_soc_init(void)
{
- /* i.mx53 has the i.mx31 type gpio */
- mxc_register_gpio("imx31-gpio", 0, MX53_GPIO1_BASE_ADDR, SZ_16K, MX53_INT_GPIO1_LOW, MX53_INT_GPIO1_HIGH);
- mxc_register_gpio("imx31-gpio", 1, MX53_GPIO2_BASE_ADDR, SZ_16K, MX53_INT_GPIO2_LOW, MX53_INT_GPIO2_HIGH);
- mxc_register_gpio("imx31-gpio", 2, MX53_GPIO3_BASE_ADDR, SZ_16K, MX53_INT_GPIO3_LOW, MX53_INT_GPIO3_HIGH);
- mxc_register_gpio("imx31-gpio", 3, MX53_GPIO4_BASE_ADDR, SZ_16K, MX53_INT_GPIO4_LOW, MX53_INT_GPIO4_HIGH);
- mxc_register_gpio("imx31-gpio", 4, MX53_GPIO5_BASE_ADDR, SZ_16K, MX53_INT_GPIO5_LOW, MX53_INT_GPIO5_HIGH);
- mxc_register_gpio("imx31-gpio", 5, MX53_GPIO6_BASE_ADDR, SZ_16K, MX53_INT_GPIO6_LOW, MX53_INT_GPIO6_HIGH);
- mxc_register_gpio("imx31-gpio", 6, MX53_GPIO7_BASE_ADDR, SZ_16K, MX53_INT_GPIO7_LOW, MX53_INT_GPIO7_HIGH);
+ /* i.mx53 has the i.mx35 type gpio */
+ mxc_register_gpio("imx35-gpio", 0, MX53_GPIO1_BASE_ADDR, SZ_16K, MX53_INT_GPIO1_LOW, MX53_INT_GPIO1_HIGH);
+ mxc_register_gpio("imx35-gpio", 1, MX53_GPIO2_BASE_ADDR, SZ_16K, MX53_INT_GPIO2_LOW, MX53_INT_GPIO2_HIGH);
+ mxc_register_gpio("imx35-gpio", 2, MX53_GPIO3_BASE_ADDR, SZ_16K, MX53_INT_GPIO3_LOW, MX53_INT_GPIO3_HIGH);
+ mxc_register_gpio("imx35-gpio", 3, MX53_GPIO4_BASE_ADDR, SZ_16K, MX53_INT_GPIO4_LOW, MX53_INT_GPIO4_HIGH);
+ mxc_register_gpio("imx35-gpio", 4, MX53_GPIO5_BASE_ADDR, SZ_16K, MX53_INT_GPIO5_LOW, MX53_INT_GPIO5_HIGH);
+ mxc_register_gpio("imx35-gpio", 5, MX53_GPIO6_BASE_ADDR, SZ_16K, MX53_INT_GPIO6_LOW, MX53_INT_GPIO6_HIGH);
+ mxc_register_gpio("imx35-gpio", 6, MX53_GPIO7_BASE_ADDR, SZ_16K, MX53_INT_GPIO7_LOW, MX53_INT_GPIO7_HIGH);
pinctrl_provide_dummies();
/* i.mx53 has the i.mx35 type sdma */
diff --git a/arch/arm/mach-omap2/clock3xxx_data.c b/arch/arm/mach-omap2/clock3xxx_data.c
index 91b3d5c60bfe..83bed9ad3017 100644
--- a/arch/arm/mach-omap2/clock3xxx_data.c
+++ b/arch/arm/mach-omap2/clock3xxx_data.c
@@ -3376,15 +3376,15 @@ static struct omap_clk omap3xxx_clks[] = {
CLK(NULL, "usbhost_48m_fck", &usbhost_48m_fck, CK_3430ES2PLUS | CK_AM35XX | CK_36XX),
CLK(NULL, "usbhost_ick", &usbhost_ick, CK_3430ES2PLUS | CK_AM35XX | CK_36XX),
CLK("usbhs_omap", "usbhost_ick", &usbhost_ick, CK_3430ES2PLUS | CK_AM35XX | CK_36XX),
- CLK("usbhs_omap", "utmi_p1_gfclk", &dummy_ck, CK_3XXX),
- CLK("usbhs_omap", "utmi_p2_gfclk", &dummy_ck, CK_3XXX),
- CLK("usbhs_omap", "xclk60mhsp1_ck", &dummy_ck, CK_3XXX),
- CLK("usbhs_omap", "xclk60mhsp2_ck", &dummy_ck, CK_3XXX),
- CLK("usbhs_omap", "usb_host_hs_utmi_p1_clk", &dummy_ck, CK_3XXX),
- CLK("usbhs_omap", "usb_host_hs_utmi_p2_clk", &dummy_ck, CK_3XXX),
+ CLK(NULL, "utmi_p1_gfclk", &dummy_ck, CK_3XXX),
+ CLK(NULL, "utmi_p2_gfclk", &dummy_ck, CK_3XXX),
+ CLK(NULL, "xclk60mhsp1_ck", &dummy_ck, CK_3XXX),
+ CLK(NULL, "xclk60mhsp2_ck", &dummy_ck, CK_3XXX),
+ CLK(NULL, "usb_host_hs_utmi_p1_clk", &dummy_ck, CK_3XXX),
+ CLK(NULL, "usb_host_hs_utmi_p2_clk", &dummy_ck, CK_3XXX),
CLK("usbhs_omap", "usb_tll_hs_usb_ch0_clk", &dummy_ck, CK_3XXX),
CLK("usbhs_omap", "usb_tll_hs_usb_ch1_clk", &dummy_ck, CK_3XXX),
- CLK("usbhs_omap", "init_60m_fclk", &dummy_ck, CK_3XXX),
+ CLK(NULL, "init_60m_fclk", &dummy_ck, CK_3XXX),
CLK(NULL, "usim_fck", &usim_fck, CK_3430ES2PLUS | CK_36XX),
CLK(NULL, "gpt1_fck", &gpt1_fck, CK_3XXX),
CLK(NULL, "wkup_32k_fck", &wkup_32k_fck, CK_3XXX),
diff --git a/arch/arm/plat-spear/include/plat/keyboard.h b/arch/arm/plat-spear/include/plat/keyboard.h
index 0562f134621d..9248e3a7e333 100644
--- a/arch/arm/plat-spear/include/plat/keyboard.h
+++ b/arch/arm/plat-spear/include/plat/keyboard.h
@@ -149,6 +149,7 @@ int _name[] = { \
* keymap: pointer to keymap data (table and size)
* rep: enables key autorepeat
* mode: choose keyboard support(9x9, 6x6, 2x2)
+ * suspended_rate: rate at which keyboard would operate in suspended mode
*
* This structure is supposed to be used by platform code to supply
* keymaps to drivers that implement keyboards.
@@ -157,6 +158,7 @@ struct kbd_platform_data {
const struct matrix_keymap_data *keymap;
bool rep;
unsigned int mode;
+ unsigned int suspended_rate;
};
#endif /* __PLAT_KEYBOARD_H */
diff --git a/arch/m68k/Kconfig.bus b/arch/m68k/Kconfig.bus
index 3adb499584fb..ffc0601a2a19 100644
--- a/arch/m68k/Kconfig.bus
+++ b/arch/m68k/Kconfig.bus
@@ -48,6 +48,13 @@ config ISA
config GENERIC_ISA_DMA
def_bool ISA
+config PCI
+ bool "PCI support"
+ depends on M54xx
+ help
+ Enable the PCI bus. Support for the PCI bus hardware built into the
+ ColdFire 547x and 548x processors.
+
source "drivers/pci/Kconfig"
source "drivers/zorro/Kconfig"
diff --git a/arch/m68k/Kconfig.cpu b/arch/m68k/Kconfig.cpu
index 2b53254ad994..43a9f8f1b8eb 100644
--- a/arch/m68k/Kconfig.cpu
+++ b/arch/m68k/Kconfig.cpu
@@ -23,7 +23,7 @@ config M68KCLASSIC
config COLDFIRE
bool "Coldfire CPU family support"
select GENERIC_GPIO
- select ARCH_REQUIRE_GPIOLIB
+ select ARCH_WANT_OPTIONAL_GPIOLIB
select ARCH_HAVE_CUSTOM_GPIO_H
select CPU_HAS_NO_BITFIELDS
select CPU_HAS_NO_MULDIV64
@@ -167,6 +167,14 @@ config M5249
help
Motorola ColdFire 5249 processor support.
+config M525x
+ bool "MCF525x"
+ depends on !MMU
+ select COLDFIRE_SW_A7
+ select HAVE_MBAR
+ help
+ Freescale (Motorola) Coldfire 5251/5253 processor support.
+
config M527x
bool
@@ -253,6 +261,14 @@ config M548x
help
Freescale ColdFire 5480/5481/5482/5483/5484/5485 processor support.
+config M5441x
+ bool "MCF5441x"
+ depends on !MMU
+ select GENERIC_CLOCKEVENTS
+ select HAVE_CACHE_CB
+ help
+ Freescale Coldfire 54410/54415/54416/54417/54418 processor support.
+
endif # COLDFIRE
diff --git a/arch/m68k/Makefile b/arch/m68k/Makefile
index b7f2e2d5cd2e..7636751f2f87 100644
--- a/arch/m68k/Makefile
+++ b/arch/m68k/Makefile
@@ -41,6 +41,7 @@ cpuflags-$(CONFIG_M68030) :=
cpuflags-$(CONFIG_M68020) :=
cpuflags-$(CONFIG_M68360) := -m68332
cpuflags-$(CONFIG_M68000) := -m68000
+cpuflags-$(CONFIG_M5441x) := $(call cc-option,-mcpu=54455,-mcfv4e)
cpuflags-$(CONFIG_M54xx) := $(call cc-option,-mcpu=5475,-m5200)
cpuflags-$(CONFIG_M5407) := $(call cc-option,-mcpu=5407,-m5200)
cpuflags-$(CONFIG_M532x) := $(call cc-option,-mcpu=532x,-m5307)
@@ -50,6 +51,7 @@ cpuflags-$(CONFIG_M5275) := $(call cc-option,-mcpu=5275,-m5307)
cpuflags-$(CONFIG_M5272) := $(call cc-option,-mcpu=5272,-m5307)
cpuflags-$(CONFIG_M5271) := $(call cc-option,-mcpu=5271,-m5307)
cpuflags-$(CONFIG_M523x) := $(call cc-option,-mcpu=523x,-m5307)
+cpuflags-$(CONFIG_M525x) := $(call cc-option,-mcpu=5253,-m5200)
cpuflags-$(CONFIG_M5249) := $(call cc-option,-mcpu=5249,-m5200)
cpuflags-$(CONFIG_M520x) := $(call cc-option,-mcpu=5208,-m5200)
cpuflags-$(CONFIG_M5206e) := $(call cc-option,-mcpu=5206e,-m5200)
diff --git a/arch/m68k/include/asm/cacheflush_mm.h b/arch/m68k/include/asm/cacheflush_mm.h
index 8104bd874649..fa2c3d681d84 100644
--- a/arch/m68k/include/asm/cacheflush_mm.h
+++ b/arch/m68k/include/asm/cacheflush_mm.h
@@ -16,7 +16,48 @@
#define DCACHE_MAX_ADDR 0
#define DCACHE_SETMASK 0
#endif
+#ifndef CACHE_MODE
+#define CACHE_MODE 0
+#define CACR_ICINVA 0
+#define CACR_DCINVA 0
+#define CACR_BCINVA 0
+#endif
+
+/*
+ * ColdFire architecture has no way to clear individual cache lines, so we
+ * are stuck invalidating all the cache entries when we want a clear operation.
+ */
+static inline void clear_cf_icache(unsigned long start, unsigned long end)
+{
+ __asm__ __volatile__ (
+ "movec %0,%%cacr\n\t"
+ "nop"
+ :
+ : "r" (CACHE_MODE | CACR_ICINVA | CACR_BCINVA));
+}
+
+static inline void clear_cf_dcache(unsigned long start, unsigned long end)
+{
+ __asm__ __volatile__ (
+ "movec %0,%%cacr\n\t"
+ "nop"
+ :
+ : "r" (CACHE_MODE | CACR_DCINVA));
+}
+static inline void clear_cf_bcache(unsigned long start, unsigned long end)
+{
+ __asm__ __volatile__ (
+ "movec %0,%%cacr\n\t"
+ "nop"
+ :
+ : "r" (CACHE_MODE | CACR_ICINVA | CACR_BCINVA | CACR_DCINVA));
+}
+
+/*
+ * Use the ColdFire cpushl instruction to push (and invalidate) cache lines.
+ * The start and end addresses are cache line numbers not memory addresses.
+ */
static inline void flush_cf_icache(unsigned long start, unsigned long end)
{
unsigned long set;
diff --git a/arch/m68k/include/asm/dma.h b/arch/m68k/include/asm/dma.h
index 6fbdfe895104..0ff3fc6a6d9a 100644
--- a/arch/m68k/include/asm/dma.h
+++ b/arch/m68k/include/asm/dma.h
@@ -33,7 +33,9 @@
* Set number of channels of DMA on ColdFire for different implementations.
*/
#if defined(CONFIG_M5249) || defined(CONFIG_M5307) || defined(CONFIG_M5407) || \
- defined(CONFIG_M523x) || defined(CONFIG_M527x) || defined(CONFIG_M528x)
+ defined(CONFIG_M523x) || defined(CONFIG_M527x) || \
+ defined(CONFIG_M528x) || defined(CONFIG_M525x)
+
#define MAX_M68K_DMA_CHANNELS 4
#elif defined(CONFIG_M5272)
#define MAX_M68K_DMA_CHANNELS 1
@@ -486,6 +488,10 @@ static __inline__ int get_dma_residue(unsigned int dmanr)
extern int request_dma(unsigned int dmanr, const char * device_id); /* reserve a DMA channel */
extern void free_dma(unsigned int dmanr); /* release it again */
+#ifdef CONFIG_PCI
+extern int isa_dma_bridge_buggy;
+#else
#define isa_dma_bridge_buggy (0)
+#endif
#endif /* _M68K_DMA_H */
diff --git a/arch/m68k/include/asm/gpio.h b/arch/m68k/include/asm/gpio.h
index 00d0071de4c3..4395ffc51fdb 100644
--- a/arch/m68k/include/asm/gpio.h
+++ b/arch/m68k/include/asm/gpio.h
@@ -17,170 +17,9 @@
#define coldfire_gpio_h
#include <linux/io.h>
-#include <asm-generic/gpio.h>
#include <asm/coldfire.h>
#include <asm/mcfsim.h>
-
-/*
- * The Freescale Coldfire family is quite varied in how they implement GPIO.
- * Some parts have 8 bit ports, some have 16bit and some have 32bit; some have
- * only one port, others have multiple ports; some have a single data latch
- * for both input and output, others have a separate pin data register to read
- * input; some require a read-modify-write access to change an output, others
- * have set and clear registers for some of the outputs; Some have all the
- * GPIOs in a single control area, others have some GPIOs implemented in
- * different modules.
- *
- * This implementation attempts accommodate the differences while presenting
- * a generic interface that will optimize to as few instructions as possible.
- */
-#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \
- defined(CONFIG_M520x) || defined(CONFIG_M523x) || \
- defined(CONFIG_M527x) || defined(CONFIG_M528x) || \
- defined(CONFIG_M532x) || defined(CONFIG_M54xx)
-
-/* These parts have GPIO organized by 8 bit ports */
-
-#define MCFGPIO_PORTTYPE u8
-#define MCFGPIO_PORTSIZE 8
-#define mcfgpio_read(port) __raw_readb(port)
-#define mcfgpio_write(data, port) __raw_writeb(data, port)
-
-#elif defined(CONFIG_M5307) || defined(CONFIG_M5407) || defined(CONFIG_M5272)
-
-/* These parts have GPIO organized by 16 bit ports */
-
-#define MCFGPIO_PORTTYPE u16
-#define MCFGPIO_PORTSIZE 16
-#define mcfgpio_read(port) __raw_readw(port)
-#define mcfgpio_write(data, port) __raw_writew(data, port)
-
-#elif defined(CONFIG_M5249)
-
-/* These parts have GPIO organized by 32 bit ports */
-
-#define MCFGPIO_PORTTYPE u32
-#define MCFGPIO_PORTSIZE 32
-#define mcfgpio_read(port) __raw_readl(port)
-#define mcfgpio_write(data, port) __raw_writel(data, port)
-
-#endif
-
-#define mcfgpio_bit(gpio) (1 << ((gpio) % MCFGPIO_PORTSIZE))
-#define mcfgpio_port(gpio) ((gpio) / MCFGPIO_PORTSIZE)
-
-#if defined(CONFIG_M520x) || defined(CONFIG_M523x) || \
- defined(CONFIG_M527x) || defined(CONFIG_M528x) || defined(CONFIG_M532x)
-/*
- * These parts have an 'Edge' Port module (external interrupt/GPIO) which uses
- * read-modify-write to change an output and a GPIO module which has separate
- * set/clr registers to directly change outputs with a single write access.
- */
-#if defined(CONFIG_M528x)
-/*
- * The 528x also has GPIOs in other modules (GPT, QADC) which use
- * read-modify-write as well as those controlled by the EPORT and GPIO modules.
- */
-#define MCFGPIO_SCR_START 40
-#else
-#define MCFGPIO_SCR_START 8
-#endif
-
-#define MCFGPIO_SETR_PORT(gpio) (MCFGPIO_SETR + \
- mcfgpio_port(gpio - MCFGPIO_SCR_START))
-
-#define MCFGPIO_CLRR_PORT(gpio) (MCFGPIO_CLRR + \
- mcfgpio_port(gpio - MCFGPIO_SCR_START))
-#else
-
-#define MCFGPIO_SCR_START MCFGPIO_PIN_MAX
-/* with MCFGPIO_SCR == MCFGPIO_PIN_MAX, these will be optimized away */
-#define MCFGPIO_SETR_PORT(gpio) 0
-#define MCFGPIO_CLRR_PORT(gpio) 0
-
-#endif
-/*
- * Coldfire specific helper functions
- */
-
-/* return the port pin data register for a gpio */
-static inline u32 __mcf_gpio_ppdr(unsigned gpio)
-{
-#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \
- defined(CONFIG_M5307) || defined(CONFIG_M5407)
- return MCFSIM_PADAT;
-#elif defined(CONFIG_M5272)
- if (gpio < 16)
- return MCFSIM_PADAT;
- else if (gpio < 32)
- return MCFSIM_PBDAT;
- else
- return MCFSIM_PCDAT;
-#elif defined(CONFIG_M5249)
- if (gpio < 32)
- return MCFSIM2_GPIOREAD;
- else
- return MCFSIM2_GPIO1READ;
-#elif defined(CONFIG_M520x) || defined(CONFIG_M523x) || \
- defined(CONFIG_M527x) || defined(CONFIG_M528x) || defined(CONFIG_M532x)
- if (gpio < 8)
- return MCFEPORT_EPPDR;
-#if defined(CONFIG_M528x)
- else if (gpio < 16)
- return MCFGPTA_GPTPORT;
- else if (gpio < 24)
- return MCFGPTB_GPTPORT;
- else if (gpio < 32)
- return MCFQADC_PORTQA;
- else if (gpio < 40)
- return MCFQADC_PORTQB;
-#endif
- else
- return MCFGPIO_PPDR + mcfgpio_port(gpio - MCFGPIO_SCR_START);
-#else
- return 0;
-#endif
-}
-
-/* return the port output data register for a gpio */
-static inline u32 __mcf_gpio_podr(unsigned gpio)
-{
-#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \
- defined(CONFIG_M5307) || defined(CONFIG_M5407)
- return MCFSIM_PADAT;
-#elif defined(CONFIG_M5272)
- if (gpio < 16)
- return MCFSIM_PADAT;
- else if (gpio < 32)
- return MCFSIM_PBDAT;
- else
- return MCFSIM_PCDAT;
-#elif defined(CONFIG_M5249)
- if (gpio < 32)
- return MCFSIM2_GPIOWRITE;
- else
- return MCFSIM2_GPIO1WRITE;
-#elif defined(CONFIG_M520x) || defined(CONFIG_M523x) || \
- defined(CONFIG_M527x) || defined(CONFIG_M528x) || defined(CONFIG_M532x)
- if (gpio < 8)
- return MCFEPORT_EPDR;
-#if defined(CONFIG_M528x)
- else if (gpio < 16)
- return MCFGPTA_GPTPORT;
- else if (gpio < 24)
- return MCFGPTB_GPTPORT;
- else if (gpio < 32)
- return MCFQADC_PORTQA;
- else if (gpio < 40)
- return MCFQADC_PORTQB;
-#endif
- else
- return MCFGPIO_PODR + mcfgpio_port(gpio - MCFGPIO_SCR_START);
-#else
- return 0;
-#endif
-}
-
+#include <asm/mcfgpio.h>
/*
* The Generic GPIO functions
*
@@ -191,7 +30,7 @@ static inline u32 __mcf_gpio_podr(unsigned gpio)
static inline int gpio_get_value(unsigned gpio)
{
if (__builtin_constant_p(gpio) && gpio < MCFGPIO_PIN_MAX)
- return mcfgpio_read(__mcf_gpio_ppdr(gpio)) & mcfgpio_bit(gpio);
+ return mcfgpio_read(__mcfgpio_ppdr(gpio)) & mcfgpio_bit(gpio);
else
return __gpio_get_value(gpio);
}
@@ -204,12 +43,12 @@ static inline void gpio_set_value(unsigned gpio, int value)
MCFGPIO_PORTTYPE data;
local_irq_save(flags);
- data = mcfgpio_read(__mcf_gpio_podr(gpio));
+ data = mcfgpio_read(__mcfgpio_podr(gpio));
if (value)
data |= mcfgpio_bit(gpio);
else
data &= ~mcfgpio_bit(gpio);
- mcfgpio_write(data, __mcf_gpio_podr(gpio));
+ mcfgpio_write(data, __mcfgpio_podr(gpio));
local_irq_restore(flags);
} else {
if (value)
@@ -225,8 +64,14 @@ static inline void gpio_set_value(unsigned gpio, int value)
static inline int gpio_to_irq(unsigned gpio)
{
- return (gpio < MCFGPIO_IRQ_MAX) ? gpio + MCFGPIO_IRQ_VECBASE
- : __gpio_to_irq(gpio);
+#if defined(MCFGPIO_IRQ_MIN)
+ if ((gpio >= MCFGPIO_IRQ_MIN) && (gpio < MCFGPIO_IRQ_MAX))
+#else
+ if (gpio < MCFGPIO_IRQ_MAX)
+#endif
+ return gpio + MCFGPIO_IRQ_VECBASE;
+ else
+ return __gpio_to_irq(gpio);
}
static inline int irq_to_gpio(unsigned irq)
diff --git a/arch/m68k/include/asm/io_mm.h b/arch/m68k/include/asm/io_mm.h
index fa4324bcf566..a6686d26fe17 100644
--- a/arch/m68k/include/asm/io_mm.h
+++ b/arch/m68k/include/asm/io_mm.h
@@ -65,7 +65,53 @@
-#ifdef CONFIG_ISA
+#if defined(CONFIG_PCI) && defined(CONFIG_COLDFIRE)
+
+#define HAVE_ARCH_PIO_SIZE
+#define PIO_OFFSET 0
+#define PIO_MASK 0xffff
+#define PIO_RESERVED 0x10000
+
+u8 mcf_pci_inb(u32 addr);
+u16 mcf_pci_inw(u32 addr);
+u32 mcf_pci_inl(u32 addr);
+void mcf_pci_insb(u32 addr, u8 *buf, u32 len);
+void mcf_pci_insw(u32 addr, u16 *buf, u32 len);
+void mcf_pci_insl(u32 addr, u32 *buf, u32 len);
+
+void mcf_pci_outb(u8 v, u32 addr);
+void mcf_pci_outw(u16 v, u32 addr);
+void mcf_pci_outl(u32 v, u32 addr);
+void mcf_pci_outsb(u32 addr, const u8 *buf, u32 len);
+void mcf_pci_outsw(u32 addr, const u16 *buf, u32 len);
+void mcf_pci_outsl(u32 addr, const u32 *buf, u32 len);
+
+#define inb mcf_pci_inb
+#define inb_p mcf_pci_inb
+#define inw mcf_pci_inw
+#define inw_p mcf_pci_inw
+#define inl mcf_pci_inl
+#define inl_p mcf_pci_inl
+#define insb mcf_pci_insb
+#define insw mcf_pci_insw
+#define insl mcf_pci_insl
+
+#define outb mcf_pci_outb
+#define outb_p mcf_pci_outb
+#define outw mcf_pci_outw
+#define outw_p mcf_pci_outw
+#define outl mcf_pci_outl
+#define outl_p mcf_pci_outl
+#define outsb mcf_pci_outsb
+#define outsw mcf_pci_outsw
+#define outsl mcf_pci_outsl
+
+#define readb(addr) in_8(addr)
+#define writeb(v, addr) out_8((addr), (v))
+#define readw(addr) in_le16(addr)
+#define writew(v, addr) out_le16((addr), (v))
+
+#elif defined(CONFIG_ISA)
#if MULTI_ISA == 0
#undef MULTI_ISA
@@ -340,4 +386,6 @@ static inline void memcpy_toio(volatile void __iomem *dst, const void *src, int
*/
#define xlate_dev_kmem_ptr(p) p
+#define ioport_map(port, nr) ((void __iomem *)(port))
+
#endif /* _IO_H */
diff --git a/arch/m68k/include/asm/m520xsim.h b/arch/m68k/include/asm/m520xsim.h
index 17f2aab9cf97..db3f8ee4a6c6 100644
--- a/arch/m68k/include/asm/m520xsim.h
+++ b/arch/m68k/include/asm/m520xsim.h
@@ -42,6 +42,9 @@
#define MCFINTC1_SIMR (0)
#define MCFINTC1_CIMR (0)
#define MCFINTC1_ICR0 (0)
+#define MCFINTC2_SIMR (0)
+#define MCFINTC2_CIMR (0)
+#define MCFINTC2_ICR0 (0)
#define MCFINT_VECBASE 64
#define MCFINT_UART0 26 /* Interrupt number for UART0 */
@@ -62,6 +65,7 @@
#define MCF_IRQ_FECENTC0 (MCFINT_VECBASE + MCFINT_FECENTC0)
#define MCF_IRQ_QSPI (MCFINT_VECBASE + MCFINT_QSPI)
+#define MCF_IRQ_PIT1 (MCFINT_VECBASE + MCFINT_PIT1)
/*
* SDRAM configuration registers.
@@ -186,5 +190,15 @@
#define MCF_RCR_SWRESET 0x80 /* Software reset bit */
#define MCF_RCR_FRCSTOUT 0x40 /* Force external reset */
+/*
+ * Power Management.
+ */
+#define MCFPM_WCR 0xfc040013
+#define MCFPM_PPMSR0 0xfc04002c
+#define MCFPM_PPMCR0 0xfc04002d
+#define MCFPM_PPMHR0 0xfc040030
+#define MCFPM_PPMLR0 0xfc040034
+#define MCFPM_LPCR 0xfc0a0007
+
/****************************************************************************/
#endif /* m520xsim_h */
diff --git a/arch/m68k/include/asm/m523xsim.h b/arch/m68k/include/asm/m523xsim.h
index 075062d4eecd..91d3abc3f2a5 100644
--- a/arch/m68k/include/asm/m523xsim.h
+++ b/arch/m68k/include/asm/m523xsim.h
@@ -52,6 +52,7 @@
#define MCF_IRQ_FECENTC0 (MCFINT_VECBASE + MCFINT_FECENTC0)
#define MCF_IRQ_QSPI (MCFINT_VECBASE + MCFINT_QSPI)
+#define MCF_IRQ_PIT1 (MCFINT_VECBASE + MCFINT_PIT1)
/*
* SDRAM configuration registers.
diff --git a/arch/m68k/include/asm/m525xsim.h b/arch/m68k/include/asm/m525xsim.h
new file mode 100644
index 000000000000..6da24f653902
--- /dev/null
+++ b/arch/m68k/include/asm/m525xsim.h
@@ -0,0 +1,194 @@
+/****************************************************************************/
+
+/*
+ * m525xsim.h -- ColdFire 525x System Integration Module support.
+ *
+ * (C) Copyright 2012, Steven king <sfking@fdwdc.com>
+ * (C) Copyright 2002, Greg Ungerer (gerg@snapgear.com)
+ */
+
+/****************************************************************************/
+#ifndef m525xsim_h
+#define m525xsim_h
+/****************************************************************************/
+
+#define CPU_NAME "COLDFIRE(m525x)"
+#define CPU_INSTR_PER_JIFFY 3
+#define MCF_BUSCLK (MCF_CLK / 2)
+
+#include <asm/m52xxacr.h>
+
+/*
+ * The 525x has a second MBAR region, define its address.
+ */
+#define MCF_MBAR2 0x80000000
+
+/*
+ * Define the 525x SIM register set addresses.
+ */
+#define MCFSIM_RSR 0x00 /* Reset Status reg (r/w) */
+#define MCFSIM_SYPCR 0x01 /* System Protection reg (r/w)*/
+#define MCFSIM_SWIVR 0x02 /* SW Watchdog intr reg (r/w) */
+#define MCFSIM_SWSR 0x03 /* SW Watchdog service (r/w) */
+#define MCFSIM_MPARK 0x0C /* BUS Master Control Reg*/
+#define MCFSIM_IPR 0x40 /* Interrupt Pend reg (r/w) */
+#define MCFSIM_IMR 0x44 /* Interrupt Mask reg (r/w) */
+#define MCFSIM_ICR0 0x4c /* Intr Ctrl reg 0 (r/w) */
+#define MCFSIM_ICR1 0x4d /* Intr Ctrl reg 1 (r/w) */
+#define MCFSIM_ICR2 0x4e /* Intr Ctrl reg 2 (r/w) */
+#define MCFSIM_ICR3 0x4f /* Intr Ctrl reg 3 (r/w) */
+#define MCFSIM_ICR4 0x50 /* Intr Ctrl reg 4 (r/w) */
+#define MCFSIM_ICR5 0x51 /* Intr Ctrl reg 5 (r/w) */
+#define MCFSIM_ICR6 0x52 /* Intr Ctrl reg 6 (r/w) */
+#define MCFSIM_ICR7 0x53 /* Intr Ctrl reg 7 (r/w) */
+#define MCFSIM_ICR8 0x54 /* Intr Ctrl reg 8 (r/w) */
+#define MCFSIM_ICR9 0x55 /* Intr Ctrl reg 9 (r/w) */
+#define MCFSIM_ICR10 0x56 /* Intr Ctrl reg 10 (r/w) */
+#define MCFSIM_ICR11 0x57 /* Intr Ctrl reg 11 (r/w) */
+
+#define MCFSIM_CSAR0 0x80 /* CS 0 Address 0 reg (r/w) */
+#define MCFSIM_CSMR0 0x84 /* CS 0 Mask 0 reg (r/w) */
+#define MCFSIM_CSCR0 0x8a /* CS 0 Control reg (r/w) */
+#define MCFSIM_CSAR1 0x8c /* CS 1 Address reg (r/w) */
+#define MCFSIM_CSMR1 0x90 /* CS 1 Mask reg (r/w) */
+#define MCFSIM_CSCR1 0x96 /* CS 1 Control reg (r/w) */
+#define MCFSIM_CSAR2 0x98 /* CS 2 Address reg (r/w) */
+#define MCFSIM_CSMR2 0x9c /* CS 2 Mask reg (r/w) */
+#define MCFSIM_CSCR2 0xa2 /* CS 2 Control reg (r/w) */
+#define MCFSIM_CSAR3 0xa4 /* CS 3 Address reg (r/w) */
+#define MCFSIM_CSMR3 0xa8 /* CS 3 Mask reg (r/w) */
+#define MCFSIM_CSCR3 0xae /* CS 3 Control reg (r/w) */
+#define MCFSIM_CSAR4 0xb0 /* CS 4 Address reg (r/w) */
+#define MCFSIM_CSMR4 0xb4 /* CS 4 Mask reg (r/w) */
+#define MCFSIM_CSCR4 0xba /* CS 4 Control reg (r/w) */
+
+#define MCFSIM_DCR (MCF_MBAR + 0x100) /* DRAM Control */
+#define MCFSIM_DACR0 (MCF_MBAR + 0x108) /* DRAM 0 Addr/Ctrl */
+#define MCFSIM_DMR0 (MCF_MBAR + 0x10c) /* DRAM 0 Mask */
+
+/*
+ * Secondary Interrupt Controller (in MBAR2)
+*/
+#define MCFINTC2_INTBASE (MCF_MBAR2 + 0x168) /* Base Vector Reg */
+#define MCFINTC2_INTPRI1 (MCF_MBAR2 + 0x140) /* 0-7 priority */
+#define MCFINTC2_INTPRI2 (MCF_MBAR2 + 0x144) /* 8-15 priority */
+#define MCFINTC2_INTPRI3 (MCF_MBAR2 + 0x148) /* 16-23 priority */
+#define MCFINTC2_INTPRI4 (MCF_MBAR2 + 0x14c) /* 24-31 priority */
+#define MCFINTC2_INTPRI5 (MCF_MBAR2 + 0x150) /* 32-39 priority */
+#define MCFINTC2_INTPRI6 (MCF_MBAR2 + 0x154) /* 40-47 priority */
+#define MCFINTC2_INTPRI7 (MCF_MBAR2 + 0x158) /* 48-55 priority */
+#define MCFINTC2_INTPRI8 (MCF_MBAR2 + 0x15c) /* 56-63 priority */
+
+#define MCFINTC2_INTPRI_REG(i) (MCFINTC2_INTPRI1 + \
+ ((((i) - MCFINTC2_VECBASE) / 8) * 4))
+#define MCFINTC2_INTPRI_BITS(b, i) ((b) << (((i) % 8) * 4))
+
+/*
+ * Timer module.
+ */
+#define MCFTIMER_BASE1 (MCF_MBAR + 0x140) /* Base of TIMER1 */
+#define MCFTIMER_BASE2 (MCF_MBAR + 0x180) /* Base of TIMER2 */
+
+/*
+ * UART module.
+ */
+#define MCFUART_BASE0 (MCF_MBAR + 0x1c0) /* Base address UART0 */
+#define MCFUART_BASE1 (MCF_MBAR + 0x200) /* Base address UART1 */
+
+/*
+ * QSPI module.
+ */
+#define MCFQSPI_BASE (MCF_MBAR + 0x300) /* Base address QSPI */
+#define MCFQSPI_SIZE 0x40 /* Register set size */
+
+
+#define MCFQSPI_CS0 15
+#define MCFQSPI_CS1 16
+#define MCFQSPI_CS2 24
+#define MCFQSPI_CS3 28
+
+/*
+ * I2C module.
+ */
+#define MCFI2C_BASE0 (MCF_MBAR + 0x280) /* Base addreess I2C0 */
+#define MCFI2C_SIZE0 0x20 /* Register set size */
+
+#define MCFI2C_BASE1 (MCF_MBAR2 + 0x440) /* Base addreess I2C1 */
+#define MCFI2C_SIZE1 0x20 /* Register set size */
+/*
+ * DMA unit base addresses.
+ */
+#define MCFDMA_BASE0 (MCF_MBAR + 0x300) /* Base address DMA 0 */
+#define MCFDMA_BASE1 (MCF_MBAR + 0x340) /* Base address DMA 1 */
+#define MCFDMA_BASE2 (MCF_MBAR + 0x380) /* Base address DMA 2 */
+#define MCFDMA_BASE3 (MCF_MBAR + 0x3C0) /* Base address DMA 3 */
+
+/*
+ * Some symbol defines for the above...
+ */
+#define MCFSIM_SWDICR MCFSIM_ICR0 /* Watchdog timer ICR */
+#define MCFSIM_TIMER1ICR MCFSIM_ICR1 /* Timer 1 ICR */
+#define MCFSIM_TIMER2ICR MCFSIM_ICR2 /* Timer 2 ICR */
+#define MCFSIM_I2CICR MCFSIM_ICR3 /* I2C ICR */
+#define MCFSIM_UART1ICR MCFSIM_ICR4 /* UART 1 ICR */
+#define MCFSIM_UART2ICR MCFSIM_ICR5 /* UART 2 ICR */
+#define MCFSIM_DMA0ICR MCFSIM_ICR6 /* DMA 0 ICR */
+#define MCFSIM_DMA1ICR MCFSIM_ICR7 /* DMA 1 ICR */
+#define MCFSIM_DMA2ICR MCFSIM_ICR8 /* DMA 2 ICR */
+#define MCFSIM_DMA3ICR MCFSIM_ICR9 /* DMA 3 ICR */
+#define MCFSIM_QSPIICR MCFSIM_ICR10 /* QSPI ICR */
+
+/*
+ * Define system peripheral IRQ usage.
+ */
+#define MCF_IRQ_QSPI 28 /* QSPI, Level 4 */
+#define MCF_IRQ_I2C0 29
+#define MCF_IRQ_TIMER 30 /* Timer0, Level 6 */
+#define MCF_IRQ_PROFILER 31 /* Timer1, Level 7 */
+
+#define MCF_IRQ_UART0 73 /* UART0 */
+#define MCF_IRQ_UART1 74 /* UART1 */
+
+/*
+ * Define the base interrupt for the second interrupt controller.
+ * We set it to 128, out of the way of the base interrupts, and plenty
+ * of room for its 64 interrupts.
+ */
+#define MCFINTC2_VECBASE 128
+
+#define MCF_IRQ_GPIO0 (MCFINTC2_VECBASE + 32)
+#define MCF_IRQ_GPIO1 (MCFINTC2_VECBASE + 33)
+#define MCF_IRQ_GPIO2 (MCFINTC2_VECBASE + 34)
+#define MCF_IRQ_GPIO3 (MCFINTC2_VECBASE + 35)
+#define MCF_IRQ_GPIO4 (MCFINTC2_VECBASE + 36)
+#define MCF_IRQ_GPIO5 (MCFINTC2_VECBASE + 37)
+#define MCF_IRQ_GPIO6 (MCFINTC2_VECBASE + 38)
+
+#define MCF_IRQ_USBWUP (MCFINTC2_VECBASE + 40)
+#define MCF_IRQ_I2C1 (MCFINTC2_VECBASE + 62)
+
+/*
+ * General purpose IO registers (in MBAR2).
+ */
+#define MCFSIM2_GPIOREAD (MCF_MBAR2 + 0x000) /* GPIO read values */
+#define MCFSIM2_GPIOWRITE (MCF_MBAR2 + 0x004) /* GPIO write values */
+#define MCFSIM2_GPIOENABLE (MCF_MBAR2 + 0x008) /* GPIO enabled */
+#define MCFSIM2_GPIOFUNC (MCF_MBAR2 + 0x00C) /* GPIO function */
+#define MCFSIM2_GPIO1READ (MCF_MBAR2 + 0x0B0) /* GPIO1 read values */
+#define MCFSIM2_GPIO1WRITE (MCF_MBAR2 + 0x0B4) /* GPIO1 write values */
+#define MCFSIM2_GPIO1ENABLE (MCF_MBAR2 + 0x0B8) /* GPIO1 enabled */
+#define MCFSIM2_GPIO1FUNC (MCF_MBAR2 + 0x0BC) /* GPIO1 function */
+
+#define MCFSIM2_GPIOINTSTAT (MCF_MBAR2 + 0xc0) /* GPIO intr status */
+#define MCFSIM2_GPIOINTCLEAR (MCF_MBAR2 + 0xc0) /* GPIO intr clear */
+#define MCFSIM2_GPIOINTENABLE (MCF_MBAR2 + 0xc4) /* GPIO intr enable */
+
+/*
+ * Generic GPIO support
+ */
+#define MCFGPIO_PIN_MAX 64
+#define MCFGPIO_IRQ_MAX 7
+#define MCFGPIO_IRQ_VECBASE MCF_IRQ_GPIO0
+
+/****************************************************************************/
+#endif /* m525xsim_h */
diff --git a/arch/m68k/include/asm/m527xsim.h b/arch/m68k/include/asm/m527xsim.h
index 83db8106f50a..71aa5104d3d6 100644
--- a/arch/m68k/include/asm/m527xsim.h
+++ b/arch/m68k/include/asm/m527xsim.h
@@ -60,6 +60,7 @@
#define MCF_IRQ_FECENTC1 (MCFINT2_VECBASE + MCFINT2_FECENTC1)
#define MCF_IRQ_QSPI (MCFINT_VECBASE + MCFINT_QSPI)
+#define MCF_IRQ_PIT1 (MCFINT_VECBASE + MCFINT_PIT1)
/*
* SDRAM configuration registers.
diff --git a/arch/m68k/include/asm/m528xsim.h b/arch/m68k/include/asm/m528xsim.h
index 497c31c803ff..4acb3c0a642e 100644
--- a/arch/m68k/include/asm/m528xsim.h
+++ b/arch/m68k/include/asm/m528xsim.h
@@ -52,7 +52,7 @@
#define MCF_IRQ_FECENTC0 (MCFINT_VECBASE + MCFINT_FECENTC0)
#define MCF_IRQ_QSPI (MCFINT_VECBASE + MCFINT_QSPI)
-
+#define MCF_IRQ_PIT1 (MCFINT_VECBASE + MCFINT_PIT1)
/*
* SDRAM configuration registers.
*/
diff --git a/arch/m68k/include/asm/m532xsim.h b/arch/m68k/include/asm/m532xsim.h
index 29b66e21413a..5ca7b298c6eb 100644
--- a/arch/m68k/include/asm/m532xsim.h
+++ b/arch/m68k/include/asm/m532xsim.h
@@ -82,6 +82,9 @@
#define MCFINTC1_SIMR 0xFC04C01C
#define MCFINTC1_CIMR 0xFC04C01D
#define MCFINTC1_ICR0 0xFC04C040
+#define MCFINTC2_SIMR (0)
+#define MCFINTC2_CIMR (0)
+#define MCFINTC2_ICR0 (0)
#define MCFSIM_ICR_TIMER1 (0xFC048040+32)
#define MCFSIM_ICR_TIMER2 (0xFC048040+33)
@@ -135,6 +138,20 @@
#define MCF_RCR_SWRESET 0x80 /* Software reset bit */
#define MCF_RCR_FRCSTOUT 0x40 /* Force external reset */
+
+/*
+ * Power Management
+ */
+#define MCFPM_WCR 0xfc040013
+#define MCFPM_PPMSR0 0xfc04002c
+#define MCFPM_PPMCR0 0xfc04002d
+#define MCFPM_PPMSR1 0xfc04002e
+#define MCFPM_PPMCR1 0xfc04002f
+#define MCFPM_PPMHR0 0xfc040030
+#define MCFPM_PPMLR0 0xfc040034
+#define MCFPM_PPMHR1 0xfc040038
+#define MCFPM_LPCR 0xec090007
+
/*********************************************************************
*
* Inter-IC (I2C) Module
diff --git a/arch/m68k/include/asm/m5441xsim.h b/arch/m68k/include/asm/m5441xsim.h
new file mode 100644
index 000000000000..cc798ab9524b
--- /dev/null
+++ b/arch/m68k/include/asm/m5441xsim.h
@@ -0,0 +1,276 @@
+/*
+ * m5441xsim.h -- Coldfire 5441x register definitions
+ *
+ * (C) Copyright 2012, Steven King <sfking@fdwdc.com>
+*/
+
+#ifndef m5441xsim_h
+#define m5441xsim_h
+
+#define CPU_NAME "COLDFIRE(m5441x)"
+#define CPU_INSTR_PER_JIFFY 2
+#define MCF_BUSCLK (MCF_CLK / 2)
+
+#include <asm/m54xxacr.h>
+
+/*
+ * Reset Controller Module.
+ */
+
+#define MCF_RCR 0xec090000
+#define MCF_RSR 0xec090001
+
+#define MCF_RCR_SWRESET 0x80 /* Software reset bit */
+#define MCF_RCR_FRCSTOUT 0x40 /* Force external reset */
+
+/*
+ * Interrupt Controller Modules.
+ */
+/* the 5441x have 3 interrupt controllers, each control 64 interrupts */
+#define MCFINT_VECBASE 64
+#define MCFINT0_VECBASE MCFINT_VECBASE
+#define MCFINT1_VECBASE (MCFINT0_VECBASE + 64)
+#define MCFINT2_VECBASE (MCFINT1_VECBASE + 64)
+
+/* interrupt controller 0 */
+#define MCFINTC0_SIMR 0xfc04801c
+#define MCFINTC0_CIMR 0xfc04801d
+#define MCFINTC0_ICR0 0xfc048040
+/* interrupt controller 1 */
+#define MCFINTC1_SIMR 0xfc04c01c
+#define MCFINTC1_CIMR 0xfc04c01d
+#define MCFINTC1_ICR0 0xfc04c040
+/* interrupt controller 2 */
+#define MCFINTC2_SIMR 0xfc05001c
+#define MCFINTC2_CIMR 0xfc05001d
+#define MCFINTC2_ICR0 0xfc050040
+
+/* on interrupt controller 0 */
+#define MCFINT0_EPORT0 1
+#define MCFINT0_UART0 26
+#define MCFINT0_UART1 27
+#define MCFINT0_UART2 28
+#define MCFINT0_UART3 29
+#define MCFINT0_I2C0 30
+#define MCFINT0_DSPI0 31
+
+#define MCFINT0_TIMER0 32
+#define MCFINT0_TIMER1 33
+#define MCFINT0_TIMER2 34
+#define MCFINT0_TIMER3 35
+
+#define MCFINT0_FECRX0 36
+#define MCFINT0_FECTX0 40
+#define MCFINT0_FECENTC0 42
+
+#define MCFINT0_FECRX1 49
+#define MCFINT0_FECTX1 53
+#define MCFINT0_FECENTC1 55
+
+/* on interrupt controller 1 */
+#define MCFINT1_UART4 48
+#define MCFINT1_UART5 49
+#define MCFINT1_UART6 50
+#define MCFINT1_UART7 51
+#define MCFINT1_UART8 52
+#define MCFINT1_UART9 53
+#define MCFINT1_DSPI1 54
+#define MCFINT1_DSPI2 55
+#define MCFINT1_DSPI3 56
+#define MCFINT1_I2C1 57
+#define MCFINT1_I2C2 58
+#define MCFINT1_I2C3 59
+#define MCFINT1_I2C4 60
+#define MCFINT1_I2C5 61
+
+/* on interrupt controller 2 */
+#define MCFINT2_PIT0 13
+#define MCFINT2_PIT1 14
+#define MCFINT2_PIT2 15
+#define MCFINT2_PIT3 16
+#define MCFINT2_RTC 26
+
+/*
+ * PIT timer module.
+ */
+#define MCFPIT_BASE0 0xFC080000 /* Base address of TIMER0 */
+#define MCFPIT_BASE1 0xFC084000 /* Base address of TIMER1 */
+#define MCFPIT_BASE2 0xFC088000 /* Base address of TIMER2 */
+#define MCFPIT_BASE3 0xFC08C000 /* Base address of TIMER3 */
+
+
+#define MCF_IRQ_PIT1 (MCFINT2_VECBASE + MCFINT2_PIT1)
+
+/*
+ * Power Management
+ */
+#define MCFPM_WCR 0xfc040013
+#define MCFPM_PPMSR0 0xfc04002c
+#define MCFPM_PPMCR0 0xfc04002d
+#define MCFPM_PPMSR1 0xfc04002e
+#define MCFPM_PPMCR1 0xfc04002f
+#define MCFPM_PPMHR0 0xfc040030
+#define MCFPM_PPMLR0 0xfc040034
+#define MCFPM_PPMHR1 0xfc040038
+#define MCFPM_PPMLR1 0xfc04003c
+#define MCFPM_LPCR 0xec090007
+/*
+ * UART module.
+ */
+#define MCFUART_BASE0 0xfc060000 /* Base address of UART0 */
+#define MCFUART_BASE1 0xfc064000 /* Base address of UART1 */
+#define MCFUART_BASE2 0xfc068000 /* Base address of UART2 */
+#define MCFUART_BASE3 0xfc06c000 /* Base address of UART3 */
+#define MCFUART_BASE4 0xec060000 /* Base address of UART4 */
+#define MCFUART_BASE5 0xec064000 /* Base address of UART5 */
+#define MCFUART_BASE6 0xec068000 /* Base address of UART6 */
+#define MCFUART_BASE7 0xec06c000 /* Base address of UART7 */
+#define MCFUART_BASE8 0xec070000 /* Base address of UART8 */
+#define MCFUART_BASE9 0xec074000 /* Base address of UART9 */
+
+#define MCF_IRQ_UART0 (MCFINT0_VECBASE + MCFINT0_UART0)
+#define MCF_IRQ_UART1 (MCFINT0_VECBASE + MCFINT0_UART1)
+#define MCF_IRQ_UART2 (MCFINT0_VECBASE + MCFINT0_UART2)
+#define MCF_IRQ_UART3 (MCFINT0_VECBASE + MCFINT0_UART3)
+#define MCF_IRQ_UART4 (MCFINT1_VECBASE + MCFINT1_UART4)
+#define MCF_IRQ_UART5 (MCFINT1_VECBASE + MCFINT1_UART5)
+#define MCF_IRQ_UART6 (MCFINT1_VECBASE + MCFINT1_UART6)
+#define MCF_IRQ_UART7 (MCFINT1_VECBASE + MCFINT1_UART7)
+#define MCF_IRQ_UART8 (MCFINT1_VECBASE + MCFINT1_UART8)
+#define MCF_IRQ_UART9 (MCFINT1_VECBASE + MCFINT1_UART9)
+/*
+ * FEC modules.
+ */
+#define MCFFEC_BASE0 0xfc0d4000
+#define MCFFEC_SIZE0 0x800
+#define MCF_IRQ_FECRX0 (MCFINT0_VECBASE + MCFINT0_FECRX0)
+#define MCF_IRQ_FECTX0 (MCFINT0_VECBASE + MCFINT0_FECTX0)
+#define MCF_IRQ_FECENTC0 (MCFINT0_VECBASE + MCFINT0_FECENTC0)
+
+#define MCFFEC_BASE1 0xfc0d8000
+#define MCFFEC_SIZE1 0x800
+#define MCF_IRQ_FECRX1 (MCFINT0_VECBASE + MCFINT0_FECRX1)
+#define MCF_IRQ_FECTX1 (MCFINT0_VECBASE + MCFINT0_FECTX1)
+#define MCF_IRQ_FECENTC1 (MCFINT0_VECBASE + MCFINT0_FECENTC1)
+/*
+ * I2C modules.
+ */
+#define MCFI2C_BASE0 0xfc058000
+#define MCFI2C_SIZE0 0x20
+#define MCFI2C_BASE1 0xfc038000
+#define MCFI2C_SIZE1 0x20
+#define MCFI2C_BASE2 0xec010000
+#define MCFI2C_SIZE2 0x20
+#define MCFI2C_BASE3 0xec014000
+#define MCFI2C_SIZE3 0x20
+#define MCFI2C_BASE4 0xec018000
+#define MCFI2C_SIZE4 0x20
+#define MCFI2C_BASE5 0xec01c000
+#define MCFI2C_SIZE5 0x20
+
+#define MCF_IRQ_I2C0 (MCFINT0_VECBASE + MCFINT0_I2C0)
+#define MCF_IRQ_I2C1 (MCFINT1_VECBASE + MCFINT1_I2C1)
+#define MCF_IRQ_I2C2 (MCFINT1_VECBASE + MCFINT1_I2C2)
+#define MCF_IRQ_I2C3 (MCFINT1_VECBASE + MCFINT1_I2C3)
+#define MCF_IRQ_I2C4 (MCFINT1_VECBASE + MCFINT1_I2C4)
+#define MCF_IRQ_I2C5 (MCFINT1_VECBASE + MCFINT1_I2C5)
+/*
+ * EPORT Module.
+ */
+#define MCFEPORT_EPPAR 0xfc090000
+#define MCFEPORT_EPIER 0xfc090003
+#define MCFEPORT_EPFR 0xfc090006
+/*
+ * RTC Module.
+ */
+#define MCFRTC_BASE 0xfc0a8000
+#define MCFRTC_SIZE (0xfc0a8840 - 0xfc0a8000)
+#define MCF_IRQ_RTC (MCFINT2_VECBASE + MCFINT2_RTC)
+
+/*
+ * GPIO Module.
+ */
+#define MCFGPIO_PODR_A 0xec094000
+#define MCFGPIO_PODR_B 0xec094001
+#define MCFGPIO_PODR_C 0xec094002
+#define MCFGPIO_PODR_D 0xec094003
+#define MCFGPIO_PODR_E 0xec094004
+#define MCFGPIO_PODR_F 0xec094005
+#define MCFGPIO_PODR_G 0xec094006
+#define MCFGPIO_PODR_H 0xec094007
+#define MCFGPIO_PODR_I 0xec094008
+#define MCFGPIO_PODR_J 0xec094009
+#define MCFGPIO_PODR_K 0xec09400a
+
+#define MCFGPIO_PDDR_A 0xec09400c
+#define MCFGPIO_PDDR_B 0xec09400d
+#define MCFGPIO_PDDR_C 0xec09400e
+#define MCFGPIO_PDDR_D 0xec09400f
+#define MCFGPIO_PDDR_E 0xec094010
+#define MCFGPIO_PDDR_F 0xec094011
+#define MCFGPIO_PDDR_G 0xec094012
+#define MCFGPIO_PDDR_H 0xec094013
+#define MCFGPIO_PDDR_I 0xec094014
+#define MCFGPIO_PDDR_J 0xec094015
+#define MCFGPIO_PDDR_K 0xec094016
+
+#define MCFGPIO_PPDSDR_A 0xec094018
+#define MCFGPIO_PPDSDR_B 0xec094019
+#define MCFGPIO_PPDSDR_C 0xec09401a
+#define MCFGPIO_PPDSDR_D 0xec09401b
+#define MCFGPIO_PPDSDR_E 0xec09401c
+#define MCFGPIO_PPDSDR_F 0xec09401d
+#define MCFGPIO_PPDSDR_G 0xec09401e
+#define MCFGPIO_PPDSDR_H 0xec09401f
+#define MCFGPIO_PPDSDR_I 0xec094020
+#define MCFGPIO_PPDSDR_J 0xec094021
+#define MCFGPIO_PPDSDR_K 0xec094022
+
+#define MCFGPIO_PCLRR_A 0xec094024
+#define MCFGPIO_PCLRR_B 0xec094025
+#define MCFGPIO_PCLRR_C 0xec094026
+#define MCFGPIO_PCLRR_D 0xec094027
+#define MCFGPIO_PCLRR_E 0xec094028
+#define MCFGPIO_PCLRR_F 0xec094029
+#define MCFGPIO_PCLRR_G 0xec09402a
+#define MCFGPIO_PCLRR_H 0xec09402b
+#define MCFGPIO_PCLRR_I 0xec09402c
+#define MCFGPIO_PCLRR_J 0xec09402d
+#define MCFGPIO_PCLRR_K 0xec09402e
+
+#define MCFGPIO_PAR_FBCTL 0xec094048
+#define MCFGPIO_PAR_BE 0xec094049
+#define MCFGPIO_PAR_CS 0xec09404a
+#define MCFGPIO_PAR_CANI2C 0xec09404b
+#define MCFGPIO_PAR_IRQ0H 0xec09404c
+#define MCFGPIO_PAR_IRQ0L 0xec09404d
+#define MCFGPIO_PAR_DSPIOWH 0xec09404e
+#define MCFGPIO_PAR_DSPIOWL 0xec09404f
+#define MCFGPIO_PAR_TIMER 0xec094050
+#define MCFGPIO_PAR_UART2 0xec094051
+#define MCFGPIO_PAR_UART1 0xec094052
+#define MCFGPIO_PAR_UART0 0xec094053
+#define MCFGPIO_PAR_SDHCH 0xec094054
+#define MCFGPIO_PAR_SDHCL 0xec094055
+#define MCFGPIO_PAR_SIMP0H 0xec094056
+#define MCFGPIO_PAR_SIMP0L 0xec094057
+#define MCFGPIO_PAR_SSI0H 0xec094058
+#define MCFGPIO_PAR_SSI0L 0xec094059
+#define MCFGPIO_PAR_DEBUGH1 0xec09405a
+#define MCFGPIO_PAR_DEBUGH0 0xec09405b
+#define MCFGPIO_PAR_DEBUGl 0xec09405c
+#define MCFGPIO_PAR_FEC 0xec09405e
+
+/* generalization for generic gpio support */
+#define MCFGPIO_PODR MCFGPIO_PODR_A
+#define MCFGPIO_PDDR MCFGPIO_PDDR_A
+#define MCFGPIO_PPDR MCFGPIO_PPDSDR_A
+#define MCFGPIO_SETR MCFGPIO_PPDSDR_A
+#define MCFGPIO_CLRR MCFGPIO_PCLRR_A
+
+#define MCFGPIO_IRQ_MIN 17
+#define MCFGPIO_IRQ_MAX 24
+#define MCFGPIO_IRQ_VECBASE (MCFINT_VECBASE - MCFGPIO_IRQ_MIN)
+#define MCFGPIO_PIN_MAX 87
+
+#endif /* m5441xsim_h */
diff --git a/arch/m68k/include/asm/m54xxacr.h b/arch/m68k/include/asm/m54xxacr.h
index 47906aafbf67..192bbfeabf70 100644
--- a/arch/m68k/include/asm/m54xxacr.h
+++ b/arch/m68k/include/asm/m54xxacr.h
@@ -55,6 +55,10 @@
#define ICACHE_SIZE 0x8000 /* instruction - 32k */
#define DCACHE_SIZE 0x8000 /* data - 32k */
+#elif defined(CONFIG_M5441x)
+
+#define ICACHE_SIZE 0x2000 /* instruction - 8k */
+#define DCACHE_SIZE 0x2000 /* data - 8k */
#endif
#define CACHE_LINE_SIZE 0x0010 /* 16 bytes */
diff --git a/arch/m68k/include/asm/m54xxpci.h b/arch/m68k/include/asm/m54xxpci.h
new file mode 100644
index 000000000000..6fbf54f72f2e
--- /dev/null
+++ b/arch/m68k/include/asm/m54xxpci.h
@@ -0,0 +1,138 @@
+/****************************************************************************/
+
+/*
+ * m54xxpci.h -- ColdFire 547x and 548x PCI bus support
+ *
+ * (C) Copyright 2011, Greg Ungerer <gerg@uclinux.org>
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file COPYING in the main directory of this archive
+ * for more details.
+ */
+
+/****************************************************************************/
+#ifndef M54XXPCI_H
+#define M54XXPCI_H
+/****************************************************************************/
+
+/*
+ * The core set of PCI support registers are mapped into the MBAR region.
+ */
+#define PCIIDR (CONFIG_MBAR + 0xb00) /* PCI device/vendor ID */
+#define PCISCR (CONFIG_MBAR + 0xb04) /* PCI status/command */
+#define PCICCRIR (CONFIG_MBAR + 0xb08) /* PCI class/revision */
+#define PCICR1 (CONFIG_MBAR + 0xb0c) /* PCI configuration 1 */
+#define PCIBAR0 (CONFIG_MBAR + 0xb10) /* PCI base address 0 */
+#define PCIBAR1 (CONFIG_MBAR + 0xb14) /* PCI base address 1 */
+#define PCICCPR (CONFIG_MBAR + 0xb28) /* PCI cardbus CIS pointer */
+#define PCISID (CONFIG_MBAR + 0xb2c) /* PCI subsystem IDs */
+#define PCIERBAR (CONFIG_MBAR + 0xb30) /* PCI expansion ROM */
+#define PCICPR (CONFIG_MBAR + 0xb34) /* PCI capabilities pointer */
+#define PCICR2 (CONFIG_MBAR + 0xb3c) /* PCI configuration 2 */
+
+#define PCIGSCR (CONFIG_MBAR + 0xb60) /* Global status/control */
+#define PCITBATR0 (CONFIG_MBAR + 0xb64) /* Target base translation 0 */
+#define PCITBATR1 (CONFIG_MBAR + 0xb68) /* Target base translation 1 */
+#define PCITCR (CONFIG_MBAR + 0xb6c) /* Target control */
+#define PCIIW0BTAR (CONFIG_MBAR + 0xb70) /* Initiator window 0 */
+#define PCIIW1BTAR (CONFIG_MBAR + 0xb74) /* Initiator window 1 */
+#define PCIIW2BTAR (CONFIG_MBAR + 0xb78) /* Initiator window 2 */
+#define PCIIWCR (CONFIG_MBAR + 0xb80) /* Initiator window config */
+#define PCIICR (CONFIG_MBAR + 0xb84) /* Initiator control */
+#define PCIISR (CONFIG_MBAR + 0xb88) /* Initiator status */
+#define PCICAR (CONFIG_MBAR + 0xbf8) /* Configuration address */
+
+#define PCITPSR (CONFIG_MBAR + 0x8400) /* TX packet size */
+#define PCITSAR (CONFIG_MBAR + 0x8404) /* TX start address */
+#define PCITTCR (CONFIG_MBAR + 0x8408) /* TX transaction control */
+#define PCITER (CONFIG_MBAR + 0x840c) /* TX enables */
+#define PCITNAR (CONFIG_MBAR + 0x8410) /* TX next address */
+#define PCITLWR (CONFIG_MBAR + 0x8414) /* TX last word */
+#define PCITDCR (CONFIG_MBAR + 0x8418) /* TX done counts */
+#define PCITSR (CONFIG_MBAR + 0x841c) /* TX status */
+#define PCITFDR (CONFIG_MBAR + 0x8440) /* TX FIFO data */
+#define PCITFSR (CONFIG_MBAR + 0x8444) /* TX FIFO status */
+#define PCITFCR (CONFIG_MBAR + 0x8448) /* TX FIFO control */
+#define PCITFAR (CONFIG_MBAR + 0x844c) /* TX FIFO alarm */
+#define PCITFRPR (CONFIG_MBAR + 0x8450) /* TX FIFO read pointer */
+#define PCITFWPR (CONFIG_MBAR + 0x8454) /* TX FIFO write pointer */
+
+#define PCIRPSR (CONFIG_MBAR + 0x8480) /* RX packet size */
+#define PCIRSAR (CONFIG_MBAR + 0x8484) /* RX start address */
+#define PCIRTCR (CONFIG_MBAR + 0x8488) /* RX transaction control */
+#define PCIRER (CONFIG_MBAR + 0x848c) /* RX enables */
+#define PCIRNAR (CONFIG_MBAR + 0x8490) /* RX next address */
+#define PCIRDCR (CONFIG_MBAR + 0x8498) /* RX done counts */
+#define PCIRSR (CONFIG_MBAR + 0x849c) /* RX status */
+#define PCIRFDR (CONFIG_MBAR + 0x84c0) /* RX FIFO data */
+#define PCIRFSR (CONFIG_MBAR + 0x84c4) /* RX FIFO status */
+#define PCIRFCR (CONFIG_MBAR + 0x84c8) /* RX FIFO control */
+#define PCIRFAR (CONFIG_MBAR + 0x84cc) /* RX FIFO alarm */
+#define PCIRFRPR (CONFIG_MBAR + 0x84d0) /* RX FIFO read pointer */
+#define PCIRFWPR (CONFIG_MBAR + 0x84d4) /* RX FIFO write pointer */
+
+#define PACR (CONFIG_MBAR + 0xc00) /* PCI arbiter control */
+#define PASR (COFNIG_MBAR + 0xc04) /* PCI arbiter status */
+
+/*
+ * Definitions for the Global status and control register.
+ */
+#define PCIGSCR_PE 0x20000000 /* Parity error detected */
+#define PCIGSCR_SE 0x10000000 /* System error detected */
+#define PCIGSCR_XCLKBIN 0x07000000 /* XLB2CLKIN mask */
+#define PCIGSCR_PEE 0x00002000 /* Parity error intr enable */
+#define PCIGSCR_SEE 0x00001000 /* System error intr enable */
+#define PCIGSCR_RESET 0x00000001 /* Reset bit */
+
+/*
+ * Bit definitions for the PCICAR configuration address register.
+ */
+#define PCICAR_E 0x80000000 /* Enable config space */
+#define PCICAR_BUSN 16 /* Move bus bits */
+#define PCICAR_DEVFNN 8 /* Move devfn bits */
+#define PCICAR_DWORDN 0 /* Move dword bits */
+
+/*
+ * The initiator windows hold the memory and IO mapping information.
+ * This macro creates the register values from the desired addresses.
+ */
+#define WXBTAR(hostaddr, pciaddr, size) \
+ (((hostaddr) & 0xff000000) | \
+ ((((size) - 1) & 0xff000000) >> 8) | \
+ (((pciaddr) & 0xff000000) >> 16))
+
+#define PCIIWCR_W0_MEM 0x00000000 /* Window 0 is memory */
+#define PCIIWCR_W0_IO 0x08000000 /* Window 0 is IO */
+#define PCIIWCR_W0_MRD 0x00000000 /* Window 0 memory read */
+#define PCIIWCR_W0_MRDL 0x02000000 /* Window 0 memory read line */
+#define PCIIWCR_W0_MRDM 0x04000000 /* Window 0 memory read mult */
+#define PCIIWCR_W0_E 0x01000000 /* Window 0 enable */
+
+#define PCIIWCR_W1_MEM 0x00000000 /* Window 0 is memory */
+#define PCIIWCR_W1_IO 0x00080000 /* Window 0 is IO */
+#define PCIIWCR_W1_MRD 0x00000000 /* Window 0 memory read */
+#define PCIIWCR_W1_MRDL 0x00020000 /* Window 0 memory read line */
+#define PCIIWCR_W1_MRDM 0x00040000 /* Window 0 memory read mult */
+#define PCIIWCR_W1_E 0x00010000 /* Window 0 enable */
+
+/*
+ * Bit definitions for the PCIBATR registers.
+ */
+#define PCITBATR0_E 0x00000001 /* Enable window 0 */
+#define PCITBATR1_E 0x00000001 /* Enable window 1 */
+
+/*
+ * PCI arbiter support definitions and macros.
+ */
+#define PACR_INTMPRI 0x00000001
+#define PACR_EXTMPRI(x) (((x) & 0x1f) << 1)
+#define PACR_INTMINTE 0x00010000
+#define PACR_EXTMINTE(x) (((x) & 0x1f) << 17)
+#define PACR_PKMD 0x40000000
+#define PACR_DS 0x80000000
+
+#define PCICR1_CL(x) ((x) & 0xf) /* Cacheline size field */
+#define PCICR1_LT(x) (((x) & 0xff) << 8) /* Latency timer field */
+
+/****************************************************************************/
+#endif /* M54XXPCI_H */
diff --git a/arch/m68k/include/asm/m54xxsim.h b/arch/m68k/include/asm/m54xxsim.h
index ae56b8848a9d..d3c5e0dbdadf 100644
--- a/arch/m68k/include/asm/m54xxsim.h
+++ b/arch/m68k/include/asm/m54xxsim.h
@@ -81,4 +81,7 @@
#define MCF_PAR_PSC_RTS_RTS (0x30)
#define MCF_PAR_PSC_CANRX (0x40)
+#define MCF_PAR_PCIBG (CONFIG_MBAR + 0xa48) /* PCI bus grant */
+#define MCF_PAR_PCIBR (CONFIG_MBAR + 0xa4a) /* PCI */
+
#endif /* m54xxsim_h */
diff --git a/arch/m68k/include/asm/mcfclk.h b/arch/m68k/include/asm/mcfclk.h
new file mode 100644
index 000000000000..b676a02bb392
--- /dev/null
+++ b/arch/m68k/include/asm/mcfclk.h
@@ -0,0 +1,43 @@
+/*
+ * mcfclk.h -- coldfire specific clock structure
+ */
+
+
+#ifndef mcfclk_h
+#define mcfclk_h
+
+struct clk;
+
+#ifdef MCFPM_PPMCR0
+struct clk_ops {
+ void (*enable)(struct clk *);
+ void (*disable)(struct clk *);
+};
+
+struct clk {
+ const char *name;
+ struct clk_ops *clk_ops;
+ unsigned long rate;
+ unsigned long enabled;
+ u8 slot;
+};
+
+extern struct clk *mcf_clks[];
+extern struct clk_ops clk_ops0;
+#ifdef MCFPM_PPMCR1
+extern struct clk_ops clk_ops1;
+#endif /* MCFPM_PPMCR1 */
+
+#define DEFINE_CLK(clk_bank, clk_name, clk_slot, clk_rate) \
+static struct clk __clk_##clk_bank##_##clk_slot = { \
+ .name = clk_name, \
+ .clk_ops = &clk_ops##clk_bank, \
+ .rate = clk_rate, \
+ .slot = clk_slot, \
+}
+
+void __clk_init_enabled(struct clk *);
+void __clk_init_disabled(struct clk *);
+#endif /* MCFPM_PPMCR0 */
+
+#endif /* mcfclk_h */
diff --git a/arch/m68k/include/asm/mcfgpio.h b/arch/m68k/include/asm/mcfgpio.h
index fe468eaa51e0..fa1059f50dfc 100644
--- a/arch/m68k/include/asm/mcfgpio.h
+++ b/arch/m68k/include/asm/mcfgpio.h
@@ -16,82 +16,289 @@
#ifndef mcfgpio_h
#define mcfgpio_h
-#include <linux/io.h>
+#ifdef CONFIG_GPIOLIB
#include <asm-generic/gpio.h>
+#else
+
+int __mcfgpio_get_value(unsigned gpio);
+void __mcfgpio_set_value(unsigned gpio, int value);
+int __mcfgpio_direction_input(unsigned gpio);
+int __mcfgpio_direction_output(unsigned gpio, int value);
+int __mcfgpio_request(unsigned gpio);
+void __mcfgpio_free(unsigned gpio);
+
+/* our alternate 'gpiolib' functions */
+static inline int __gpio_get_value(unsigned gpio)
+{
+ if (gpio < MCFGPIO_PIN_MAX)
+ return __mcfgpio_get_value(gpio);
+ else
+ return -EINVAL;
+}
+
+static inline void __gpio_set_value(unsigned gpio, int value)
+{
+ if (gpio < MCFGPIO_PIN_MAX)
+ __mcfgpio_set_value(gpio, value);
+}
+
+static inline int __gpio_cansleep(unsigned gpio)
+{
+ if (gpio < MCFGPIO_PIN_MAX)
+ return 0;
+ else
+ return -EINVAL;
+}
+
+static inline int __gpio_to_irq(unsigned gpio)
+{
+ return -EINVAL;
+}
+
+static inline int gpio_direction_input(unsigned gpio)
+{
+ if (gpio < MCFGPIO_PIN_MAX)
+ return __mcfgpio_direction_input(gpio);
+ else
+ return -EINVAL;
+}
+
+static inline int gpio_direction_output(unsigned gpio, int value)
+{
+ if (gpio < MCFGPIO_PIN_MAX)
+ return __mcfgpio_direction_output(gpio, value);
+ else
+ return -EINVAL;
+}
+
+static inline int gpio_request(unsigned gpio, const char *label)
+{
+ if (gpio < MCFGPIO_PIN_MAX)
+ return __mcfgpio_request(gpio);
+ else
+ return -EINVAL;
+}
+
+static inline void gpio_free(unsigned gpio)
+{
+ if (gpio < MCFGPIO_PIN_MAX)
+ __mcfgpio_free(gpio);
+}
+
+#endif /* CONFIG_GPIOLIB */
-struct mcf_gpio_chip {
- struct gpio_chip gpio_chip;
- void __iomem *pddr;
- void __iomem *podr;
- void __iomem *ppdr;
- void __iomem *setr;
- void __iomem *clrr;
- const u8 *gpio_to_pinmux;
-};
-
-extern struct mcf_gpio_chip mcf_gpio_chips[];
-extern unsigned int mcf_gpio_chips_size;
-
-int mcf_gpio_direction_input(struct gpio_chip *, unsigned);
-int mcf_gpio_get_value(struct gpio_chip *, unsigned);
-int mcf_gpio_direction_output(struct gpio_chip *, unsigned, int);
-void mcf_gpio_set_value(struct gpio_chip *, unsigned, int);
-void mcf_gpio_set_value_fast(struct gpio_chip *, unsigned, int);
-int mcf_gpio_request(struct gpio_chip *, unsigned);
-void mcf_gpio_free(struct gpio_chip *, unsigned);
/*
- * Define macros to ease the pain of setting up the GPIO tables. There
- * are two cases we need to deal with here, they cover all currently
- * available ColdFire GPIO hardware. There are of course minor differences
- * in the layout and number of bits in each ColdFire part, but the macros
- * take all that in.
+ * The Freescale Coldfire family is quite varied in how they implement GPIO.
+ * Some parts have 8 bit ports, some have 16bit and some have 32bit; some have
+ * only one port, others have multiple ports; some have a single data latch
+ * for both input and output, others have a separate pin data register to read
+ * input; some require a read-modify-write access to change an output, others
+ * have set and clear registers for some of the outputs; Some have all the
+ * GPIOs in a single control area, others have some GPIOs implemented in
+ * different modules.
*
- * Firstly is the conventional GPIO registers where we toggle individual
- * bits in a register, preserving the other bits in the register. For
- * lack of a better term I have called this the slow method.
+ * This implementation attempts accommodate the differences while presenting
+ * a generic interface that will optimize to as few instructions as possible.
+ */
+#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \
+ defined(CONFIG_M520x) || defined(CONFIG_M523x) || \
+ defined(CONFIG_M527x) || defined(CONFIG_M528x) || \
+ defined(CONFIG_M532x) || defined(CONFIG_M54xx) || \
+ defined(CONFIG_M5441x)
+
+/* These parts have GPIO organized by 8 bit ports */
+
+#define MCFGPIO_PORTTYPE u8
+#define MCFGPIO_PORTSIZE 8
+#define mcfgpio_read(port) __raw_readb(port)
+#define mcfgpio_write(data, port) __raw_writeb(data, port)
+
+#elif defined(CONFIG_M5307) || defined(CONFIG_M5407) || defined(CONFIG_M5272)
+
+/* These parts have GPIO organized by 16 bit ports */
+
+#define MCFGPIO_PORTTYPE u16
+#define MCFGPIO_PORTSIZE 16
+#define mcfgpio_read(port) __raw_readw(port)
+#define mcfgpio_write(data, port) __raw_writew(data, port)
+
+#elif defined(CONFIG_M5249) || defined(CONFIG_M525x)
+
+/* These parts have GPIO organized by 32 bit ports */
+
+#define MCFGPIO_PORTTYPE u32
+#define MCFGPIO_PORTSIZE 32
+#define mcfgpio_read(port) __raw_readl(port)
+#define mcfgpio_write(data, port) __raw_writel(data, port)
+
+#endif
+
+#define mcfgpio_bit(gpio) (1 << ((gpio) % MCFGPIO_PORTSIZE))
+#define mcfgpio_port(gpio) ((gpio) / MCFGPIO_PORTSIZE)
+
+#if defined(CONFIG_M520x) || defined(CONFIG_M523x) || \
+ defined(CONFIG_M527x) || defined(CONFIG_M528x) || \
+ defined(CONFIG_M532x) || defined(CONFIG_M5441x)
+/*
+ * These parts have an 'Edge' Port module (external interrupt/GPIO) which uses
+ * read-modify-write to change an output and a GPIO module which has separate
+ * set/clr registers to directly change outputs with a single write access.
+ */
+#if defined(CONFIG_M528x)
+/*
+ * The 528x also has GPIOs in other modules (GPT, QADC) which use
+ * read-modify-write as well as those controlled by the EPORT and GPIO modules.
*/
-#define MCFGPS(mlabel, mbase, mngpio, mpddr, mpodr, mppdr) \
- { \
- .gpio_chip = { \
- .label = #mlabel, \
- .request = mcf_gpio_request, \
- .free = mcf_gpio_free, \
- .direction_input = mcf_gpio_direction_input, \
- .direction_output = mcf_gpio_direction_output,\
- .get = mcf_gpio_get_value, \
- .set = mcf_gpio_set_value, \
- .base = mbase, \
- .ngpio = mngpio, \
- }, \
- .pddr = (void __iomem *) mpddr, \
- .podr = (void __iomem *) mpodr, \
- .ppdr = (void __iomem *) mppdr, \
- }
+#define MCFGPIO_SCR_START 40
+#elif defined(CONFIGM5441x)
+/* The m5441x EPORT doesn't have its own GPIO port, uses PORT C */
+#define MCFGPIO_SCR_START 0
+#else
+#define MCFGPIO_SCR_START 8
+#endif
+#define MCFGPIO_SETR_PORT(gpio) (MCFGPIO_SETR + \
+ mcfgpio_port(gpio - MCFGPIO_SCR_START))
+
+#define MCFGPIO_CLRR_PORT(gpio) (MCFGPIO_CLRR + \
+ mcfgpio_port(gpio - MCFGPIO_SCR_START))
+#else
+
+#define MCFGPIO_SCR_START MCFGPIO_PIN_MAX
+/* with MCFGPIO_SCR == MCFGPIO_PIN_MAX, these will be optimized away */
+#define MCFGPIO_SETR_PORT(gpio) 0
+#define MCFGPIO_CLRR_PORT(gpio) 0
+
+#endif
/*
- * Secondly is the faster case, where we have set and clear registers
- * that allow us to set or clear a bit with a single write, not having
- * to worry about preserving other bits.
+ * Coldfire specific helper functions
*/
-#define MCFGPF(mlabel, mbase, mngpio) \
- { \
- .gpio_chip = { \
- .label = #mlabel, \
- .request = mcf_gpio_request, \
- .free = mcf_gpio_free, \
- .direction_input = mcf_gpio_direction_input, \
- .direction_output = mcf_gpio_direction_output,\
- .get = mcf_gpio_get_value, \
- .set = mcf_gpio_set_value_fast, \
- .base = mbase, \
- .ngpio = mngpio, \
- }, \
- .pddr = (void __iomem *) MCFGPIO_PDDR_##mlabel, \
- .podr = (void __iomem *) MCFGPIO_PODR_##mlabel, \
- .ppdr = (void __iomem *) MCFGPIO_PPDSDR_##mlabel, \
- .setr = (void __iomem *) MCFGPIO_PPDSDR_##mlabel, \
- .clrr = (void __iomem *) MCFGPIO_PCLRR_##mlabel, \
- }
+/* return the port pin data register for a gpio */
+static inline u32 __mcfgpio_ppdr(unsigned gpio)
+{
+#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \
+ defined(CONFIG_M5307) || defined(CONFIG_M5407)
+ return MCFSIM_PADAT;
+#elif defined(CONFIG_M5272)
+ if (gpio < 16)
+ return MCFSIM_PADAT;
+ else if (gpio < 32)
+ return MCFSIM_PBDAT;
+ else
+ return MCFSIM_PCDAT;
+#elif defined(CONFIG_M5249) || defined(CONFIG_M525x)
+ if (gpio < 32)
+ return MCFSIM2_GPIOREAD;
+ else
+ return MCFSIM2_GPIO1READ;
+#elif defined(CONFIG_M520x) || defined(CONFIG_M523x) || \
+ defined(CONFIG_M527x) || defined(CONFIG_M528x) || \
+ defined(CONFIG_M532x) || defined(CONFIG_M5441x)
+#if !defined(CONFIG_M5441x)
+ if (gpio < 8)
+ return MCFEPORT_EPPDR;
+#if defined(CONFIG_M528x)
+ else if (gpio < 16)
+ return MCFGPTA_GPTPORT;
+ else if (gpio < 24)
+ return MCFGPTB_GPTPORT;
+ else if (gpio < 32)
+ return MCFQADC_PORTQA;
+ else if (gpio < 40)
+ return MCFQADC_PORTQB;
+#endif /* defined(CONFIG_M528x) */
+ else
+#endif /* !defined(CONFIG_M5441x) */
+ return MCFGPIO_PPDR + mcfgpio_port(gpio - MCFGPIO_SCR_START);
+#else
+ return 0;
#endif
+}
+
+/* return the port output data register for a gpio */
+static inline u32 __mcfgpio_podr(unsigned gpio)
+{
+#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \
+ defined(CONFIG_M5307) || defined(CONFIG_M5407)
+ return MCFSIM_PADAT;
+#elif defined(CONFIG_M5272)
+ if (gpio < 16)
+ return MCFSIM_PADAT;
+ else if (gpio < 32)
+ return MCFSIM_PBDAT;
+ else
+ return MCFSIM_PCDAT;
+#elif defined(CONFIG_M5249) || defined(CONFIG_M525x)
+ if (gpio < 32)
+ return MCFSIM2_GPIOWRITE;
+ else
+ return MCFSIM2_GPIO1WRITE;
+#elif defined(CONFIG_M520x) || defined(CONFIG_M523x) || \
+ defined(CONFIG_M527x) || defined(CONFIG_M528x) || \
+ defined(CONFIG_M532x) || defined(CONFIG_M5441x)
+#if !defined(CONFIG_M5441x)
+ if (gpio < 8)
+ return MCFEPORT_EPDR;
+#if defined(CONFIG_M528x)
+ else if (gpio < 16)
+ return MCFGPTA_GPTPORT;
+ else if (gpio < 24)
+ return MCFGPTB_GPTPORT;
+ else if (gpio < 32)
+ return MCFQADC_PORTQA;
+ else if (gpio < 40)
+ return MCFQADC_PORTQB;
+#endif /* defined(CONFIG_M528x) */
+ else
+#endif /* !defined(CONFIG_M5441x) */
+ return MCFGPIO_PODR + mcfgpio_port(gpio - MCFGPIO_SCR_START);
+#else
+ return 0;
+#endif
+}
+
+/* return the port direction data register for a gpio */
+static inline u32 __mcfgpio_pddr(unsigned gpio)
+{
+#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \
+ defined(CONFIG_M5307) || defined(CONFIG_M5407)
+ return MCFSIM_PADDR;
+#elif defined(CONFIG_M5272)
+ if (gpio < 16)
+ return MCFSIM_PADDR;
+ else if (gpio < 32)
+ return MCFSIM_PBDDR;
+ else
+ return MCFSIM_PCDDR;
+#elif defined(CONFIG_M5249) || defined(CONFIG_M525x)
+ if (gpio < 32)
+ return MCFSIM2_GPIOENABLE;
+ else
+ return MCFSIM2_GPIO1ENABLE;
+#elif defined(CONFIG_M520x) || defined(CONFIG_M523x) || \
+ defined(CONFIG_M527x) || defined(CONFIG_M528x) || \
+ defined(CONFIG_M532x) || defined(CONFIG_M5441x)
+#if !defined(CONFIG_M5441x)
+ if (gpio < 8)
+ return MCFEPORT_EPDDR;
+#if defined(CONFIG_M528x)
+ else if (gpio < 16)
+ return MCFGPTA_GPTDDR;
+ else if (gpio < 24)
+ return MCFGPTB_GPTDDR;
+ else if (gpio < 32)
+ return MCFQADC_DDRQA;
+ else if (gpio < 40)
+ return MCFQADC_DDRQB;
+#endif /* defined(CONFIG_M528x) */
+ else
+#endif /* !defined(CONFIG_M5441x) */
+ return MCFGPIO_PDDR + mcfgpio_port(gpio - MCFGPIO_SCR_START);
+#else
+ return 0;
+#endif
+}
+
+#endif /* mcfgpio_h */
diff --git a/arch/m68k/include/asm/mcfsim.h b/arch/m68k/include/asm/mcfsim.h
index ebd0304054ad..7a83e619e73b 100644
--- a/arch/m68k/include/asm/mcfsim.h
+++ b/arch/m68k/include/asm/mcfsim.h
@@ -27,6 +27,9 @@
#elif defined(CONFIG_M5249)
#include <asm/m5249sim.h>
#include <asm/mcfintc.h>
+#elif defined(CONFIG_M525x)
+#include <asm/m525xsim.h>
+#include <asm/mcfintc.h>
#elif defined(CONFIG_M527x)
#include <asm/m527xsim.h>
#elif defined(CONFIG_M5272)
@@ -43,6 +46,8 @@
#include <asm/mcfintc.h>
#elif defined(CONFIG_M54xx)
#include <asm/m54xxsim.h>
+#elif defined(CONFIG_M5441x)
+#include <asm/m5441xsim.h>
#endif
/****************************************************************************/
diff --git a/arch/m68k/include/asm/mcftimer.h b/arch/m68k/include/asm/mcftimer.h
index 351c27237874..da2fa43c2e45 100644
--- a/arch/m68k/include/asm/mcftimer.h
+++ b/arch/m68k/include/asm/mcftimer.h
@@ -19,7 +19,7 @@
#define MCFTIMER_TRR 0x04 /* Timer Reference (r/w) */
#define MCFTIMER_TCR 0x08 /* Timer Capture reg (r/w) */
#define MCFTIMER_TCN 0x0C /* Timer Counter reg (r/w) */
-#if defined(CONFIG_M532x)
+#if defined(CONFIG_M532x) || defined(CONFIG_M5441x)
#define MCFTIMER_TER 0x03 /* Timer Event reg (r/w) */
#else
#define MCFTIMER_TER 0x11 /* Timer Event reg (r/w) */
diff --git a/arch/m68k/include/asm/mcfuart.h b/arch/m68k/include/asm/mcfuart.h
index 2d3bc774b3c5..b40c20f66647 100644
--- a/arch/m68k/include/asm/mcfuart.h
+++ b/arch/m68k/include/asm/mcfuart.h
@@ -43,8 +43,8 @@ struct mcf_platform_uart {
#define MCFUART_UFPD 0x30 /* Frac Prec. Divider (r/w) */
#endif
#if defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \
- defined(CONFIG_M5249) || defined(CONFIG_M5307) || \
- defined(CONFIG_M5407)
+ defined(CONFIG_M5249) || defined(CONFIG_M525x) || \
+ defined(CONFIG_M5307) || defined(CONFIG_M5407)
#define MCFUART_UIVR 0x30 /* Interrupt Vector (r/w) */
#endif
#define MCFUART_UIPR 0x34 /* Input Port (r) */
diff --git a/arch/m68k/include/asm/pci.h b/arch/m68k/include/asm/pci.h
index 4ad0aea48ab4..848c3dfaad50 100644
--- a/arch/m68k/include/asm/pci.h
+++ b/arch/m68k/include/asm/pci.h
@@ -2,6 +2,7 @@
#define _ASM_M68K_PCI_H
#include <asm-generic/pci-dma-compat.h>
+#include <asm-generic/pci.h>
/* The PCI address space does equal the physical memory
* address space. The networking and block device layers use
@@ -9,4 +10,9 @@
*/
#define PCI_DMA_BUS_IS_PHYS (1)
+#define pcibios_assign_all_busses() 1
+
+#define PCIBIOS_MIN_IO 0x00000100
+#define PCIBIOS_MIN_MEM 0x02000000
+
#endif /* _ASM_M68K_PCI_H */
diff --git a/arch/m68k/include/asm/pinmux.h b/arch/m68k/include/asm/pinmux.h
deleted file mode 100644
index 119ee686dbd1..000000000000
--- a/arch/m68k/include/asm/pinmux.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Coldfire generic GPIO pinmux support.
- *
- * (C) Copyright 2009, Steven King <sfking@fdwdc.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; version 2 of the License.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
-
-#ifndef pinmux_h
-#define pinmux_h
-
-#define MCFPINMUX_NONE -1
-
-extern int mcf_pinmux_request(unsigned, unsigned);
-extern void mcf_pinmux_release(unsigned, unsigned);
-
-static inline int mcf_pinmux_is_valid(unsigned pinmux)
-{
- return pinmux != MCFPINMUX_NONE;
-}
-
-#endif
-
diff --git a/arch/m68k/kernel/Makefile b/arch/m68k/kernel/Makefile
index 5c7070e21eb7..068ad49210d6 100644
--- a/arch/m68k/kernel/Makefile
+++ b/arch/m68k/kernel/Makefile
@@ -18,6 +18,7 @@ obj-y += setup.o signal.o sys_m68k.o syscalltable.o time.o traps.o
obj-$(CONFIG_MMU_MOTOROLA) += ints.o vectors.o
obj-$(CONFIG_MMU_SUN3) += ints.o vectors.o
+obj-$(CONFIG_PCI) += pcibios.o
ifndef CONFIG_MMU_SUN3
obj-y += dma.o
diff --git a/arch/m68k/kernel/dma.c b/arch/m68k/kernel/dma.c
index f6daf6e15d2e..e546a5534dd4 100644
--- a/arch/m68k/kernel/dma.c
+++ b/arch/m68k/kernel/dma.c
@@ -16,7 +16,7 @@
#include <asm/pgalloc.h>
-#ifdef CONFIG_MMU
+#if defined(CONFIG_MMU) && !defined(CONFIG_COLDFIRE)
void *dma_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *handle, gfp_t flag)
@@ -96,7 +96,7 @@ void dma_free_coherent(struct device *dev, size_t size,
free_pages((unsigned long)vaddr, get_order(size));
}
-#endif /* CONFIG_MMU */
+#endif /* CONFIG_MMU && !CONFIG_COLDFIRE */
EXPORT_SYMBOL(dma_alloc_coherent);
EXPORT_SYMBOL(dma_free_coherent);
@@ -105,6 +105,7 @@ void dma_sync_single_for_device(struct device *dev, dma_addr_t handle,
size_t size, enum dma_data_direction dir)
{
switch (dir) {
+ case DMA_BIDIRECTIONAL:
case DMA_TO_DEVICE:
cache_push(handle, size);
break;
diff --git a/arch/m68k/kernel/entry.S b/arch/m68k/kernel/entry.S
index b8daf64e347d..165ee9f9d5c9 100644
--- a/arch/m68k/kernel/entry.S
+++ b/arch/m68k/kernel/entry.S
@@ -1,5 +1,451 @@
-#if defined(CONFIG_MMU) && !defined(CONFIG_COLDFIRE)
-#include "entry_mm.S"
+/* -*- mode: asm -*-
+ *
+ * linux/arch/m68k/kernel/entry.S
+ *
+ * Copyright (C) 1991, 1992 Linus Torvalds
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file README.legal in the main directory of this archive
+ * for more details.
+ *
+ * Linux/m68k support by Hamish Macdonald
+ *
+ * 68060 fixes by Jesper Skov
+ *
+ */
+
+/*
+ * entry.S contains the system-call and fault low-level handling routines.
+ * This also contains the timer-interrupt handler, as well as all interrupts
+ * and faults that can result in a task-switch.
+ *
+ * NOTE: This code handles signal-recognition, which happens every time
+ * after a timer-interrupt and after each system call.
+ *
+ */
+
+/*
+ * 12/03/96 Jes: Currently we only support m68k single-cpu systems, so
+ * all pointers that used to be 'current' are now entry
+ * number 0 in the 'current_set' list.
+ *
+ * 6/05/00 RZ: addedd writeback completion after return from sighandler
+ * for 68040
+ */
+
+#include <linux/linkage.h>
+#include <asm/errno.h>
+#include <asm/setup.h>
+#include <asm/segment.h>
+#include <asm/traps.h>
+#include <asm/unistd.h>
+#include <asm/asm-offsets.h>
+#include <asm/entry.h>
+
+.globl system_call, buserr, trap, resume
+.globl sys_call_table
+.globl sys_fork, sys_clone, sys_vfork
+.globl ret_from_interrupt, bad_interrupt
+.globl auto_irqhandler_fixup
+.globl user_irqvec_fixup
+
+.text
+ENTRY(sys_fork)
+ SAVE_SWITCH_STACK
+ pea %sp@(SWITCH_STACK_SIZE)
+ jbsr m68k_fork
+ addql #4,%sp
+ RESTORE_SWITCH_STACK
+ rts
+
+ENTRY(sys_clone)
+ SAVE_SWITCH_STACK
+ pea %sp@(SWITCH_STACK_SIZE)
+ jbsr m68k_clone
+ addql #4,%sp
+ RESTORE_SWITCH_STACK
+ rts
+
+ENTRY(sys_vfork)
+ SAVE_SWITCH_STACK
+ pea %sp@(SWITCH_STACK_SIZE)
+ jbsr m68k_vfork
+ addql #4,%sp
+ RESTORE_SWITCH_STACK
+ rts
+
+ENTRY(sys_sigreturn)
+ SAVE_SWITCH_STACK
+ jbsr do_sigreturn
+ RESTORE_SWITCH_STACK
+ rts
+
+ENTRY(sys_rt_sigreturn)
+ SAVE_SWITCH_STACK
+ jbsr do_rt_sigreturn
+ RESTORE_SWITCH_STACK
+ rts
+
+ENTRY(buserr)
+ SAVE_ALL_INT
+ GET_CURRENT(%d0)
+ movel %sp,%sp@- | stack frame pointer argument
+ jbsr buserr_c
+ addql #4,%sp
+ jra ret_from_exception
+
+ENTRY(trap)
+ SAVE_ALL_INT
+ GET_CURRENT(%d0)
+ movel %sp,%sp@- | stack frame pointer argument
+ jbsr trap_c
+ addql #4,%sp
+ jra ret_from_exception
+
+ | After a fork we jump here directly from resume,
+ | so that %d1 contains the previous task
+ | schedule_tail now used regardless of CONFIG_SMP
+ENTRY(ret_from_fork)
+ movel %d1,%sp@-
+ jsr schedule_tail
+ addql #4,%sp
+ jra ret_from_exception
+
+#if defined(CONFIG_COLDFIRE) || !defined(CONFIG_MMU)
+
+#ifdef TRAP_DBG_INTERRUPT
+
+.globl dbginterrupt
+ENTRY(dbginterrupt)
+ SAVE_ALL_INT
+ GET_CURRENT(%d0)
+ movel %sp,%sp@- /* stack frame pointer argument */
+ jsr dbginterrupt_c
+ addql #4,%sp
+ jra ret_from_exception
+#endif
+
+ENTRY(reschedule)
+ /* save top of frame */
+ pea %sp@
+ jbsr set_esp0
+ addql #4,%sp
+ pea ret_from_exception
+ jmp schedule
+
+ENTRY(ret_from_user_signal)
+ moveq #__NR_sigreturn,%d0
+ trap #0
+
+ENTRY(ret_from_user_rt_signal)
+ movel #__NR_rt_sigreturn,%d0
+ trap #0
+
#else
-#include "entry_no.S"
+
+do_trace_entry:
+ movel #-ENOSYS,%sp@(PT_OFF_D0)| needed for strace
+ subql #4,%sp
+ SAVE_SWITCH_STACK
+ jbsr syscall_trace
+ RESTORE_SWITCH_STACK
+ addql #4,%sp
+ movel %sp@(PT_OFF_ORIG_D0),%d0
+ cmpl #NR_syscalls,%d0
+ jcs syscall
+badsys:
+ movel #-ENOSYS,%sp@(PT_OFF_D0)
+ jra ret_from_syscall
+
+do_trace_exit:
+ subql #4,%sp
+ SAVE_SWITCH_STACK
+ jbsr syscall_trace
+ RESTORE_SWITCH_STACK
+ addql #4,%sp
+ jra .Lret_from_exception
+
+ENTRY(ret_from_signal)
+ movel %curptr@(TASK_STACK),%a1
+ tstb %a1@(TINFO_FLAGS+2)
+ jge 1f
+ jbsr syscall_trace
+1: RESTORE_SWITCH_STACK
+ addql #4,%sp
+/* on 68040 complete pending writebacks if any */
+#ifdef CONFIG_M68040
+ bfextu %sp@(PT_OFF_FORMATVEC){#0,#4},%d0
+ subql #7,%d0 | bus error frame ?
+ jbne 1f
+ movel %sp,%sp@-
+ jbsr berr_040cleanup
+ addql #4,%sp
+1:
+#endif
+ jra .Lret_from_exception
+
+ENTRY(system_call)
+ SAVE_ALL_SYS
+
+ GET_CURRENT(%d1)
+ movel %d1,%a1
+
+ | save top of frame
+ movel %sp,%curptr@(TASK_THREAD+THREAD_ESP0)
+
+ | syscall trace?
+ tstb %a1@(TINFO_FLAGS+2)
+ jmi do_trace_entry
+ cmpl #NR_syscalls,%d0
+ jcc badsys
+syscall:
+ jbsr @(sys_call_table,%d0:l:4)@(0)
+ movel %d0,%sp@(PT_OFF_D0) | save the return value
+ret_from_syscall:
+ |oriw #0x0700,%sr
+ movel %curptr@(TASK_STACK),%a1
+ movew %a1@(TINFO_FLAGS+2),%d0
+ jne syscall_exit_work
+1: RESTORE_ALL
+
+syscall_exit_work:
+ btst #5,%sp@(PT_OFF_SR) | check if returning to kernel
+ bnes 1b | if so, skip resched, signals
+ lslw #1,%d0
+ jcs do_trace_exit
+ jmi do_delayed_trace
+ lslw #8,%d0
+ jne do_signal_return
+ pea resume_userspace
+ jra schedule
+
+
+ENTRY(ret_from_exception)
+.Lret_from_exception:
+ btst #5,%sp@(PT_OFF_SR) | check if returning to kernel
+ bnes 1f | if so, skip resched, signals
+ | only allow interrupts when we are really the last one on the
+ | kernel stack, otherwise stack overflow can occur during
+ | heavy interrupt load
+ andw #ALLOWINT,%sr
+
+resume_userspace:
+ movel %curptr@(TASK_STACK),%a1
+ moveb %a1@(TINFO_FLAGS+3),%d0
+ jne exit_work
+1: RESTORE_ALL
+
+exit_work:
+ | save top of frame
+ movel %sp,%curptr@(TASK_THREAD+THREAD_ESP0)
+ lslb #1,%d0
+ jne do_signal_return
+ pea resume_userspace
+ jra schedule
+
+
+do_signal_return:
+ |andw #ALLOWINT,%sr
+ subql #4,%sp | dummy return address
+ SAVE_SWITCH_STACK
+ pea %sp@(SWITCH_STACK_SIZE)
+ bsrl do_notify_resume
+ addql #4,%sp
+ RESTORE_SWITCH_STACK
+ addql #4,%sp
+ jbra resume_userspace
+
+do_delayed_trace:
+ bclr #7,%sp@(PT_OFF_SR) | clear trace bit in SR
+ pea 1 | send SIGTRAP
+ movel %curptr,%sp@-
+ pea LSIGTRAP
+ jbsr send_sig
+ addql #8,%sp
+ addql #4,%sp
+ jbra resume_userspace
+
+
+/* This is the main interrupt handler for autovector interrupts */
+
+ENTRY(auto_inthandler)
+ SAVE_ALL_INT
+ GET_CURRENT(%d0)
+ movel %d0,%a1
+ addqb #1,%a1@(TINFO_PREEMPT+1)
+ | put exception # in d0
+ bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0
+ subw #VEC_SPUR,%d0
+
+ movel %sp,%sp@-
+ movel %d0,%sp@- | put vector # on stack
+auto_irqhandler_fixup = . + 2
+ jsr do_IRQ | process the IRQ
+ addql #8,%sp | pop parameters off stack
+
+ret_from_interrupt:
+ movel %curptr@(TASK_STACK),%a1
+ subqb #1,%a1@(TINFO_PREEMPT+1)
+ jeq ret_from_last_interrupt
+2: RESTORE_ALL
+
+ ALIGN
+ret_from_last_interrupt:
+ moveq #(~ALLOWINT>>8)&0xff,%d0
+ andb %sp@(PT_OFF_SR),%d0
+ jne 2b
+
+ /* check if we need to do software interrupts */
+ tstl irq_stat+CPUSTAT_SOFTIRQ_PENDING
+ jeq .Lret_from_exception
+ pea ret_from_exception
+ jra do_softirq
+
+/* Handler for user defined interrupt vectors */
+
+ENTRY(user_inthandler)
+ SAVE_ALL_INT
+ GET_CURRENT(%d0)
+ movel %d0,%a1
+ addqb #1,%a1@(TINFO_PREEMPT+1)
+ | put exception # in d0
+ bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0
+user_irqvec_fixup = . + 2
+ subw #VEC_USER,%d0
+
+ movel %sp,%sp@-
+ movel %d0,%sp@- | put vector # on stack
+ jsr do_IRQ | process the IRQ
+ addql #8,%sp | pop parameters off stack
+
+ movel %curptr@(TASK_STACK),%a1
+ subqb #1,%a1@(TINFO_PREEMPT+1)
+ jeq ret_from_last_interrupt
+ RESTORE_ALL
+
+/* Handler for uninitialized and spurious interrupts */
+
+ENTRY(bad_inthandler)
+ SAVE_ALL_INT
+ GET_CURRENT(%d0)
+ movel %d0,%a1
+ addqb #1,%a1@(TINFO_PREEMPT+1)
+
+ movel %sp,%sp@-
+ jsr handle_badint
+ addql #4,%sp
+
+ movel %curptr@(TASK_STACK),%a1
+ subqb #1,%a1@(TINFO_PREEMPT+1)
+ jeq ret_from_last_interrupt
+ RESTORE_ALL
+
+
+resume:
+ /*
+ * Beware - when entering resume, prev (the current task) is
+ * in a0, next (the new task) is in a1,so don't change these
+ * registers until their contents are no longer needed.
+ */
+
+ /* save sr */
+ movew %sr,%a0@(TASK_THREAD+THREAD_SR)
+
+ /* save fs (sfc,%dfc) (may be pointing to kernel memory) */
+ movec %sfc,%d0
+ movew %d0,%a0@(TASK_THREAD+THREAD_FS)
+
+ /* save usp */
+ /* it is better to use a movel here instead of a movew 8*) */
+ movec %usp,%d0
+ movel %d0,%a0@(TASK_THREAD+THREAD_USP)
+
+ /* save non-scratch registers on stack */
+ SAVE_SWITCH_STACK
+
+ /* save current kernel stack pointer */
+ movel %sp,%a0@(TASK_THREAD+THREAD_KSP)
+
+ /* save floating point context */
+#ifndef CONFIG_M68KFPU_EMU_ONLY
+#ifdef CONFIG_M68KFPU_EMU
+ tstl m68k_fputype
+ jeq 3f
+#endif
+ fsave %a0@(TASK_THREAD+THREAD_FPSTATE)
+
+#if defined(CONFIG_M68060)
+#if !defined(CPU_M68060_ONLY)
+ btst #3,m68k_cputype+3
+ beqs 1f
+#endif
+ /* The 060 FPU keeps status in bits 15-8 of the first longword */
+ tstb %a0@(TASK_THREAD+THREAD_FPSTATE+2)
+ jeq 3f
+#if !defined(CPU_M68060_ONLY)
+ jra 2f
+#endif
+#endif /* CONFIG_M68060 */
+#if !defined(CPU_M68060_ONLY)
+1: tstb %a0@(TASK_THREAD+THREAD_FPSTATE)
+ jeq 3f
+#endif
+2: fmovemx %fp0-%fp7,%a0@(TASK_THREAD+THREAD_FPREG)
+ fmoveml %fpcr/%fpsr/%fpiar,%a0@(TASK_THREAD+THREAD_FPCNTL)
+3:
+#endif /* CONFIG_M68KFPU_EMU_ONLY */
+ /* Return previous task in %d1 */
+ movel %curptr,%d1
+
+ /* switch to new task (a1 contains new task) */
+ movel %a1,%curptr
+
+ /* restore floating point context */
+#ifndef CONFIG_M68KFPU_EMU_ONLY
+#ifdef CONFIG_M68KFPU_EMU
+ tstl m68k_fputype
+ jeq 4f
+#endif
+#if defined(CONFIG_M68060)
+#if !defined(CPU_M68060_ONLY)
+ btst #3,m68k_cputype+3
+ beqs 1f
+#endif
+ /* The 060 FPU keeps status in bits 15-8 of the first longword */
+ tstb %a1@(TASK_THREAD+THREAD_FPSTATE+2)
+ jeq 3f
+#if !defined(CPU_M68060_ONLY)
+ jra 2f
+#endif
+#endif /* CONFIG_M68060 */
+#if !defined(CPU_M68060_ONLY)
+1: tstb %a1@(TASK_THREAD+THREAD_FPSTATE)
+ jeq 3f
#endif
+2: fmovemx %a1@(TASK_THREAD+THREAD_FPREG),%fp0-%fp7
+ fmoveml %a1@(TASK_THREAD+THREAD_FPCNTL),%fpcr/%fpsr/%fpiar
+3: frestore %a1@(TASK_THREAD+THREAD_FPSTATE)
+4:
+#endif /* CONFIG_M68KFPU_EMU_ONLY */
+
+ /* restore the kernel stack pointer */
+ movel %a1@(TASK_THREAD+THREAD_KSP),%sp
+
+ /* restore non-scratch registers */
+ RESTORE_SWITCH_STACK
+
+ /* restore user stack pointer */
+ movel %a1@(TASK_THREAD+THREAD_USP),%a0
+ movel %a0,%usp
+
+ /* restore fs (sfc,%dfc) */
+ movew %a1@(TASK_THREAD+THREAD_FS),%a0
+ movec %a0,%sfc
+ movec %a0,%dfc
+
+ /* restore status register */
+ movew %a1@(TASK_THREAD+THREAD_SR),%sr
+
+ rts
+
+#endif /* CONFIG_MMU && !CONFIG_COLDFIRE */
diff --git a/arch/m68k/kernel/entry_mm.S b/arch/m68k/kernel/entry_mm.S
deleted file mode 100644
index f29e73ca9dbb..000000000000
--- a/arch/m68k/kernel/entry_mm.S
+++ /dev/null
@@ -1,419 +0,0 @@
-/* -*- mode: asm -*-
- *
- * linux/arch/m68k/kernel/entry.S
- *
- * Copyright (C) 1991, 1992 Linus Torvalds
- *
- * This file is subject to the terms and conditions of the GNU General Public
- * License. See the file README.legal in the main directory of this archive
- * for more details.
- *
- * Linux/m68k support by Hamish Macdonald
- *
- * 68060 fixes by Jesper Skov
- *
- */
-
-/*
- * entry.S contains the system-call and fault low-level handling routines.
- * This also contains the timer-interrupt handler, as well as all interrupts
- * and faults that can result in a task-switch.
- *
- * NOTE: This code handles signal-recognition, which happens every time
- * after a timer-interrupt and after each system call.
- *
- */
-
-/*
- * 12/03/96 Jes: Currently we only support m68k single-cpu systems, so
- * all pointers that used to be 'current' are now entry
- * number 0 in the 'current_set' list.
- *
- * 6/05/00 RZ: addedd writeback completion after return from sighandler
- * for 68040
- */
-
-#include <linux/linkage.h>
-#include <asm/entry.h>
-#include <asm/errno.h>
-#include <asm/setup.h>
-#include <asm/segment.h>
-#include <asm/traps.h>
-#include <asm/unistd.h>
-
-#include <asm/asm-offsets.h>
-
-.globl system_call, buserr, trap, resume
-.globl sys_call_table
-.globl sys_fork, sys_clone, sys_vfork
-.globl ret_from_interrupt, bad_interrupt
-.globl auto_irqhandler_fixup
-.globl user_irqvec_fixup
-
-.text
-ENTRY(buserr)
- SAVE_ALL_INT
- GET_CURRENT(%d0)
- movel %sp,%sp@- | stack frame pointer argument
- bsrl buserr_c
- addql #4,%sp
- jra .Lret_from_exception
-
-ENTRY(trap)
- SAVE_ALL_INT
- GET_CURRENT(%d0)
- movel %sp,%sp@- | stack frame pointer argument
- bsrl trap_c
- addql #4,%sp
- jra .Lret_from_exception
-
- | After a fork we jump here directly from resume,
- | so that %d1 contains the previous task
- | schedule_tail now used regardless of CONFIG_SMP
-ENTRY(ret_from_fork)
- movel %d1,%sp@-
- jsr schedule_tail
- addql #4,%sp
- jra .Lret_from_exception
-
-do_trace_entry:
- movel #-ENOSYS,%sp@(PT_OFF_D0)| needed for strace
- subql #4,%sp
- SAVE_SWITCH_STACK
- jbsr syscall_trace
- RESTORE_SWITCH_STACK
- addql #4,%sp
- movel %sp@(PT_OFF_ORIG_D0),%d0
- cmpl #NR_syscalls,%d0
- jcs syscall
-badsys:
- movel #-ENOSYS,%sp@(PT_OFF_D0)
- jra ret_from_syscall
-
-do_trace_exit:
- subql #4,%sp
- SAVE_SWITCH_STACK
- jbsr syscall_trace
- RESTORE_SWITCH_STACK
- addql #4,%sp
- jra .Lret_from_exception
-
-ENTRY(ret_from_signal)
- movel %curptr@(TASK_STACK),%a1
- tstb %a1@(TINFO_FLAGS+2)
- jge 1f
- jbsr syscall_trace
-1: RESTORE_SWITCH_STACK
- addql #4,%sp
-/* on 68040 complete pending writebacks if any */
-#ifdef CONFIG_M68040
- bfextu %sp@(PT_OFF_FORMATVEC){#0,#4},%d0
- subql #7,%d0 | bus error frame ?
- jbne 1f
- movel %sp,%sp@-
- jbsr berr_040cleanup
- addql #4,%sp
-1:
-#endif
- jra .Lret_from_exception
-
-ENTRY(system_call)
- SAVE_ALL_SYS
-
- GET_CURRENT(%d1)
- movel %d1,%a1
-
- | save top of frame
- movel %sp,%curptr@(TASK_THREAD+THREAD_ESP0)
-
- | syscall trace?
- tstb %a1@(TINFO_FLAGS+2)
- jmi do_trace_entry
- cmpl #NR_syscalls,%d0
- jcc badsys
-syscall:
- jbsr @(sys_call_table,%d0:l:4)@(0)
- movel %d0,%sp@(PT_OFF_D0) | save the return value
-ret_from_syscall:
- |oriw #0x0700,%sr
- movel %curptr@(TASK_STACK),%a1
- movew %a1@(TINFO_FLAGS+2),%d0
- jne syscall_exit_work
-1: RESTORE_ALL
-
-syscall_exit_work:
- btst #5,%sp@(PT_OFF_SR) | check if returning to kernel
- bnes 1b | if so, skip resched, signals
- lslw #1,%d0
- jcs do_trace_exit
- jmi do_delayed_trace
- lslw #8,%d0
- jne do_signal_return
- pea resume_userspace
- jra schedule
-
-
-ENTRY(ret_from_exception)
-.Lret_from_exception:
- btst #5,%sp@(PT_OFF_SR) | check if returning to kernel
- bnes 1f | if so, skip resched, signals
- | only allow interrupts when we are really the last one on the
- | kernel stack, otherwise stack overflow can occur during
- | heavy interrupt load
- andw #ALLOWINT,%sr
-
-resume_userspace:
- movel %curptr@(TASK_STACK),%a1
- moveb %a1@(TINFO_FLAGS+3),%d0
- jne exit_work
-1: RESTORE_ALL
-
-exit_work:
- | save top of frame
- movel %sp,%curptr@(TASK_THREAD+THREAD_ESP0)
- lslb #1,%d0
- jne do_signal_return
- pea resume_userspace
- jra schedule
-
-
-do_signal_return:
- |andw #ALLOWINT,%sr
- subql #4,%sp | dummy return address
- SAVE_SWITCH_STACK
- pea %sp@(SWITCH_STACK_SIZE)
- bsrl do_notify_resume
- addql #4,%sp
- RESTORE_SWITCH_STACK
- addql #4,%sp
- jbra resume_userspace
-
-do_delayed_trace:
- bclr #7,%sp@(PT_OFF_SR) | clear trace bit in SR
- pea 1 | send SIGTRAP
- movel %curptr,%sp@-
- pea LSIGTRAP
- jbsr send_sig
- addql #8,%sp
- addql #4,%sp
- jbra resume_userspace
-
-
-/* This is the main interrupt handler for autovector interrupts */
-
-ENTRY(auto_inthandler)
- SAVE_ALL_INT
- GET_CURRENT(%d0)
- movel %d0,%a1
- addqb #1,%a1@(TINFO_PREEMPT+1)
- | put exception # in d0
- bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0
- subw #VEC_SPUR,%d0
-
- movel %sp,%sp@-
- movel %d0,%sp@- | put vector # on stack
-auto_irqhandler_fixup = . + 2
- jsr do_IRQ | process the IRQ
- addql #8,%sp | pop parameters off stack
-
-ret_from_interrupt:
- movel %curptr@(TASK_STACK),%a1
- subqb #1,%a1@(TINFO_PREEMPT+1)
- jeq ret_from_last_interrupt
-2: RESTORE_ALL
-
- ALIGN
-ret_from_last_interrupt:
- moveq #(~ALLOWINT>>8)&0xff,%d0
- andb %sp@(PT_OFF_SR),%d0
- jne 2b
-
- /* check if we need to do software interrupts */
- tstl irq_stat+CPUSTAT_SOFTIRQ_PENDING
- jeq .Lret_from_exception
- pea ret_from_exception
- jra do_softirq
-
-/* Handler for user defined interrupt vectors */
-
-ENTRY(user_inthandler)
- SAVE_ALL_INT
- GET_CURRENT(%d0)
- movel %d0,%a1
- addqb #1,%a1@(TINFO_PREEMPT+1)
- | put exception # in d0
- bfextu %sp@(PT_OFF_FORMATVEC){#4,#10},%d0
-user_irqvec_fixup = . + 2
- subw #VEC_USER,%d0
-
- movel %sp,%sp@-
- movel %d0,%sp@- | put vector # on stack
- jsr do_IRQ | process the IRQ
- addql #8,%sp | pop parameters off stack
-
- movel %curptr@(TASK_STACK),%a1
- subqb #1,%a1@(TINFO_PREEMPT+1)
- jeq ret_from_last_interrupt
- RESTORE_ALL
-
-/* Handler for uninitialized and spurious interrupts */
-
-ENTRY(bad_inthandler)
- SAVE_ALL_INT
- GET_CURRENT(%d0)
- movel %d0,%a1
- addqb #1,%a1@(TINFO_PREEMPT+1)
-
- movel %sp,%sp@-
- jsr handle_badint
- addql #4,%sp
-
- movel %curptr@(TASK_STACK),%a1
- subqb #1,%a1@(TINFO_PREEMPT+1)
- jeq ret_from_last_interrupt
- RESTORE_ALL
-
-
-ENTRY(sys_fork)
- SAVE_SWITCH_STACK
- pea %sp@(SWITCH_STACK_SIZE)
- jbsr m68k_fork
- addql #4,%sp
- RESTORE_SWITCH_STACK
- rts
-
-ENTRY(sys_clone)
- SAVE_SWITCH_STACK
- pea %sp@(SWITCH_STACK_SIZE)
- jbsr m68k_clone
- addql #4,%sp
- RESTORE_SWITCH_STACK
- rts
-
-ENTRY(sys_vfork)
- SAVE_SWITCH_STACK
- pea %sp@(SWITCH_STACK_SIZE)
- jbsr m68k_vfork
- addql #4,%sp
- RESTORE_SWITCH_STACK
- rts
-
-ENTRY(sys_sigreturn)
- SAVE_SWITCH_STACK
- jbsr do_sigreturn
- RESTORE_SWITCH_STACK
- rts
-
-ENTRY(sys_rt_sigreturn)
- SAVE_SWITCH_STACK
- jbsr do_rt_sigreturn
- RESTORE_SWITCH_STACK
- rts
-
-resume:
- /*
- * Beware - when entering resume, prev (the current task) is
- * in a0, next (the new task) is in a1,so don't change these
- * registers until their contents are no longer needed.
- */
-
- /* save sr */
- movew %sr,%a0@(TASK_THREAD+THREAD_SR)
-
- /* save fs (sfc,%dfc) (may be pointing to kernel memory) */
- movec %sfc,%d0
- movew %d0,%a0@(TASK_THREAD+THREAD_FS)
-
- /* save usp */
- /* it is better to use a movel here instead of a movew 8*) */
- movec %usp,%d0
- movel %d0,%a0@(TASK_THREAD+THREAD_USP)
-
- /* save non-scratch registers on stack */
- SAVE_SWITCH_STACK
-
- /* save current kernel stack pointer */
- movel %sp,%a0@(TASK_THREAD+THREAD_KSP)
-
- /* save floating point context */
-#ifndef CONFIG_M68KFPU_EMU_ONLY
-#ifdef CONFIG_M68KFPU_EMU
- tstl m68k_fputype
- jeq 3f
-#endif
- fsave %a0@(TASK_THREAD+THREAD_FPSTATE)
-
-#if defined(CONFIG_M68060)
-#if !defined(CPU_M68060_ONLY)
- btst #3,m68k_cputype+3
- beqs 1f
-#endif
- /* The 060 FPU keeps status in bits 15-8 of the first longword */
- tstb %a0@(TASK_THREAD+THREAD_FPSTATE+2)
- jeq 3f
-#if !defined(CPU_M68060_ONLY)
- jra 2f
-#endif
-#endif /* CONFIG_M68060 */
-#if !defined(CPU_M68060_ONLY)
-1: tstb %a0@(TASK_THREAD+THREAD_FPSTATE)
- jeq 3f
-#endif
-2: fmovemx %fp0-%fp7,%a0@(TASK_THREAD+THREAD_FPREG)
- fmoveml %fpcr/%fpsr/%fpiar,%a0@(TASK_THREAD+THREAD_FPCNTL)
-3:
-#endif /* CONFIG_M68KFPU_EMU_ONLY */
- /* Return previous task in %d1 */
- movel %curptr,%d1
-
- /* switch to new task (a1 contains new task) */
- movel %a1,%curptr
-
- /* restore floating point context */
-#ifndef CONFIG_M68KFPU_EMU_ONLY
-#ifdef CONFIG_M68KFPU_EMU
- tstl m68k_fputype
- jeq 4f
-#endif
-#if defined(CONFIG_M68060)
-#if !defined(CPU_M68060_ONLY)
- btst #3,m68k_cputype+3
- beqs 1f
-#endif
- /* The 060 FPU keeps status in bits 15-8 of the first longword */
- tstb %a1@(TASK_THREAD+THREAD_FPSTATE+2)
- jeq 3f
-#if !defined(CPU_M68060_ONLY)
- jra 2f
-#endif
-#endif /* CONFIG_M68060 */
-#if !defined(CPU_M68060_ONLY)
-1: tstb %a1@(TASK_THREAD+THREAD_FPSTATE)
- jeq 3f
-#endif
-2: fmovemx %a1@(TASK_THREAD+THREAD_FPREG),%fp0-%fp7
- fmoveml %a1@(TASK_THREAD+THREAD_FPCNTL),%fpcr/%fpsr/%fpiar
-3: frestore %a1@(TASK_THREAD+THREAD_FPSTATE)
-4:
-#endif /* CONFIG_M68KFPU_EMU_ONLY */
-
- /* restore the kernel stack pointer */
- movel %a1@(TASK_THREAD+THREAD_KSP),%sp
-
- /* restore non-scratch registers */
- RESTORE_SWITCH_STACK
-
- /* restore user stack pointer */
- movel %a1@(TASK_THREAD+THREAD_USP),%a0
- movel %a0,%usp
-
- /* restore fs (sfc,%dfc) */
- movew %a1@(TASK_THREAD+THREAD_FS),%a0
- movec %a0,%sfc
- movec %a0,%dfc
-
- /* restore status register */
- movew %a1@(TASK_THREAD+THREAD_SR),%sr
-
- rts
-
diff --git a/arch/m68k/kernel/entry_no.S b/arch/m68k/kernel/entry_no.S
deleted file mode 100644
index d80cba45589f..000000000000
--- a/arch/m68k/kernel/entry_no.S
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- * linux/arch/m68knommu/kernel/entry.S
- *
- * Copyright (C) 1999-2002, Greg Ungerer (gerg@snapgear.com)
- * Copyright (C) 1998 D. Jeff Dionne <jeff@lineo.ca>,
- * Kenneth Albanowski <kjahds@kjahds.com>,
- * Copyright (C) 2000 Lineo Inc. (www.lineo.com)
- *
- * Based on:
- *
- * linux/arch/m68k/kernel/entry.S
- *
- * Copyright (C) 1991, 1992 Linus Torvalds
- *
- * This file is subject to the terms and conditions of the GNU General Public
- * License. See the file README.legal in the main directory of this archive
- * for more details.
- *
- * Linux/m68k support by Hamish Macdonald
- *
- * 68060 fixes by Jesper Skov
- * ColdFire support by Greg Ungerer (gerg@snapgear.com)
- * 5307 fixes by David W. Miller
- * linux 2.4 support David McCullough <davidm@snapgear.com>
- */
-
-#include <linux/linkage.h>
-#include <asm/errno.h>
-#include <asm/setup.h>
-#include <asm/segment.h>
-#include <asm/asm-offsets.h>
-#include <asm/entry.h>
-#include <asm/unistd.h>
-
-.text
-
-.globl buserr
-.globl trap
-.globl ret_from_exception
-.globl ret_from_signal
-.globl sys_fork
-.globl sys_clone
-.globl sys_vfork
-
-ENTRY(buserr)
- SAVE_ALL_INT
- GET_CURRENT(%d0)
- movel %sp,%sp@- /* stack frame pointer argument */
- jsr buserr_c
- addql #4,%sp
- jra ret_from_exception
-
-ENTRY(trap)
- SAVE_ALL_INT
- GET_CURRENT(%d0)
- movel %sp,%sp@- /* stack frame pointer argument */
- jsr trap_c
- addql #4,%sp
- jra ret_from_exception
-
-#ifdef TRAP_DBG_INTERRUPT
-
-.globl dbginterrupt
-ENTRY(dbginterrupt)
- SAVE_ALL_INT
- GET_CURRENT(%d0)
- movel %sp,%sp@- /* stack frame pointer argument */
- jsr dbginterrupt_c
- addql #4,%sp
- jra ret_from_exception
-#endif
-
-ENTRY(reschedule)
- /* save top of frame */
- pea %sp@
- jbsr set_esp0
- addql #4,%sp
- pea ret_from_exception
- jmp schedule
-
-ENTRY(ret_from_fork)
- movel %d1,%sp@-
- jsr schedule_tail
- addql #4,%sp
- jra ret_from_exception
-
-ENTRY(sys_fork)
- SAVE_SWITCH_STACK
- pea %sp@(SWITCH_STACK_SIZE)
- jbsr m68k_fork
- addql #4,%sp
- RESTORE_SWITCH_STACK
- rts
-
-ENTRY(sys_vfork)
- SAVE_SWITCH_STACK
- pea %sp@(SWITCH_STACK_SIZE)
- jbsr m68k_vfork
- addql #4,%sp
- RESTORE_SWITCH_STACK
- rts
-
-ENTRY(sys_clone)
- SAVE_SWITCH_STACK
- pea %sp@(SWITCH_STACK_SIZE)
- jbsr m68k_clone
- addql #4,%sp
- RESTORE_SWITCH_STACK
- rts
-
-ENTRY(sys_sigreturn)
- SAVE_SWITCH_STACK
- jbsr do_sigreturn
- RESTORE_SWITCH_STACK
- rts
-
-ENTRY(sys_rt_sigreturn)
- SAVE_SWITCH_STACK
- jbsr do_rt_sigreturn
- RESTORE_SWITCH_STACK
- rts
-
-ENTRY(ret_from_user_signal)
- moveq #__NR_sigreturn,%d0
- trap #0
-
-ENTRY(ret_from_user_rt_signal)
- movel #__NR_rt_sigreturn,%d0
- trap #0
-
diff --git a/arch/m68k/kernel/pcibios.c b/arch/m68k/kernel/pcibios.c
new file mode 100644
index 000000000000..b2988aa1840b
--- /dev/null
+++ b/arch/m68k/kernel/pcibios.c
@@ -0,0 +1,109 @@
+/*
+ * pci.c -- basic PCI support code
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the
+ * Free Software Foundation; either version 2 of the License, or (at your
+ * option) any later version.
+ *
+ * (C) Copyright 2011, Greg Ungerer <gerg@uclinux.org>
+ */
+
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/mm.h>
+#include <linux/init.h>
+#include <linux/pci.h>
+
+/*
+ * From arch/i386/kernel/pci-i386.c:
+ *
+ * We need to avoid collisions with `mirrored' VGA ports
+ * and other strange ISA hardware, so we always want the
+ * addresses to be allocated in the 0x000-0x0ff region
+ * modulo 0x400.
+ *
+ * Why? Because some silly external IO cards only decode
+ * the low 10 bits of the IO address. The 0x00-0xff region
+ * is reserved for motherboard devices that decode all 16
+ * bits, so it's ok to allocate at, say, 0x2800-0x28ff,
+ * but we want to try to avoid allocating at 0x2900-0x2bff
+ * which might be mirrored at 0x0100-0x03ff..
+ */
+resource_size_t pcibios_align_resource(void *data, const struct resource *res,
+ resource_size_t size, resource_size_t align)
+{
+ resource_size_t start = res->start;
+
+ if ((res->flags & IORESOURCE_IO) && (start & 0x300))
+ start = (start + 0x3ff) & ~0x3ff;
+
+ start = (start + align - 1) & ~(align - 1);
+
+ return start;
+}
+
+/*
+ * This is taken from the ARM code for this.
+ */
+int pcibios_enable_device(struct pci_dev *dev, int mask)
+{
+ struct resource *r;
+ u16 cmd, newcmd;
+ int idx;
+
+ pci_read_config_word(dev, PCI_COMMAND, &cmd);
+ newcmd = cmd;
+
+ for (idx = 0; idx < 6; idx++) {
+ /* Only set up the requested stuff */
+ if (!(mask & (1 << idx)))
+ continue;
+
+ r = dev->resource + idx;
+ if (!r->start && r->end) {
+ pr_err(KERN_ERR "PCI: Device %s not available because of resource collisions\n",
+ pci_name(dev));
+ return -EINVAL;
+ }
+ if (r->flags & IORESOURCE_IO)
+ newcmd |= PCI_COMMAND_IO;
+ if (r->flags & IORESOURCE_MEM)
+ newcmd |= PCI_COMMAND_MEMORY;
+ }
+
+ /*
+ * Bridges (eg, cardbus bridges) need to be fully enabled
+ */
+ if ((dev->class >> 16) == PCI_BASE_CLASS_BRIDGE)
+ newcmd |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY;
+
+
+ if (newcmd != cmd) {
+ pr_info("PCI: enabling device %s (0x%04x -> 0x%04x)\n",
+ pci_name(dev), cmd, newcmd);
+ pci_write_config_word(dev, PCI_COMMAND, newcmd);
+ }
+ return 0;
+}
+
+void pcibios_update_irq(struct pci_dev *dev, int irq)
+{
+ pci_write_config_byte(dev, PCI_INTERRUPT_LINE, irq);
+}
+
+void __devinit pcibios_fixup_bus(struct pci_bus *bus)
+{
+ struct pci_dev *dev;
+
+ list_for_each_entry(dev, &bus->devices, bus_list) {
+ pci_write_config_byte(dev, PCI_CACHE_LINE_SIZE, 8);
+ pci_write_config_byte(dev, PCI_LATENCY_TIMER, 32);
+ }
+}
+
+char __devinit *pcibios_setup(char *str)
+{
+ return str;
+}
+
diff --git a/arch/m68k/mm/memory.c b/arch/m68k/mm/memory.c
index 250b8b786f4f..51bc9d258ede 100644
--- a/arch/m68k/mm/memory.c
+++ b/arch/m68k/mm/memory.c
@@ -203,7 +203,7 @@ static inline void pushcl040(unsigned long paddr)
void cache_clear (unsigned long paddr, int len)
{
if (CPU_IS_COLDFIRE) {
- flush_cf_bcache(0, DCACHE_MAX_ADDR);
+ clear_cf_bcache(0, DCACHE_MAX_ADDR);
} else if (CPU_IS_040_OR_060) {
int tmp;
diff --git a/arch/m68k/platform/coldfire/Makefile b/arch/m68k/platform/coldfire/Makefile
index 76d389d9a84e..02591a109f8c 100644
--- a/arch/m68k/platform/coldfire/Makefile
+++ b/arch/m68k/platform/coldfire/Makefile
@@ -20,6 +20,7 @@ obj-$(CONFIG_M5206e) += m5206.o timers.o intc.o reset.o
obj-$(CONFIG_M520x) += m520x.o pit.o intc-simr.o reset.o
obj-$(CONFIG_M523x) += m523x.o pit.o dma_timer.o intc-2.o reset.o
obj-$(CONFIG_M5249) += m5249.o timers.o intc.o intc-5249.o reset.o
+obj-$(CONFIG_M525x) += m525x.o timers.o intc.o intc-525x.o reset.o
obj-$(CONFIG_M527x) += m527x.o pit.o intc-2.o reset.o
obj-$(CONFIG_M5272) += m5272.o intc-5272.o timers.o
obj-$(CONFIG_M528x) += m528x.o pit.o intc-2.o reset.o
@@ -27,10 +28,14 @@ obj-$(CONFIG_M5307) += m5307.o timers.o intc.o reset.o
obj-$(CONFIG_M532x) += m532x.o timers.o intc-simr.o reset.o
obj-$(CONFIG_M5407) += m5407.o timers.o intc.o reset.o
obj-$(CONFIG_M54xx) += m54xx.o sltimers.o intc-2.o
+obj-$(CONFIG_M5441x) += m5441x.o pit.o intc-simr.o reset.o
obj-$(CONFIG_NETtel) += nettel.o
obj-$(CONFIG_CLEOPATRA) += nettel.o
obj-$(CONFIG_FIREBEE) += firebee.o
+obj-$(CONFIG_MCF8390) += mcf8390.o
-obj-y += pinmux.o gpio.o
+obj-$(CONFIG_PCI) += pci.o
+
+obj-y += gpio.o
extra-y := head.o
diff --git a/arch/m68k/platform/coldfire/clk.c b/arch/m68k/platform/coldfire/clk.c
index 44da406897e5..75f9ee967ea7 100644
--- a/arch/m68k/platform/coldfire/clk.c
+++ b/arch/m68k/platform/coldfire/clk.c
@@ -10,11 +10,17 @@
#include <linux/kernel.h>
#include <linux/module.h>
+#include <linux/platform_device.h>
+#include <linux/mutex.h>
#include <linux/clk.h>
+#include <linux/io.h>
+#include <linux/err.h>
#include <asm/coldfire.h>
+#include <asm/mcfsim.h>
+#include <asm/mcfclk.h>
/***************************************************************************/
-
+#ifndef MCFPM_PPMCR0
struct clk *clk_get(struct device *dev, const char *id)
{
return NULL;
@@ -42,11 +48,107 @@ unsigned long clk_get_rate(struct clk *clk)
return MCF_CLK;
}
EXPORT_SYMBOL(clk_get_rate);
+#else
+static DEFINE_SPINLOCK(clk_lock);
+
+struct clk *clk_get(struct device *dev, const char *id)
+{
+ const char *clk_name = dev ? dev_name(dev) : id ? id : NULL;
+ struct clk *clk;
+ unsigned i;
+
+ for (i = 0; (clk = mcf_clks[i]) != NULL; ++i)
+ if (!strcmp(clk->name, clk_name))
+ return clk;
+ pr_warn("clk_get: didn't find clock %s\n", clk_name);
+ return ERR_PTR(-ENOENT);
+}
+EXPORT_SYMBOL(clk_get);
+
+int clk_enable(struct clk *clk)
+{
+ unsigned long flags;
+ spin_lock_irqsave(&clk_lock, flags);
+ if ((clk->enabled++ == 0) && clk->clk_ops)
+ clk->clk_ops->enable(clk);
+ spin_unlock_irqrestore(&clk_lock, flags);
+
+ return 0;
+}
+EXPORT_SYMBOL(clk_enable);
+
+void clk_disable(struct clk *clk)
+{
+ unsigned long flags;
+ spin_lock_irqsave(&clk_lock, flags);
+ if ((--clk->enabled == 0) && clk->clk_ops)
+ clk->clk_ops->disable(clk);
+ spin_unlock_irqrestore(&clk_lock, flags);
+}
+EXPORT_SYMBOL(clk_disable);
+
+void clk_put(struct clk *clk)
+{
+ if (clk->enabled != 0)
+ pr_warn("clk_put %s still enabled\n", clk->name);
+}
+EXPORT_SYMBOL(clk_put);
+
+unsigned long clk_get_rate(struct clk *clk)
+{
+ return clk->rate;
+}
+EXPORT_SYMBOL(clk_get_rate);
+
+/***************************************************************************/
+
+void __clk_init_enabled(struct clk *clk)
+{
+ clk->enabled = 1;
+ clk->clk_ops->enable(clk);
+}
+
+void __clk_init_disabled(struct clk *clk)
+{
+ clk->enabled = 0;
+ clk->clk_ops->disable(clk);
+}
+
+static void __clk_enable0(struct clk *clk)
+{
+ __raw_writeb(clk->slot, MCFPM_PPMCR0);
+}
+
+static void __clk_disable0(struct clk *clk)
+{
+ __raw_writeb(clk->slot, MCFPM_PPMSR0);
+}
+
+struct clk_ops clk_ops0 = {
+ .enable = __clk_enable0,
+ .disable = __clk_disable0,
+};
+
+#ifdef MCFPM_PPMCR1
+static void __clk_enable1(struct clk *clk)
+{
+ __raw_writeb(clk->slot, MCFPM_PPMCR1);
+}
+
+static void __clk_disable1(struct clk *clk)
+{
+ __raw_writeb(clk->slot, MCFPM_PPMSR1);
+}
+
+struct clk_ops clk_ops1 = {
+ .enable = __clk_enable1,
+ .disable = __clk_disable1,
+};
+#endif /* MCFPM_PPMCR1 */
+#endif /* MCFPM_PPMCR0 */
struct clk *devm_clk_get(struct device *dev, const char *id)
{
return NULL;
}
EXPORT_SYMBOL(devm_clk_get);
-
-/***************************************************************************/
diff --git a/arch/m68k/platform/coldfire/device.c b/arch/m68k/platform/coldfire/device.c
index 3aa77ddea89d..81f0fb5e51cf 100644
--- a/arch/m68k/platform/coldfire/device.c
+++ b/arch/m68k/platform/coldfire/device.c
@@ -13,6 +13,7 @@
#include <linux/io.h>
#include <linux/spi/spi.h>
#include <linux/gpio.h>
+#include <linux/fec.h>
#include <asm/traps.h>
#include <asm/coldfire.h>
#include <asm/mcfsim.h>
@@ -20,7 +21,7 @@
#include <asm/mcfqspi.h>
/*
- * All current ColdFire parts contain from 2, 3 or 4 UARTS.
+ * All current ColdFire parts contain from 2, 3, 4 or 10 UARTS.
*/
static struct mcf_platform_uart mcf_uart_platform_data[] = {
{
@@ -43,6 +44,42 @@ static struct mcf_platform_uart mcf_uart_platform_data[] = {
.irq = MCF_IRQ_UART3,
},
#endif
+#ifdef MCFUART_BASE4
+ {
+ .mapbase = MCFUART_BASE4,
+ .irq = MCF_IRQ_UART4,
+ },
+#endif
+#ifdef MCFUART_BASE5
+ {
+ .mapbase = MCFUART_BASE5,
+ .irq = MCF_IRQ_UART5,
+ },
+#endif
+#ifdef MCFUART_BASE6
+ {
+ .mapbase = MCFUART_BASE6,
+ .irq = MCF_IRQ_UART6,
+ },
+#endif
+#ifdef MCFUART_BASE7
+ {
+ .mapbase = MCFUART_BASE7,
+ .irq = MCF_IRQ_UART7,
+ },
+#endif
+#ifdef MCFUART_BASE8
+ {
+ .mapbase = MCFUART_BASE8,
+ .irq = MCF_IRQ_UART8,
+ },
+#endif
+#ifdef MCFUART_BASE9
+ {
+ .mapbase = MCFUART_BASE9,
+ .irq = MCF_IRQ_UART9,
+ },
+#endif
{ },
};
@@ -53,6 +90,18 @@ static struct platform_device mcf_uart = {
};
#ifdef CONFIG_FEC
+
+#ifdef CONFIG_M5441x
+#define FEC_NAME "enet-fec"
+static struct fec_platform_data fec_pdata = {
+ .phy = PHY_INTERFACE_MODE_RMII,
+};
+#define FEC_PDATA (&fec_pdata)
+#else
+#define FEC_NAME "fec"
+#define FEC_PDATA NULL
+#endif
+
/*
* Some ColdFire cores contain the Fast Ethernet Controller (FEC)
* block. It is Freescale's own hardware block. Some ColdFires
@@ -82,10 +131,11 @@ static struct resource mcf_fec0_resources[] = {
};
static struct platform_device mcf_fec0 = {
- .name = "fec",
+ .name = FEC_NAME,
.id = 0,
.num_resources = ARRAY_SIZE(mcf_fec0_resources),
.resource = mcf_fec0_resources,
+ .dev.platform_data = FEC_PDATA,
};
#ifdef MCFFEC_BASE1
@@ -113,10 +163,11 @@ static struct resource mcf_fec1_resources[] = {
};
static struct platform_device mcf_fec1 = {
- .name = "fec",
+ .name = FEC_NAME,
.id = 1,
.num_resources = ARRAY_SIZE(mcf_fec1_resources),
.resource = mcf_fec1_resources,
+ .dev.platform_data = FEC_PDATA,
};
#endif /* MCFFEC_BASE1 */
#endif /* CONFIG_FEC */
diff --git a/arch/m68k/platform/coldfire/gpio.c b/arch/m68k/platform/coldfire/gpio.c
index 4c8c42450a4e..9cd2b5c70519 100644
--- a/arch/m68k/platform/coldfire/gpio.c
+++ b/arch/m68k/platform/coldfire/gpio.c
@@ -14,119 +14,161 @@
*/
#include <linux/kernel.h>
+#include <linux/module.h>
#include <linux/init.h>
#include <linux/device.h>
-#include <asm/gpio.h>
-#include <asm/pinmux.h>
+#include <linux/io.h>
+#include <asm/coldfire.h>
+#include <asm/mcfsim.h>
#include <asm/mcfgpio.h>
-#define MCF_CHIP(chip) container_of(chip, struct mcf_gpio_chip, gpio_chip)
+int __mcfgpio_get_value(unsigned gpio)
+{
+ return mcfgpio_read(__mcfgpio_ppdr(gpio)) & mcfgpio_bit(gpio);
+}
+EXPORT_SYMBOL(__mcfgpio_get_value);
+
+void __mcfgpio_set_value(unsigned gpio, int value)
+{
+ if (gpio < MCFGPIO_SCR_START) {
+ unsigned long flags;
+ MCFGPIO_PORTTYPE data;
+
+ local_irq_save(flags);
+ data = mcfgpio_read(__mcfgpio_podr(gpio));
+ if (value)
+ data |= mcfgpio_bit(gpio);
+ else
+ data &= ~mcfgpio_bit(gpio);
+ mcfgpio_write(data, __mcfgpio_podr(gpio));
+ local_irq_restore(flags);
+ } else {
+ if (value)
+ mcfgpio_write(mcfgpio_bit(gpio),
+ MCFGPIO_SETR_PORT(gpio));
+ else
+ mcfgpio_write(~mcfgpio_bit(gpio),
+ MCFGPIO_CLRR_PORT(gpio));
+ }
+}
+EXPORT_SYMBOL(__mcfgpio_set_value);
-int mcf_gpio_direction_input(struct gpio_chip *chip, unsigned offset)
+int __mcfgpio_direction_input(unsigned gpio)
{
unsigned long flags;
MCFGPIO_PORTTYPE dir;
- struct mcf_gpio_chip *mcf_chip = MCF_CHIP(chip);
local_irq_save(flags);
- dir = mcfgpio_read(mcf_chip->pddr);
- dir &= ~mcfgpio_bit(chip->base + offset);
- mcfgpio_write(dir, mcf_chip->pddr);
+ dir = mcfgpio_read(__mcfgpio_pddr(gpio));
+ dir &= ~mcfgpio_bit(gpio);
+ mcfgpio_write(dir, __mcfgpio_pddr(gpio));
local_irq_restore(flags);
return 0;
}
+EXPORT_SYMBOL(__mcfgpio_direction_input);
-int mcf_gpio_get_value(struct gpio_chip *chip, unsigned offset)
-{
- struct mcf_gpio_chip *mcf_chip = MCF_CHIP(chip);
-
- return mcfgpio_read(mcf_chip->ppdr) & mcfgpio_bit(chip->base + offset);
-}
-
-int mcf_gpio_direction_output(struct gpio_chip *chip, unsigned offset,
- int value)
+int __mcfgpio_direction_output(unsigned gpio, int value)
{
unsigned long flags;
MCFGPIO_PORTTYPE data;
- struct mcf_gpio_chip *mcf_chip = MCF_CHIP(chip);
local_irq_save(flags);
- /* write the value to the output latch */
- data = mcfgpio_read(mcf_chip->podr);
+ data = mcfgpio_read(__mcfgpio_pddr(gpio));
if (value)
- data |= mcfgpio_bit(chip->base + offset);
+ data |= mcfgpio_bit(gpio);
else
- data &= ~mcfgpio_bit(chip->base + offset);
- mcfgpio_write(data, mcf_chip->podr);
-
- /* now set the direction to output */
- data = mcfgpio_read(mcf_chip->pddr);
- data |= mcfgpio_bit(chip->base + offset);
- mcfgpio_write(data, mcf_chip->pddr);
+ data &= mcfgpio_bit(gpio);
+ mcfgpio_write(data, __mcfgpio_pddr(gpio));
+
+ /* now set the data to output */
+ if (gpio < MCFGPIO_SCR_START) {
+ data = mcfgpio_read(__mcfgpio_podr(gpio));
+ if (value)
+ data |= mcfgpio_bit(gpio);
+ else
+ data &= ~mcfgpio_bit(gpio);
+ mcfgpio_write(data, __mcfgpio_podr(gpio));
+ } else {
+ if (value)
+ mcfgpio_write(mcfgpio_bit(gpio),
+ MCFGPIO_SETR_PORT(gpio));
+ else
+ mcfgpio_write(~mcfgpio_bit(gpio),
+ MCFGPIO_CLRR_PORT(gpio));
+ }
local_irq_restore(flags);
+ return 0;
+}
+EXPORT_SYMBOL(__mcfgpio_direction_output);
+int __mcfgpio_request(unsigned gpio)
+{
return 0;
}
+EXPORT_SYMBOL(__mcfgpio_request);
-void mcf_gpio_set_value(struct gpio_chip *chip, unsigned offset, int value)
+void __mcfgpio_free(unsigned gpio)
{
- struct mcf_gpio_chip *mcf_chip = MCF_CHIP(chip);
+ __mcfgpio_direction_input(gpio);
+}
+EXPORT_SYMBOL(__mcfgpio_free);
- unsigned long flags;
- MCFGPIO_PORTTYPE data;
+#ifdef CONFIG_GPIOLIB
- local_irq_save(flags);
- data = mcfgpio_read(mcf_chip->podr);
- if (value)
- data |= mcfgpio_bit(chip->base + offset);
- else
- data &= ~mcfgpio_bit(chip->base + offset);
- mcfgpio_write(data, mcf_chip->podr);
- local_irq_restore(flags);
+int mcfgpio_direction_input(struct gpio_chip *chip, unsigned offset)
+{
+ return __mcfgpio_direction_input(offset);
}
-void mcf_gpio_set_value_fast(struct gpio_chip *chip, unsigned offset, int value)
+int mcfgpio_get_value(struct gpio_chip *chip, unsigned offset)
{
- struct mcf_gpio_chip *mcf_chip = MCF_CHIP(chip);
-
- if (value)
- mcfgpio_write(mcfgpio_bit(chip->base + offset), mcf_chip->setr);
- else
- mcfgpio_write(~mcfgpio_bit(chip->base + offset), mcf_chip->clrr);
+ return __mcfgpio_get_value(offset);
}
-int mcf_gpio_request(struct gpio_chip *chip, unsigned offset)
+int mcfgpio_direction_output(struct gpio_chip *chip, unsigned offset, int value)
{
- struct mcf_gpio_chip *mcf_chip = MCF_CHIP(chip);
-
- return mcf_chip->gpio_to_pinmux ?
- mcf_pinmux_request(mcf_chip->gpio_to_pinmux[offset], 0) : 0;
+ return __mcfgpio_direction_output(offset, value);
}
-void mcf_gpio_free(struct gpio_chip *chip, unsigned offset)
+void mcfgpio_set_value(struct gpio_chip *chip, unsigned offset, int value)
{
- struct mcf_gpio_chip *mcf_chip = MCF_CHIP(chip);
+ __mcfgpio_set_value(offset, value);
+}
- mcf_gpio_direction_input(chip, offset);
+int mcfgpio_request(struct gpio_chip *chip, unsigned offset)
+{
+ return __mcfgpio_request(offset);
+}
- if (mcf_chip->gpio_to_pinmux)
- mcf_pinmux_release(mcf_chip->gpio_to_pinmux[offset], 0);
+void mcfgpio_free(struct gpio_chip *chip, unsigned offset)
+{
+ __mcfgpio_free(offset);
}
-struct bus_type mcf_gpio_subsys = {
+struct bus_type mcfgpio_subsys = {
.name = "gpio",
.dev_name = "gpio",
};
-static int __init mcf_gpio_sysinit(void)
-{
- unsigned int i = 0;
+static struct gpio_chip mcfgpio_chip = {
+ .label = "mcfgpio",
+ .request = mcfgpio_request,
+ .free = mcfgpio_free,
+ .direction_input = mcfgpio_direction_input,
+ .direction_output = mcfgpio_direction_output,
+ .get = mcfgpio_get_value,
+ .set = mcfgpio_set_value,
+ .base = 0,
+ .ngpio = MCFGPIO_PIN_MAX,
+};
- while (i < mcf_gpio_chips_size)
- gpiochip_add((struct gpio_chip *)&mcf_gpio_chips[i++]);
- return subsys_system_register(&mcf_gpio_subsys, NULL);
+static int __init mcfgpio_sysinit(void)
+{
+ gpiochip_add(&mcfgpio_chip);
+ return subsys_system_register(&mcfgpio_subsys, NULL);
}
-core_initcall(mcf_gpio_sysinit);
+core_initcall(mcfgpio_sysinit);
+#endif
diff --git a/arch/m68k/platform/coldfire/head.S b/arch/m68k/platform/coldfire/head.S
index c3db70ed33b3..4e0c9eb3bd1f 100644
--- a/arch/m68k/platform/coldfire/head.S
+++ b/arch/m68k/platform/coldfire/head.S
@@ -31,9 +31,9 @@
.endm
#elif defined(CONFIG_M5206) || defined(CONFIG_M5206e) || \
- defined(CONFIG_M5249) || defined(CONFIG_M527x) || \
- defined(CONFIG_M528x) || defined(CONFIG_M5307) || \
- defined(CONFIG_M5407)
+ defined(CONFIG_M5249) || defined(CONFIG_M525x) || \
+ defined(CONFIG_M527x) || defined(CONFIG_M528x) || \
+ defined(CONFIG_M5307) || defined(CONFIG_M5407)
/*
* Not all these devices have exactly the same DRAM controller,
* but the DCMR register is virtually identical - give or take
diff --git a/arch/m68k/platform/coldfire/intc-525x.c b/arch/m68k/platform/coldfire/intc-525x.c
new file mode 100644
index 000000000000..b23204d059ac
--- /dev/null
+++ b/arch/m68k/platform/coldfire/intc-525x.c
@@ -0,0 +1,91 @@
+/*
+ * intc2.c -- support for the 2nd INTC controller of the 525x
+ *
+ * (C) Copyright 2012, Steven King <sfking@fdwdc.com>
+ * (C) Copyright 2009, Greg Ungerer <gerg@snapgear.com>
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file COPYING in the main directory of this archive
+ * for more details.
+ */
+
+#include <linux/types.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/io.h>
+#include <asm/coldfire.h>
+#include <asm/mcfsim.h>
+
+static void intc2_irq_gpio_mask(struct irq_data *d)
+{
+ u32 imr = readl(MCFSIM2_GPIOINTENABLE);
+ u32 type = irqd_get_trigger_type(d);
+ int irq = d->irq - MCF_IRQ_GPIO0;
+
+ if (type & IRQ_TYPE_EDGE_RISING)
+ imr &= ~(0x001 << irq);
+ if (type & IRQ_TYPE_EDGE_FALLING)
+ imr &= ~(0x100 << irq);
+ writel(imr, MCFSIM2_GPIOINTENABLE);
+}
+
+static void intc2_irq_gpio_unmask(struct irq_data *d)
+{
+ u32 imr = readl(MCFSIM2_GPIOINTENABLE);
+ u32 type = irqd_get_trigger_type(d);
+ int irq = d->irq - MCF_IRQ_GPIO0;
+
+ if (type & IRQ_TYPE_EDGE_RISING)
+ imr |= (0x001 << irq);
+ if (type & IRQ_TYPE_EDGE_FALLING)
+ imr |= (0x100 << irq);
+ writel(imr, MCFSIM2_GPIOINTENABLE);
+}
+
+static void intc2_irq_gpio_ack(struct irq_data *d)
+{
+ u32 imr = 0;
+ u32 type = irqd_get_trigger_type(d);
+ int irq = d->irq - MCF_IRQ_GPIO0;
+
+ if (type & IRQ_TYPE_EDGE_RISING)
+ imr |= (0x001 << irq);
+ if (type & IRQ_TYPE_EDGE_FALLING)
+ imr |= (0x100 << irq);
+ writel(imr, MCFSIM2_GPIOINTCLEAR);
+}
+
+static int intc2_irq_gpio_set_type(struct irq_data *d, unsigned int f)
+{
+ if (f & ~IRQ_TYPE_EDGE_BOTH)
+ return -EINVAL;
+ return 0;
+}
+
+static struct irq_chip intc2_irq_gpio_chip = {
+ .name = "CF-INTC2",
+ .irq_mask = intc2_irq_gpio_mask,
+ .irq_unmask = intc2_irq_gpio_unmask,
+ .irq_ack = intc2_irq_gpio_ack,
+ .irq_set_type = intc2_irq_gpio_set_type,
+};
+
+static int __init mcf_intc2_init(void)
+{
+ int irq;
+
+ /* set the interrupt base for the second interrupt controller */
+ writel(MCFINTC2_VECBASE, MCFINTC2_INTBASE);
+
+ /* GPIO interrupt sources */
+ for (irq = MCF_IRQ_GPIO0; (irq <= MCF_IRQ_GPIO6); irq++) {
+ irq_set_chip(irq, &intc2_irq_gpio_chip);
+ irq_set_handler(irq, handle_edge_irq);
+ }
+
+ return 0;
+}
+
+arch_initcall(mcf_intc2_init);
diff --git a/arch/m68k/platform/coldfire/intc-simr.c b/arch/m68k/platform/coldfire/intc-simr.c
index 650d52e2927e..7cf2c156f72d 100644
--- a/arch/m68k/platform/coldfire/intc-simr.c
+++ b/arch/m68k/platform/coldfire/intc-simr.c
@@ -59,16 +59,18 @@ static unsigned int inline irq2ebit(unsigned int irq)
#endif
/*
- * There maybe one or two interrupt control units, each has 64
- * interrupts. If there is no second unit then MCFINTC1_* defines
- * will be 0 (and code for them optimized away).
+ * There maybe one, two or three interrupt control units, each has 64
+ * interrupts. If there is no second or third unit then MCFINTC1_* or
+ * MCFINTC2_* defines will be 0 (and code for them optimized away).
*/
static void intc_irq_mask(struct irq_data *d)
{
unsigned int irq = d->irq - MCFINT_VECBASE;
- if (MCFINTC1_SIMR && (irq > 64))
+ if (MCFINTC2_SIMR && (irq > 128))
+ __raw_writeb(irq - 128, MCFINTC2_SIMR);
+ else if (MCFINTC1_SIMR && (irq > 64))
__raw_writeb(irq - 64, MCFINTC1_SIMR);
else
__raw_writeb(irq, MCFINTC0_SIMR);
@@ -78,7 +80,9 @@ static void intc_irq_unmask(struct irq_data *d)
{
unsigned int irq = d->irq - MCFINT_VECBASE;
- if (MCFINTC1_CIMR && (irq > 64))
+ if (MCFINTC2_CIMR && (irq > 128))
+ __raw_writeb(irq - 128, MCFINTC2_CIMR);
+ else if (MCFINTC1_CIMR && (irq > 64))
__raw_writeb(irq - 64, MCFINTC1_CIMR);
else
__raw_writeb(irq, MCFINTC0_CIMR);
@@ -99,9 +103,11 @@ static unsigned int intc_irq_startup(struct irq_data *d)
unsigned int ebit = irq2ebit(irq);
u8 v;
+#if defined(MCFEPORT_EPDDR)
/* Set EPORT line as input */
v = __raw_readb(MCFEPORT_EPDDR);
__raw_writeb(v & ~(0x1 << ebit), MCFEPORT_EPDDR);
+#endif
/* Set EPORT line as interrupt source */
v = __raw_readb(MCFEPORT_EPIER);
@@ -109,12 +115,13 @@ static unsigned int intc_irq_startup(struct irq_data *d)
}
irq -= MCFINT_VECBASE;
- if (MCFINTC1_ICR0 && (irq > 64))
+ if (MCFINTC2_ICR0 && (irq > 128))
+ __raw_writeb(5, MCFINTC2_ICR0 + irq - 128);
+ else if (MCFINTC1_ICR0 && (irq > 64))
__raw_writeb(5, MCFINTC1_ICR0 + irq - 64);
else
__raw_writeb(5, MCFINTC0_ICR0 + irq);
-
intc_irq_unmask(d);
return 0;
}
@@ -175,8 +182,11 @@ void __init init_IRQ(void)
__raw_writeb(0xff, MCFINTC0_SIMR);
if (MCFINTC1_SIMR)
__raw_writeb(0xff, MCFINTC1_SIMR);
+ if (MCFINTC2_SIMR)
+ __raw_writeb(0xff, MCFINTC2_SIMR);
- eirq = MCFINT_VECBASE + 64 + (MCFINTC1_ICR0 ? 64 : 0);
+ eirq = MCFINT_VECBASE + 64 + (MCFINTC1_ICR0 ? 64 : 0) +
+ (MCFINTC2_ICR0 ? 64 : 0);
for (irq = MCFINT_VECBASE; (irq < eirq); irq++) {
if ((irq >= EINT1) && (irq <= EINT7))
irq_set_chip(irq, &intc_irq_chip_edge_port);
diff --git a/arch/m68k/platform/coldfire/m5206.c b/arch/m68k/platform/coldfire/m5206.c
index a8b81df653f0..6bfbeebd231b 100644
--- a/arch/m68k/platform/coldfire/m5206.c
+++ b/arch/m68k/platform/coldfire/m5206.c
@@ -16,15 +16,6 @@
#include <asm/machdep.h>
#include <asm/coldfire.h>
#include <asm/mcfsim.h>
-#include <asm/mcfgpio.h>
-
-/***************************************************************************/
-
-struct mcf_gpio_chip mcf_gpio_chips[] = {
- MCFGPS(PP, 0, 8, MCFSIM_PADDR, MCFSIM_PADAT, MCFSIM_PADAT),
-};
-
-unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips);
/***************************************************************************/
diff --git a/arch/m68k/platform/coldfire/m520x.c b/arch/m68k/platform/coldfire/m520x.c
index 3264b8883d5f..ea1be0e98ad6 100644
--- a/arch/m68k/platform/coldfire/m520x.c
+++ b/arch/m68k/platform/coldfire/m520x.c
@@ -19,22 +19,102 @@
#include <asm/coldfire.h>
#include <asm/mcfsim.h>
#include <asm/mcfuart.h>
-#include <asm/mcfgpio.h>
+#include <asm/mcfclk.h>
/***************************************************************************/
-struct mcf_gpio_chip mcf_gpio_chips[] = {
- MCFGPS(PIRQ, 0, 8, MCFEPORT_EPDDR, MCFEPORT_EPDR, MCFEPORT_EPPDR),
- MCFGPF(CS, 9, 3),
- MCFGPF(FECI2C, 16, 4),
- MCFGPF(QSPI, 24, 4),
- MCFGPF(TIMER, 32, 4),
- MCFGPF(UART, 40, 8),
- MCFGPF(FECH, 48, 8),
- MCFGPF(FECL, 56, 8),
+DEFINE_CLK(0, "flexbus", 2, MCF_CLK);
+DEFINE_CLK(0, "fec.0", 12, MCF_CLK);
+DEFINE_CLK(0, "edma", 17, MCF_CLK);
+DEFINE_CLK(0, "intc.0", 18, MCF_CLK);
+DEFINE_CLK(0, "iack.0", 21, MCF_CLK);
+DEFINE_CLK(0, "mcfi2c.0", 22, MCF_CLK);
+DEFINE_CLK(0, "mcfqspi.0", 23, MCF_CLK);
+DEFINE_CLK(0, "mcfuart.0", 24, MCF_BUSCLK);
+DEFINE_CLK(0, "mcfuart.1", 25, MCF_BUSCLK);
+DEFINE_CLK(0, "mcfuart.2", 26, MCF_BUSCLK);
+DEFINE_CLK(0, "mcftmr.0", 28, MCF_CLK);
+DEFINE_CLK(0, "mcftmr.1", 29, MCF_CLK);
+DEFINE_CLK(0, "mcftmr.2", 30, MCF_CLK);
+DEFINE_CLK(0, "mcftmr.3", 31, MCF_CLK);
+
+DEFINE_CLK(0, "mcfpit.0", 32, MCF_CLK);
+DEFINE_CLK(0, "mcfpit.1", 33, MCF_CLK);
+DEFINE_CLK(0, "mcfeport.0", 34, MCF_CLK);
+DEFINE_CLK(0, "mcfwdt.0", 35, MCF_CLK);
+DEFINE_CLK(0, "pll.0", 36, MCF_CLK);
+DEFINE_CLK(0, "sys.0", 40, MCF_BUSCLK);
+DEFINE_CLK(0, "gpio.0", 41, MCF_BUSCLK);
+DEFINE_CLK(0, "sdram.0", 42, MCF_CLK);
+
+struct clk *mcf_clks[] = {
+ &__clk_0_2, /* flexbus */
+ &__clk_0_12, /* fec.0 */
+ &__clk_0_17, /* edma */
+ &__clk_0_18, /* intc.0 */
+ &__clk_0_21, /* iack.0 */
+ &__clk_0_22, /* mcfi2c.0 */
+ &__clk_0_23, /* mcfqspi.0 */
+ &__clk_0_24, /* mcfuart.0 */
+ &__clk_0_25, /* mcfuart.1 */
+ &__clk_0_26, /* mcfuart.2 */
+ &__clk_0_28, /* mcftmr.0 */
+ &__clk_0_29, /* mcftmr.1 */
+ &__clk_0_30, /* mcftmr.2 */
+ &__clk_0_31, /* mcftmr.3 */
+
+ &__clk_0_32, /* mcfpit.0 */
+ &__clk_0_33, /* mcfpit.1 */
+ &__clk_0_34, /* mcfeport.0 */
+ &__clk_0_35, /* mcfwdt.0 */
+ &__clk_0_36, /* pll.0 */
+ &__clk_0_40, /* sys.0 */
+ &__clk_0_41, /* gpio.0 */
+ &__clk_0_42, /* sdram.0 */
+NULL,
};
-unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips);
+static struct clk * const enable_clks[] __initconst = {
+ &__clk_0_2, /* flexbus */
+ &__clk_0_18, /* intc.0 */
+ &__clk_0_21, /* iack.0 */
+ &__clk_0_24, /* mcfuart.0 */
+ &__clk_0_25, /* mcfuart.1 */
+ &__clk_0_26, /* mcfuart.2 */
+
+ &__clk_0_32, /* mcfpit.0 */
+ &__clk_0_33, /* mcfpit.1 */
+ &__clk_0_34, /* mcfeport.0 */
+ &__clk_0_36, /* pll.0 */
+ &__clk_0_40, /* sys.0 */
+ &__clk_0_41, /* gpio.0 */
+ &__clk_0_42, /* sdram.0 */
+};
+
+static struct clk * const disable_clks[] __initconst = {
+ &__clk_0_12, /* fec.0 */
+ &__clk_0_17, /* edma */
+ &__clk_0_22, /* mcfi2c.0 */
+ &__clk_0_23, /* mcfqspi.0 */
+ &__clk_0_28, /* mcftmr.0 */
+ &__clk_0_29, /* mcftmr.1 */
+ &__clk_0_30, /* mcftmr.2 */
+ &__clk_0_31, /* mcftmr.3 */
+ &__clk_0_35, /* mcfwdt.0 */
+};
+
+
+static void __init m520x_clk_init(void)
+{
+ unsigned i;
+
+ /* make sure these clocks are enabled */
+ for (i = 0; i < ARRAY_SIZE(enable_clks); ++i)
+ __clk_init_enabled(enable_clks[i]);
+ /* make sure these clocks are disabled */
+ for (i = 0; i < ARRAY_SIZE(disable_clks); ++i)
+ __clk_init_disabled(disable_clks[i]);
+}
/***************************************************************************/
@@ -93,6 +173,7 @@ static void __init m520x_fec_init(void)
void __init config_BSP(char *commandp, int size)
{
mach_sched_init = hw_timer_init;
+ m520x_clk_init();
m520x_uarts_init();
m520x_fec_init();
#if IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI)
diff --git a/arch/m68k/platform/coldfire/m523x.c b/arch/m68k/platform/coldfire/m523x.c
index 5d57a4249412..d47dfd8f50a2 100644
--- a/arch/m68k/platform/coldfire/m523x.c
+++ b/arch/m68k/platform/coldfire/m523x.c
@@ -19,28 +19,6 @@
#include <asm/machdep.h>
#include <asm/coldfire.h>
#include <asm/mcfsim.h>
-#include <asm/mcfgpio.h>
-
-/***************************************************************************/
-
-struct mcf_gpio_chip mcf_gpio_chips[] = {
- MCFGPS(PIRQ, 1, 7, MCFEPORT_EPDDR, MCFEPORT_EPDR, MCFEPORT_EPPDR),
- MCFGPF(ADDR, 13, 3),
- MCFGPF(DATAH, 16, 8),
- MCFGPF(DATAL, 24, 8),
- MCFGPF(BUSCTL, 32, 8),
- MCFGPF(BS, 40, 4),
- MCFGPF(CS, 49, 7),
- MCFGPF(SDRAM, 56, 6),
- MCFGPF(FECI2C, 64, 4),
- MCFGPF(UARTH, 72, 2),
- MCFGPF(UARTL, 80, 8),
- MCFGPF(QSPI, 88, 5),
- MCFGPF(TIMER, 96, 8),
- MCFGPF(ETPU, 104, 3),
-};
-
-unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips);
/***************************************************************************/
diff --git a/arch/m68k/platform/coldfire/m5249.c b/arch/m68k/platform/coldfire/m5249.c
index fdfa1edfd1ac..300e729a58d0 100644
--- a/arch/m68k/platform/coldfire/m5249.c
+++ b/arch/m68k/platform/coldfire/m5249.c
@@ -16,16 +16,6 @@
#include <asm/machdep.h>
#include <asm/coldfire.h>
#include <asm/mcfsim.h>
-#include <asm/mcfgpio.h>
-
-/***************************************************************************/
-
-struct mcf_gpio_chip mcf_gpio_chips[] = {
- MCFGPS(GPIO0, 0, 32, MCFSIM2_GPIOENABLE, MCFSIM2_GPIOWRITE, MCFSIM2_GPIOREAD),
- MCFGPS(GPIO1, 32, 32, MCFSIM2_GPIO1ENABLE, MCFSIM2_GPIO1WRITE, MCFSIM2_GPIO1READ),
-};
-
-unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips);
/***************************************************************************/
diff --git a/arch/m68k/platform/coldfire/m525x.c b/arch/m68k/platform/coldfire/m525x.c
new file mode 100644
index 000000000000..8ce905f9b84f
--- /dev/null
+++ b/arch/m68k/platform/coldfire/m525x.c
@@ -0,0 +1,66 @@
+/***************************************************************************/
+
+/*
+ * 525x.c
+ *
+ * Copyright (C) 2012, Steven King <sfking@fdwdc.com>
+ */
+
+/***************************************************************************/
+
+#include <linux/kernel.h>
+#include <linux/param.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/platform_device.h>
+#include <asm/machdep.h>
+#include <asm/coldfire.h>
+#include <asm/mcfsim.h>
+
+/***************************************************************************/
+
+static void __init m525x_qspi_init(void)
+{
+#if IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI)
+ /* set the GPIO function for the qspi cs gpios */
+ /* FIXME: replace with pinmux/pinctl support */
+ u32 f = readl(MCFSIM2_GPIOFUNC);
+ f |= (1 << MCFQSPI_CS2) | (1 << MCFQSPI_CS1) | (1 << MCFQSPI_CS0);
+ writel(f, MCFSIM2_GPIOFUNC);
+
+ /* QSPI irq setup */
+ writeb(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL4 | MCFSIM_ICR_PRI0,
+ MCF_MBAR + MCFSIM_QSPIICR);
+ mcf_mapirq2imr(MCF_IRQ_QSPI, MCFINTC_QSPI);
+#endif /* IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI) */
+}
+
+static void __init m525x_i2c_init(void)
+{
+#if IS_ENABLED(CONFIG_I2C_COLDFIRE)
+ u32 r;
+
+ /* first I2C controller uses regular irq setup */
+ writeb(MCFSIM_ICR_AUTOVEC | MCFSIM_ICR_LEVEL5 | MCFSIM_ICR_PRI0,
+ MCF_MBAR + MCFSIM_I2CICR);
+ mcf_mapirq2imr(MCF_IRQ_I2C0, MCFINTC_I2C);
+
+ /* second I2C controller is completely different */
+ r = readl(MCFINTC2_INTPRI_REG(MCF_IRQ_I2C1));
+ r &= ~MCFINTC2_INTPRI_BITS(0xf, MCF_IRQ_I2C1);
+ r |= MCFINTC2_INTPRI_BITS(0x5, MCF_IRQ_I2C1);
+ writel(r, MCFINTC2_INTPRI_REG(MCF_IRQ_I2C1));
+#endif /* IS_ENABLED(CONFIG_I2C_COLDFIRE) */
+}
+
+/***************************************************************************/
+
+void __init config_BSP(char *commandp, int size)
+{
+ mach_sched_init = hw_timer_init;
+
+ m525x_qspi_init();
+ m525x_i2c_init();
+}
+
+/***************************************************************************/
diff --git a/arch/m68k/platform/coldfire/m5272.c b/arch/m68k/platform/coldfire/m5272.c
index 43e36060da18..e68bc7a148eb 100644
--- a/arch/m68k/platform/coldfire/m5272.c
+++ b/arch/m68k/platform/coldfire/m5272.c
@@ -19,7 +19,6 @@
#include <asm/coldfire.h>
#include <asm/mcfsim.h>
#include <asm/mcfuart.h>
-#include <asm/mcfgpio.h>
/***************************************************************************/
@@ -31,16 +30,6 @@ unsigned char ledbank = 0xff;
/***************************************************************************/
-struct mcf_gpio_chip mcf_gpio_chips[] = {
- MCFGPS(PA, 0, 16, MCFSIM_PADDR, MCFSIM_PADAT, MCFSIM_PADAT),
- MCFGPS(PB, 16, 16, MCFSIM_PBDDR, MCFSIM_PBDAT, MCFSIM_PBDAT),
- MCFGPS(Pc, 32, 16, MCFSIM_PCDDR, MCFSIM_PCDAT, MCFSIM_PCDAT),
-};
-
-unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips);
-
-/***************************************************************************/
-
static void __init m5272_uarts_init(void)
{
u32 v;
diff --git a/arch/m68k/platform/coldfire/m527x.c b/arch/m68k/platform/coldfire/m527x.c
index 9b0b66aabd1b..b3cb378c5e94 100644
--- a/arch/m68k/platform/coldfire/m527x.c
+++ b/arch/m68k/platform/coldfire/m527x.c
@@ -20,49 +20,6 @@
#include <asm/coldfire.h>
#include <asm/mcfsim.h>
#include <asm/mcfuart.h>
-#include <asm/mcfgpio.h>
-
-/***************************************************************************/
-
-struct mcf_gpio_chip mcf_gpio_chips[] = {
-#if defined(CONFIG_M5271)
- MCFGPS(PIRQ, 1, 7, MCFEPORT_EPDDR, MCFEPORT_EPDR, MCFEPORT_EPPDR),
- MCFGPF(ADDR, 13, 3),
- MCFGPF(DATAH, 16, 8),
- MCFGPF(DATAL, 24, 8),
- MCFGPF(BUSCTL, 32, 8),
- MCFGPF(BS, 40, 4),
- MCFGPF(CS, 49, 7),
- MCFGPF(SDRAM, 56, 6),
- MCFGPF(FECI2C, 64, 4),
- MCFGPF(UARTH, 72, 2),
- MCFGPF(UARTL, 80, 8),
- MCFGPF(QSPI, 88, 5),
- MCFGPF(TIMER, 96, 8),
-#elif defined(CONFIG_M5275)
- MCFGPS(PIRQ, 1, 7, MCFEPORT_EPDDR, MCFEPORT_EPDR, MCFEPORT_EPPDR),
- MCFGPF(BUSCTL, 8, 8),
- MCFGPF(ADDR, 21, 3),
- MCFGPF(CS, 25, 7),
- MCFGPF(FEC0H, 32, 8),
- MCFGPF(FEC0L, 40, 8),
- MCFGPF(FECI2C, 48, 6),
- MCFGPF(QSPI, 56, 7),
- MCFGPF(SDRAM, 64, 8),
- MCFGPF(TIMERH, 72, 4),
- MCFGPF(TIMERL, 80, 4),
- MCFGPF(UARTL, 88, 8),
- MCFGPF(FEC1H, 96, 8),
- MCFGPF(FEC1L, 104, 8),
- MCFGPF(BS, 114, 2),
- MCFGPF(IRQ, 121, 7),
- MCFGPF(USBH, 128, 1),
- MCFGPF(USBL, 136, 8),
- MCFGPF(UARTH, 144, 4),
-#endif
-};
-
-unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips);
/***************************************************************************/
diff --git a/arch/m68k/platform/coldfire/m528x.c b/arch/m68k/platform/coldfire/m528x.c
index 7ed1276b29dc..f1319e5d2546 100644
--- a/arch/m68k/platform/coldfire/m528x.c
+++ b/arch/m68k/platform/coldfire/m528x.c
@@ -21,37 +21,6 @@
#include <asm/coldfire.h>
#include <asm/mcfsim.h>
#include <asm/mcfuart.h>
-#include <asm/mcfgpio.h>
-
-/***************************************************************************/
-
-struct mcf_gpio_chip mcf_gpio_chips[] = {
- MCFGPS(NQ, 1, 7, MCFEPORT_EPDDR, MCFEPORT_EPDR, MCFEPORT_EPPDR),
- MCFGPS(TA, 8, 4, MCFGPTA_GPTDDR, MCFGPTA_GPTPORT, MCFGPTB_GPTPORT),
- MCFGPS(TB, 16, 4, MCFGPTB_GPTDDR, MCFGPTB_GPTPORT, MCFGPTB_GPTPORT),
- MCFGPS(QA, 24, 4, MCFQADC_DDRQA, MCFQADC_PORTQA, MCFQADC_PORTQA),
- MCFGPS(QB, 32, 4, MCFQADC_DDRQB, MCFQADC_PORTQB, MCFQADC_PORTQB),
- MCFGPF(A, 40, 8),
- MCFGPF(B, 48, 8),
- MCFGPF(C, 56, 8),
- MCFGPF(D, 64, 8),
- MCFGPF(E, 72, 8),
- MCFGPF(F, 80, 8),
- MCFGPF(G, 88, 8),
- MCFGPF(H, 96, 8),
- MCFGPF(J, 104, 8),
- MCFGPF(DD, 112, 8),
- MCFGPF(EH, 120, 8),
- MCFGPF(EL, 128, 8),
- MCFGPF(AS, 136, 6),
- MCFGPF(QS, 144, 7),
- MCFGPF(SD, 152, 6),
- MCFGPF(TC, 160, 4),
- MCFGPF(TD, 168, 4),
- MCFGPF(UA, 176, 4),
-};
-
-unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips);
/***************************************************************************/
@@ -74,7 +43,7 @@ static void __init m528x_uarts_init(void)
/* make sure PUAPAR is set for UART0 and UART1 */
port = readb(MCF5282_GPIO_PUAPAR);
port |= 0x03 | (0x03 << 2);
- writeb(port, MCF5282_GPIO_PUAPAR);
+ writeb(port, MCFGPIO_PUAPAR);
}
/***************************************************************************/
diff --git a/arch/m68k/platform/coldfire/m5307.c b/arch/m68k/platform/coldfire/m5307.c
index 93b484976ab3..a568d2870d15 100644
--- a/arch/m68k/platform/coldfire/m5307.c
+++ b/arch/m68k/platform/coldfire/m5307.c
@@ -16,7 +16,6 @@
#include <asm/machdep.h>
#include <asm/coldfire.h>
#include <asm/mcfsim.h>
-#include <asm/mcfgpio.h>
#include <asm/mcfwdebug.h>
/***************************************************************************/
@@ -29,14 +28,6 @@ unsigned char ledbank = 0xff;
/***************************************************************************/
-struct mcf_gpio_chip mcf_gpio_chips[] = {
- MCFGPS(PP, 0, 16, MCFSIM_PADDR, MCFSIM_PADAT, MCFSIM_PADAT),
-};
-
-unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips);
-
-/***************************************************************************/
-
void __init config_BSP(char *commandp, int size)
{
#if defined(CONFIG_NETtel) || \
diff --git a/arch/m68k/platform/coldfire/m532x.c b/arch/m68k/platform/coldfire/m532x.c
index 5394223639f8..4819a44991ed 100644
--- a/arch/m68k/platform/coldfire/m532x.c
+++ b/arch/m68k/platform/coldfire/m532x.c
@@ -26,32 +26,144 @@
#include <asm/mcfsim.h>
#include <asm/mcfuart.h>
#include <asm/mcfdma.h>
-#include <asm/mcfgpio.h>
#include <asm/mcfwdebug.h>
+#include <asm/mcfclk.h>
/***************************************************************************/
-struct mcf_gpio_chip mcf_gpio_chips[] = {
- MCFGPS(PIRQ, 0, 8, MCFEPORT_EPDDR, MCFEPORT_EPDR, MCFEPORT_EPPDR),
- MCFGPF(FECH, 8, 8),
- MCFGPF(FECL, 16, 8),
- MCFGPF(SSI, 24, 5),
- MCFGPF(BUSCTL, 32, 4),
- MCFGPF(BE, 40, 4),
- MCFGPF(CS, 49, 5),
- MCFGPF(PWM, 58, 4),
- MCFGPF(FECI2C, 64, 4),
- MCFGPF(UART, 72, 8),
- MCFGPF(QSPI, 80, 6),
- MCFGPF(TIMER, 88, 4),
- MCFGPF(LCDDATAH, 96, 2),
- MCFGPF(LCDDATAM, 104, 8),
- MCFGPF(LCDDATAL, 112, 8),
- MCFGPF(LCDCTLH, 120, 1),
- MCFGPF(LCDCTLL, 128, 8),
+DEFINE_CLK(0, "flexbus", 2, MCF_CLK);
+DEFINE_CLK(0, "mcfcan.0", 8, MCF_CLK);
+DEFINE_CLK(0, "fec.0", 12, MCF_CLK);
+DEFINE_CLK(0, "edma", 17, MCF_CLK);
+DEFINE_CLK(0, "intc.0", 18, MCF_CLK);
+DEFINE_CLK(0, "intc.1", 19, MCF_CLK);
+DEFINE_CLK(0, "iack.0", 21, MCF_CLK);
+DEFINE_CLK(0, "mcfi2c.0", 22, MCF_CLK);
+DEFINE_CLK(0, "mcfqspi.0", 23, MCF_CLK);
+DEFINE_CLK(0, "mcfuart.0", 24, MCF_BUSCLK);
+DEFINE_CLK(0, "mcfuart.1", 25, MCF_BUSCLK);
+DEFINE_CLK(0, "mcfuart.2", 26, MCF_BUSCLK);
+DEFINE_CLK(0, "mcftmr.0", 28, MCF_CLK);
+DEFINE_CLK(0, "mcftmr.1", 29, MCF_CLK);
+DEFINE_CLK(0, "mcftmr.2", 30, MCF_CLK);
+DEFINE_CLK(0, "mcftmr.3", 31, MCF_CLK);
+
+DEFINE_CLK(0, "mcfpit.0", 32, MCF_CLK);
+DEFINE_CLK(0, "mcfpit.1", 33, MCF_CLK);
+DEFINE_CLK(0, "mcfpit.2", 34, MCF_CLK);
+DEFINE_CLK(0, "mcfpit.3", 35, MCF_CLK);
+DEFINE_CLK(0, "mcfpwm.0", 36, MCF_CLK);
+DEFINE_CLK(0, "mcfeport.0", 37, MCF_CLK);
+DEFINE_CLK(0, "mcfwdt.0", 38, MCF_CLK);
+DEFINE_CLK(0, "sys.0", 40, MCF_BUSCLK);
+DEFINE_CLK(0, "gpio.0", 41, MCF_BUSCLK);
+DEFINE_CLK(0, "mcfrtc.0", 42, MCF_CLK);
+DEFINE_CLK(0, "mcflcd.0", 43, MCF_CLK);
+DEFINE_CLK(0, "mcfusb-otg.0", 44, MCF_CLK);
+DEFINE_CLK(0, "mcfusb-host.0", 45, MCF_CLK);
+DEFINE_CLK(0, "sdram.0", 46, MCF_CLK);
+DEFINE_CLK(0, "ssi.0", 47, MCF_CLK);
+DEFINE_CLK(0, "pll.0", 48, MCF_CLK);
+
+DEFINE_CLK(1, "mdha.0", 32, MCF_CLK);
+DEFINE_CLK(1, "skha.0", 33, MCF_CLK);
+DEFINE_CLK(1, "rng.0", 34, MCF_CLK);
+
+struct clk *mcf_clks[] = {
+ &__clk_0_2, /* flexbus */
+ &__clk_0_8, /* mcfcan.0 */
+ &__clk_0_12, /* fec.0 */
+ &__clk_0_17, /* edma */
+ &__clk_0_18, /* intc.0 */
+ &__clk_0_19, /* intc.1 */
+ &__clk_0_21, /* iack.0 */
+ &__clk_0_22, /* mcfi2c.0 */
+ &__clk_0_23, /* mcfqspi.0 */
+ &__clk_0_24, /* mcfuart.0 */
+ &__clk_0_25, /* mcfuart.1 */
+ &__clk_0_26, /* mcfuart.2 */
+ &__clk_0_28, /* mcftmr.0 */
+ &__clk_0_29, /* mcftmr.1 */
+ &__clk_0_30, /* mcftmr.2 */
+ &__clk_0_31, /* mcftmr.3 */
+
+ &__clk_0_32, /* mcfpit.0 */
+ &__clk_0_33, /* mcfpit.1 */
+ &__clk_0_34, /* mcfpit.2 */
+ &__clk_0_35, /* mcfpit.3 */
+ &__clk_0_36, /* mcfpwm.0 */
+ &__clk_0_37, /* mcfeport.0 */
+ &__clk_0_38, /* mcfwdt.0 */
+ &__clk_0_40, /* sys.0 */
+ &__clk_0_41, /* gpio.0 */
+ &__clk_0_42, /* mcfrtc.0 */
+ &__clk_0_43, /* mcflcd.0 */
+ &__clk_0_44, /* mcfusb-otg.0 */
+ &__clk_0_45, /* mcfusb-host.0 */
+ &__clk_0_46, /* sdram.0 */
+ &__clk_0_47, /* ssi.0 */
+ &__clk_0_48, /* pll.0 */
+
+ &__clk_1_32, /* mdha.0 */
+ &__clk_1_33, /* skha.0 */
+ &__clk_1_34, /* rng.0 */
+ NULL,
+};
+
+static struct clk * const enable_clks[] __initconst = {
+ &__clk_0_2, /* flexbus */
+ &__clk_0_18, /* intc.0 */
+ &__clk_0_19, /* intc.1 */
+ &__clk_0_21, /* iack.0 */
+ &__clk_0_24, /* mcfuart.0 */
+ &__clk_0_25, /* mcfuart.1 */
+ &__clk_0_26, /* mcfuart.2 */
+
+ &__clk_0_32, /* mcfpit.0 */
+ &__clk_0_33, /* mcfpit.1 */
+ &__clk_0_37, /* mcfeport.0 */
+ &__clk_0_40, /* sys.0 */
+ &__clk_0_41, /* gpio.0 */
+ &__clk_0_46, /* sdram.0 */
+ &__clk_0_48, /* pll.0 */
+};
+
+static struct clk * const disable_clks[] __initconst = {
+ &__clk_0_8, /* mcfcan.0 */
+ &__clk_0_12, /* fec.0 */
+ &__clk_0_17, /* edma */
+ &__clk_0_22, /* mcfi2c.0 */
+ &__clk_0_23, /* mcfqspi.0 */
+ &__clk_0_28, /* mcftmr.0 */
+ &__clk_0_29, /* mcftmr.1 */
+ &__clk_0_30, /* mcftmr.2 */
+ &__clk_0_31, /* mcftmr.3 */
+ &__clk_0_34, /* mcfpit.2 */
+ &__clk_0_35, /* mcfpit.3 */
+ &__clk_0_36, /* mcfpwm.0 */
+ &__clk_0_38, /* mcfwdt.0 */
+ &__clk_0_42, /* mcfrtc.0 */
+ &__clk_0_43, /* mcflcd.0 */
+ &__clk_0_44, /* mcfusb-otg.0 */
+ &__clk_0_45, /* mcfusb-host.0 */
+ &__clk_0_47, /* ssi.0 */
+ &__clk_1_32, /* mdha.0 */
+ &__clk_1_33, /* skha.0 */
+ &__clk_1_34, /* rng.0 */
};
-unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips);
+
+static void __init m532x_clk_init(void)
+{
+ unsigned i;
+
+ /* make sure these clocks are enabled */
+ for (i = 0; i < ARRAY_SIZE(enable_clks); ++i)
+ __clk_init_enabled(enable_clks[i]);
+ /* make sure these clocks are disabled */
+ for (i = 0; i < ARRAY_SIZE(disable_clks); ++i)
+ __clk_init_disabled(disable_clks[i]);
+}
/***************************************************************************/
@@ -98,8 +210,8 @@ void __init config_BSP(char *commandp, int size)
memset(commandp, 0, size);
}
#endif
-
mach_sched_init = hw_timer_init;
+ m532x_clk_init();
m532x_uarts_init();
m532x_fec_init();
#if IS_ENABLED(CONFIG_SPI_COLDFIRE_QSPI)
diff --git a/arch/m68k/platform/coldfire/m5407.c b/arch/m68k/platform/coldfire/m5407.c
index faa6680b3404..bb6c746ae819 100644
--- a/arch/m68k/platform/coldfire/m5407.c
+++ b/arch/m68k/platform/coldfire/m5407.c
@@ -16,15 +16,6 @@
#include <asm/machdep.h>
#include <asm/coldfire.h>
#include <asm/mcfsim.h>
-#include <asm/mcfgpio.h>
-
-/***************************************************************************/
-
-struct mcf_gpio_chip mcf_gpio_chips[] = {
- MCFGPS(PP, 0, 16, MCFSIM_PADDR, MCFSIM_PADAT, MCFSIM_PADAT),
-};
-
-unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips);
/***************************************************************************/
diff --git a/arch/m68k/platform/coldfire/m5441x.c b/arch/m68k/platform/coldfire/m5441x.c
new file mode 100644
index 000000000000..98a13cce93d8
--- /dev/null
+++ b/arch/m68k/platform/coldfire/m5441x.c
@@ -0,0 +1,261 @@
+/*
+ * m5441x.c -- support for Coldfire m5441x processors
+ *
+ * (C) Copyright Steven King <sfking@fdwdc.com>
+ */
+
+#include <linux/kernel.h>
+#include <linux/param.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/clk.h>
+#include <asm/machdep.h>
+#include <asm/coldfire.h>
+#include <asm/mcfsim.h>
+#include <asm/mcfuart.h>
+#include <asm/mcfdma.h>
+#include <asm/mcfclk.h>
+
+DEFINE_CLK(0, "flexbus", 2, MCF_CLK);
+DEFINE_CLK(0, "mcfcan.0", 8, MCF_CLK);
+DEFINE_CLK(0, "mcfcan.1", 9, MCF_CLK);
+DEFINE_CLK(0, "mcfi2c.1", 14, MCF_CLK);
+DEFINE_CLK(0, "mcfdspi.1", 15, MCF_CLK);
+DEFINE_CLK(0, "edma", 17, MCF_CLK);
+DEFINE_CLK(0, "intc.0", 18, MCF_CLK);
+DEFINE_CLK(0, "intc.1", 19, MCF_CLK);
+DEFINE_CLK(0, "intc.2", 20, MCF_CLK);
+DEFINE_CLK(0, "mcfi2c.0", 22, MCF_CLK);
+DEFINE_CLK(0, "mcfdspi.0", 23, MCF_CLK);
+DEFINE_CLK(0, "mcfuart.0", 24, MCF_BUSCLK);
+DEFINE_CLK(0, "mcfuart.1", 25, MCF_BUSCLK);
+DEFINE_CLK(0, "mcfuart.2", 26, MCF_BUSCLK);
+DEFINE_CLK(0, "mcfuart.3", 27, MCF_BUSCLK);
+DEFINE_CLK(0, "mcftmr.0", 28, MCF_CLK);
+DEFINE_CLK(0, "mcftmr.1", 29, MCF_CLK);
+DEFINE_CLK(0, "mcftmr.2", 30, MCF_CLK);
+DEFINE_CLK(0, "mcftmr.3", 31, MCF_CLK);
+DEFINE_CLK(0, "mcfpit.0", 32, MCF_CLK);
+DEFINE_CLK(0, "mcfpit.1", 33, MCF_CLK);
+DEFINE_CLK(0, "mcfpit.2", 34, MCF_CLK);
+DEFINE_CLK(0, "mcfpit.3", 35, MCF_CLK);
+DEFINE_CLK(0, "mcfeport.0", 37, MCF_CLK);
+DEFINE_CLK(0, "mcfadc.0", 38, MCF_CLK);
+DEFINE_CLK(0, "mcfdac.0", 39, MCF_CLK);
+DEFINE_CLK(0, "mcfrtc.0", 42, MCF_CLK);
+DEFINE_CLK(0, "mcfsim.0", 43, MCF_CLK);
+DEFINE_CLK(0, "mcfusb-otg.0", 44, MCF_CLK);
+DEFINE_CLK(0, "mcfusb-host.0", 45, MCF_CLK);
+DEFINE_CLK(0, "mcfddr-sram.0", 46, MCF_CLK);
+DEFINE_CLK(0, "mcfssi.0", 47, MCF_CLK);
+DEFINE_CLK(0, "pll.0", 48, MCF_CLK);
+DEFINE_CLK(0, "mcfrng.0", 49, MCF_CLK);
+DEFINE_CLK(0, "mcfssi.1", 50, MCF_CLK);
+DEFINE_CLK(0, "mcfsdhc.0", 51, MCF_CLK);
+DEFINE_CLK(0, "enet-fec.0", 53, MCF_CLK);
+DEFINE_CLK(0, "enet-fec.1", 54, MCF_CLK);
+DEFINE_CLK(0, "switch.0", 55, MCF_CLK);
+DEFINE_CLK(0, "switch.1", 56, MCF_CLK);
+DEFINE_CLK(0, "nand.0", 63, MCF_CLK);
+
+DEFINE_CLK(1, "mcfow.0", 2, MCF_CLK);
+DEFINE_CLK(1, "mcfi2c.2", 4, MCF_CLK);
+DEFINE_CLK(1, "mcfi2c.3", 5, MCF_CLK);
+DEFINE_CLK(1, "mcfi2c.4", 6, MCF_CLK);
+DEFINE_CLK(1, "mcfi2c.5", 7, MCF_CLK);
+DEFINE_CLK(1, "mcfuart.4", 24, MCF_BUSCLK);
+DEFINE_CLK(1, "mcfuart.5", 25, MCF_BUSCLK);
+DEFINE_CLK(1, "mcfuart.6", 26, MCF_BUSCLK);
+DEFINE_CLK(1, "mcfuart.7", 27, MCF_BUSCLK);
+DEFINE_CLK(1, "mcfuart.8", 28, MCF_BUSCLK);
+DEFINE_CLK(1, "mcfuart.9", 29, MCF_BUSCLK);
+DEFINE_CLK(1, "mcfpwm.0", 34, MCF_BUSCLK);
+DEFINE_CLK(1, "sys.0", 36, MCF_BUSCLK);
+DEFINE_CLK(1, "gpio.0", 37, MCF_BUSCLK);
+
+struct clk *mcf_clks[] = {
+ &__clk_0_2,
+ &__clk_0_8,
+ &__clk_0_9,
+ &__clk_0_14,
+ &__clk_0_15,
+ &__clk_0_17,
+ &__clk_0_18,
+ &__clk_0_19,
+ &__clk_0_20,
+ &__clk_0_22,
+ &__clk_0_23,
+ &__clk_0_24,
+ &__clk_0_25,
+ &__clk_0_26,
+ &__clk_0_27,
+ &__clk_0_28,
+ &__clk_0_29,
+ &__clk_0_30,
+ &__clk_0_31,
+ &__clk_0_32,
+ &__clk_0_33,
+ &__clk_0_34,
+ &__clk_0_35,
+ &__clk_0_37,
+ &__clk_0_38,
+ &__clk_0_39,
+ &__clk_0_42,
+ &__clk_0_43,
+ &__clk_0_44,
+ &__clk_0_45,
+ &__clk_0_46,
+ &__clk_0_47,
+ &__clk_0_48,
+ &__clk_0_49,
+ &__clk_0_50,
+ &__clk_0_51,
+ &__clk_0_53,
+ &__clk_0_54,
+ &__clk_0_55,
+ &__clk_0_56,
+ &__clk_0_63,
+
+ &__clk_1_2,
+ &__clk_1_4,
+ &__clk_1_5,
+ &__clk_1_6,
+ &__clk_1_7,
+ &__clk_1_24,
+ &__clk_1_25,
+ &__clk_1_26,
+ &__clk_1_27,
+ &__clk_1_28,
+ &__clk_1_29,
+ &__clk_1_34,
+ &__clk_1_36,
+ &__clk_1_37,
+ NULL,
+};
+
+
+static struct clk * const enable_clks[] __initconst = {
+ /* make sure these clocks are enabled */
+ &__clk_0_18, /* intc0 */
+ &__clk_0_19, /* intc0 */
+ &__clk_0_20, /* intc0 */
+ &__clk_0_24, /* uart0 */
+ &__clk_0_25, /* uart1 */
+ &__clk_0_26, /* uart2 */
+ &__clk_0_27, /* uart3 */
+
+ &__clk_0_33, /* pit.1 */
+ &__clk_0_37, /* eport */
+ &__clk_0_48, /* pll */
+
+ &__clk_1_36, /* CCM/reset module/Power management */
+ &__clk_1_37, /* gpio */
+};
+static struct clk * const disable_clks[] __initconst = {
+ &__clk_0_8, /* can.0 */
+ &__clk_0_9, /* can.1 */
+ &__clk_0_14, /* i2c.1 */
+ &__clk_0_15, /* dspi.1 */
+ &__clk_0_17, /* eDMA */
+ &__clk_0_22, /* i2c.0 */
+ &__clk_0_23, /* dspi.0 */
+ &__clk_0_28, /* tmr.1 */
+ &__clk_0_29, /* tmr.2 */
+ &__clk_0_30, /* tmr.2 */
+ &__clk_0_31, /* tmr.3 */
+ &__clk_0_32, /* pit.0 */
+ &__clk_0_34, /* pit.2 */
+ &__clk_0_35, /* pit.3 */
+ &__clk_0_38, /* adc */
+ &__clk_0_39, /* dac */
+ &__clk_0_44, /* usb otg */
+ &__clk_0_45, /* usb host */
+ &__clk_0_47, /* ssi.0 */
+ &__clk_0_49, /* rng */
+ &__clk_0_50, /* ssi.1 */
+ &__clk_0_51, /* eSDHC */
+ &__clk_0_53, /* enet-fec */
+ &__clk_0_54, /* enet-fec */
+ &__clk_0_55, /* switch.0 */
+ &__clk_0_56, /* switch.1 */
+
+ &__clk_1_2, /* 1-wire */
+ &__clk_1_4, /* i2c.2 */
+ &__clk_1_5, /* i2c.3 */
+ &__clk_1_6, /* i2c.4 */
+ &__clk_1_7, /* i2c.5 */
+ &__clk_1_24, /* uart 4 */
+ &__clk_1_25, /* uart 5 */
+ &__clk_1_26, /* uart 6 */
+ &__clk_1_27, /* uart 7 */
+ &__clk_1_28, /* uart 8 */
+ &__clk_1_29, /* uart 9 */
+};
+
+static void __init m5441x_clk_init(void)
+{
+ unsigned i;
+
+ for (i = 0; i < ARRAY_SIZE(enable_clks); ++i)
+ __clk_init_enabled(enable_clks[i]);
+ /* make sure these clocks are disabled */
+ for (i = 0; i < ARRAY_SIZE(disable_clks); ++i)
+ __clk_init_disabled(disable_clks[i]);
+}
+
+static void __init m5441x_uarts_init(void)
+{
+ __raw_writeb(0x0f, MCFGPIO_PAR_UART0);
+ __raw_writeb(0x00, MCFGPIO_PAR_UART1);
+ __raw_writeb(0x00, MCFGPIO_PAR_UART2);
+}
+
+static void __init m5441x_fec_init(void)
+{
+ __raw_writeb(0x03, MCFGPIO_PAR_FEC);
+}
+
+void __init config_BSP(char *commandp, int size)
+{
+ m5441x_clk_init();
+ mach_sched_init = hw_timer_init;
+ m5441x_uarts_init();
+ m5441x_fec_init();
+}
+
+
+#if IS_ENABLED(CONFIG_RTC_DRV_M5441x)
+static struct resource m5441x_rtc_resources[] = {
+ {
+ .start = MCFRTC_BASE,
+ .end = MCFRTC_BASE + MCFRTC_SIZE - 1,
+ .flags = IORESOURCE_MEM,
+ },
+ {
+ .start = MCF_IRQ_RTC,
+ .end = MCF_IRQ_RTC,
+ .flags = IORESOURCE_IRQ,
+ },
+};
+
+static struct platform_device m5441x_rtc = {
+ .name = "mcfrtc",
+ .id = 0,
+ .resource = m5441x_rtc_resources,
+ .num_resources = ARRAY_SIZE(m5441x_rtc_resources),
+};
+#endif
+
+static struct platform_device *m5441x_devices[] __initdata = {
+#if IS_ENABLED(CONFIG_RTC_DRV_M5441x)
+ &m5441x_rtc,
+#endif
+};
+
+static int __init init_BSP(void)
+{
+ platform_add_devices(m5441x_devices, ARRAY_SIZE(m5441x_devices));
+ return 0;
+}
+
+arch_initcall(init_BSP);
diff --git a/arch/m68k/platform/coldfire/m54xx.c b/arch/m68k/platform/coldfire/m54xx.c
index 20672dadb252..2081c6cbb3de 100644
--- a/arch/m68k/platform/coldfire/m54xx.c
+++ b/arch/m68k/platform/coldfire/m54xx.c
@@ -21,19 +21,12 @@
#include <asm/m54xxsim.h>
#include <asm/mcfuart.h>
#include <asm/m54xxgpt.h>
-#include <asm/mcfgpio.h>
#ifdef CONFIG_MMU
#include <asm/mmu_context.h>
#endif
/***************************************************************************/
-struct mcf_gpio_chip mcf_gpio_chips[] = { };
-
-unsigned int mcf_gpio_chips_size = ARRAY_SIZE(mcf_gpio_chips);
-
-/***************************************************************************/
-
static void __init m54xx_uarts_init(void)
{
/* enable io pins */
diff --git a/arch/m68k/platform/coldfire/mcf8390.c b/arch/m68k/platform/coldfire/mcf8390.c
new file mode 100644
index 000000000000..23a6874a3248
--- /dev/null
+++ b/arch/m68k/platform/coldfire/mcf8390.c
@@ -0,0 +1,38 @@
+/*
+ * mcf8390.c -- platform support for 8390 ethernet on many boards
+ *
+ * (C) Copyright 2012, Greg Ungerer <gerg@uclinux.org>
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file COPYING in the main directory of this archive
+ * for more details.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/resource.h>
+#include <linux/platform_device.h>
+#include <asm/mcf8390.h>
+
+static struct resource mcf8390_resources[] = {
+ {
+ .start = NE2000_ADDR,
+ .end = NE2000_ADDR + NE2000_ADDRSIZE - 1,
+ .flags = IORESOURCE_MEM,
+ },
+ {
+ .start = NE2000_IRQ_VECTOR,
+ .end = NE2000_IRQ_VECTOR,
+ .flags = IORESOURCE_IRQ,
+ },
+};
+
+static int __init mcf8390_platform_init(void)
+{
+ platform_device_register_simple("mcf8390", -1, mcf8390_resources,
+ ARRAY_SIZE(mcf8390_resources));
+ return 0;
+}
+
+arch_initcall(mcf8390_platform_init);
diff --git a/arch/m68k/platform/coldfire/pci.c b/arch/m68k/platform/coldfire/pci.c
new file mode 100644
index 000000000000..553210d3d4c1
--- /dev/null
+++ b/arch/m68k/platform/coldfire/pci.c
@@ -0,0 +1,327 @@
+/*
+ * pci.c -- PCI bus support for ColdFire processors
+ *
+ * (C) Copyright 2012, Greg Ungerer <gerg@uclinux.com>
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file COPYING in the main directory of this archive
+ * for more details.
+ */
+
+#include <linux/types.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/interrupt.h>
+#include <linux/irq.h>
+#include <linux/io.h>
+#include <linux/pci.h>
+#include <linux/delay.h>
+#include <asm/coldfire.h>
+#include <asm/mcfsim.h>
+#include <asm/m54xxpci.h>
+
+/*
+ * Memory and IO mappings. We use a 1:1 mapping for local host memory to
+ * PCI bus memory (no reason not to really). IO space doesn't matter, we
+ * always use access functions for that. The device configuration space is
+ * mapped over the IO map space when we enable it in the PCICAR register.
+ */
+#define PCI_MEM_PA 0xf0000000 /* Host physical address */
+#define PCI_MEM_BA 0xf0000000 /* Bus physical address */
+#define PCI_MEM_SIZE 0x08000000 /* 128 MB */
+#define PCI_MEM_MASK (PCI_MEM_SIZE - 1)
+
+#define PCI_IO_PA 0xf8000000 /* Host physical address */
+#define PCI_IO_BA 0x00000000 /* Bus physical address */
+#define PCI_IO_SIZE 0x00010000 /* 64k */
+#define PCI_IO_MASK (PCI_IO_SIZE - 1)
+
+static struct pci_bus *rootbus;
+static unsigned long iospace;
+
+/*
+ * We need to be carefull probing on bus 0 (directly connected to host
+ * bridge). We should only acccess the well defined possible devices in
+ * use, ignore aliases and the like.
+ */
+static unsigned char mcf_host_slot2sid[32] = {
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+ 0, 1, 2, 0, 3, 4, 0, 0,
+ 0, 0, 0, 0, 0, 0, 0, 0,
+};
+
+static unsigned char mcf_host_irq[] = {
+ 0, 69, 69, 71, 71,
+};
+
+
+static inline void syncio(void)
+{
+ /* The ColdFire "nop" instruction waits for all bus IO to complete */
+ __asm__ __volatile__ ("nop");
+}
+
+/*
+ * Configuration space access functions. Configuration space access is
+ * through the IO mapping window, enabling it via the PCICAR register.
+ */
+static unsigned long mcf_mk_pcicar(int bus, unsigned int devfn, int where)
+{
+ return (bus << PCICAR_BUSN) | (devfn << PCICAR_DEVFNN) | (where & 0xfc);
+}
+
+static int mcf_pci_readconfig(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 *value)
+{
+ unsigned long addr;
+
+ *value = 0xffffffff;
+
+ if (bus->number == 0) {
+ if (mcf_host_slot2sid[PCI_SLOT(devfn)] == 0)
+ return PCIBIOS_SUCCESSFUL;
+ }
+
+ syncio();
+ addr = mcf_mk_pcicar(bus->number, devfn, where);
+ __raw_writel(PCICAR_E | addr, PCICAR);
+ addr = iospace + (where & 0x3);
+
+ switch (size) {
+ case 1:
+ *value = __raw_readb(addr);
+ break;
+ case 2:
+ *value = le16_to_cpu(__raw_readw(addr));
+ break;
+ default:
+ *value = le32_to_cpu(__raw_readl(addr));
+ break;
+ }
+
+ syncio();
+ __raw_writel(0, PCICAR);
+ return PCIBIOS_SUCCESSFUL;
+}
+
+static int mcf_pci_writeconfig(struct pci_bus *bus, unsigned int devfn,
+ int where, int size, u32 value)
+{
+ unsigned long addr;
+
+ if (bus->number == 0) {
+ if (mcf_host_slot2sid[PCI_SLOT(devfn)] == 0)
+ return PCIBIOS_SUCCESSFUL;
+ }
+
+ syncio();
+ addr = mcf_mk_pcicar(bus->number, devfn, where);
+ __raw_writel(PCICAR_E | addr, PCICAR);
+ addr = iospace + (where & 0x3);
+
+ switch (size) {
+ case 1:
+ __raw_writeb(value, addr);
+ break;
+ case 2:
+ __raw_writew(cpu_to_le16(value), addr);
+ break;
+ default:
+ __raw_writel(cpu_to_le32(value), addr);
+ break;
+ }
+
+ syncio();
+ __raw_writel(0, PCICAR);
+ return PCIBIOS_SUCCESSFUL;
+}
+
+static struct pci_ops mcf_pci_ops = {
+ .read = mcf_pci_readconfig,
+ .write = mcf_pci_writeconfig,
+};
+
+/*
+ * IO address space access functions. Pretty strait forward, these are
+ * directly mapped in to the IO mapping window. And that is mapped into
+ * virtual address space.
+ */
+u8 mcf_pci_inb(u32 addr)
+{
+ return __raw_readb(iospace + (addr & PCI_IO_MASK));
+}
+EXPORT_SYMBOL(mcf_pci_inb);
+
+u16 mcf_pci_inw(u32 addr)
+{
+ return le16_to_cpu(__raw_readw(iospace + (addr & PCI_IO_MASK)));
+}
+EXPORT_SYMBOL(mcf_pci_inw);
+
+u32 mcf_pci_inl(u32 addr)
+{
+ return le32_to_cpu(__raw_readl(iospace + (addr & PCI_IO_MASK)));
+}
+EXPORT_SYMBOL(mcf_pci_inl);
+
+void mcf_pci_insb(u32 addr, u8 *buf, u32 len)
+{
+ for (; len; len--)
+ *buf++ = mcf_pci_inb(addr);
+}
+EXPORT_SYMBOL(mcf_pci_insb);
+
+void mcf_pci_insw(u32 addr, u16 *buf, u32 len)
+{
+ for (; len; len--)
+ *buf++ = mcf_pci_inw(addr);
+}
+EXPORT_SYMBOL(mcf_pci_insw);
+
+void mcf_pci_insl(u32 addr, u32 *buf, u32 len)
+{
+ for (; len; len--)
+ *buf++ = mcf_pci_inl(addr);
+}
+EXPORT_SYMBOL(mcf_pci_insl);
+
+void mcf_pci_outb(u8 v, u32 addr)
+{
+ __raw_writeb(v, iospace + (addr & PCI_IO_MASK));
+}
+EXPORT_SYMBOL(mcf_pci_outb);
+
+void mcf_pci_outw(u16 v, u32 addr)
+{
+ __raw_writew(cpu_to_le16(v), iospace + (addr & PCI_IO_MASK));
+}
+EXPORT_SYMBOL(mcf_pci_outw);
+
+void mcf_pci_outl(u32 v, u32 addr)
+{
+ __raw_writel(cpu_to_le32(v), iospace + (addr & PCI_IO_MASK));
+}
+EXPORT_SYMBOL(mcf_pci_outl);
+
+void mcf_pci_outsb(u32 addr, const u8 *buf, u32 len)
+{
+ for (; len; len--)
+ mcf_pci_outb(*buf++, addr);
+}
+EXPORT_SYMBOL(mcf_pci_outsb);
+
+void mcf_pci_outsw(u32 addr, const u16 *buf, u32 len)
+{
+ for (; len; len--)
+ mcf_pci_outw(*buf++, addr);
+}
+EXPORT_SYMBOL(mcf_pci_outsw);
+
+void mcf_pci_outsl(u32 addr, const u32 *buf, u32 len)
+{
+ for (; len; len--)
+ mcf_pci_outl(*buf++, addr);
+}
+EXPORT_SYMBOL(mcf_pci_outsl);
+
+/*
+ * Initialize the PCI bus registers, and scan the bus.
+ */
+static struct resource mcf_pci_mem = {
+ .name = "PCI Memory space",
+ .start = PCI_MEM_PA,
+ .end = PCI_MEM_PA + PCI_MEM_SIZE - 1,
+ .flags = IORESOURCE_MEM,
+};
+
+static struct resource mcf_pci_io = {
+ .name = "PCI IO space",
+ .start = 0x400,
+ .end = 0x10000 - 1,
+ .flags = IORESOURCE_IO,
+};
+
+/*
+ * Interrupt mapping and setting.
+ */
+static int mcf_pci_map_irq(const struct pci_dev *dev, u8 slot, u8 pin)
+{
+ int sid;
+
+ sid = mcf_host_slot2sid[slot];
+ if (sid)
+ return mcf_host_irq[sid];
+ return 0;
+}
+
+static int __init mcf_pci_init(void)
+{
+ pr_info("ColdFire: PCI bus initialization...\n");
+
+ /* Reset the external PCI bus */
+ __raw_writel(PCIGSCR_RESET, PCIGSCR);
+ __raw_writel(0, PCITCR);
+
+ request_resource(&iomem_resource, &mcf_pci_mem);
+ request_resource(&iomem_resource, &mcf_pci_io);
+
+ /* Configure PCI arbiter */
+ __raw_writel(PACR_INTMPRI | PACR_INTMINTE | PACR_EXTMPRI(0x1f) |
+ PACR_EXTMINTE(0x1f), PACR);
+
+ /* Set required multi-function pins for PCI bus use */
+ __raw_writew(0x3ff, MCF_PAR_PCIBG);
+ __raw_writew(0x3ff, MCF_PAR_PCIBR);
+
+ /* Set up config space for local host bus controller */
+ __raw_writel(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
+ PCI_COMMAND_INVALIDATE, PCISCR);
+ __raw_writel(PCICR1_LT(32) | PCICR1_CL(8), PCICR1);
+ __raw_writel(0, PCICR2);
+
+ /*
+ * Set up the initiator windows for memory and IO mapping.
+ * These give the CPU bus access onto the PCI bus. One for each of
+ * PCI memory and IO address spaces.
+ */
+ __raw_writel(WXBTAR(PCI_MEM_PA, PCI_MEM_BA, PCI_MEM_SIZE),
+ PCIIW0BTAR);
+ __raw_writel(WXBTAR(PCI_IO_PA, PCI_IO_BA, PCI_IO_SIZE),
+ PCIIW1BTAR);
+ __raw_writel(PCIIWCR_W0_MEM /*| PCIIWCR_W0_MRDL*/ | PCIIWCR_W0_E |
+ PCIIWCR_W1_IO | PCIIWCR_W1_E, PCIIWCR);
+
+ /*
+ * Set up the target windows for access from the PCI bus back to the
+ * CPU bus. All we need is access to system RAM (for mastering).
+ */
+ __raw_writel(CONFIG_RAMBASE, PCIBAR1);
+ __raw_writel(CONFIG_RAMBASE | PCITBATR1_E, PCITBATR1);
+
+ /* Keep a virtual mapping to IO/config space active */
+ iospace = (unsigned long) ioremap(PCI_IO_PA, PCI_IO_SIZE);
+ if (iospace == 0)
+ return -ENODEV;
+ pr_info("Coldfire: PCI IO/config window mapped to 0x%x\n",
+ (u32) iospace);
+
+ /* Turn of PCI reset, and wait for devices to settle */
+ __raw_writel(0, PCIGSCR);
+ set_current_state(TASK_UNINTERRUPTIBLE);
+ schedule_timeout(msecs_to_jiffies(200));
+
+ rootbus = pci_scan_bus(0, &mcf_pci_ops, NULL);
+ rootbus->resource[0] = &mcf_pci_io;
+ rootbus->resource[1] = &mcf_pci_mem;
+
+ pci_fixup_irqs(pci_common_swizzle, mcf_pci_map_irq);
+ pci_bus_size_bridges(rootbus);
+ pci_bus_assign_resources(rootbus);
+ pci_enable_bridges(rootbus);
+ pci_bus_add_devices(rootbus);
+ return 0;
+}
+
+subsys_initcall(mcf_pci_init);
diff --git a/arch/m68k/platform/coldfire/pinmux.c b/arch/m68k/platform/coldfire/pinmux.c
deleted file mode 100644
index 8c62b825939f..000000000000
--- a/arch/m68k/platform/coldfire/pinmux.c
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Coldfire generic GPIO pinmux support.
- *
- * (C) Copyright 2009, Steven King <sfking@fdwdc.com>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; version 2 of the License.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- */
-
-#include <linux/kernel.h>
-
-#include <asm/pinmux.h>
-
-int mcf_pinmux_request(unsigned pinmux, unsigned func)
-{
- return 0;
-}
-
-void mcf_pinmux_release(unsigned pinmux, unsigned func)
-{
-}
diff --git a/arch/m68k/platform/coldfire/pit.c b/arch/m68k/platform/coldfire/pit.c
index e62dbbcb10f6..e8f3b97b0f77 100644
--- a/arch/m68k/platform/coldfire/pit.c
+++ b/arch/m68k/platform/coldfire/pit.c
@@ -93,7 +93,7 @@ struct clock_event_device cf_pit_clockevent = {
.set_mode = init_cf_pit_timer,
.set_next_event = cf_pit_next_event,
.shift = 32,
- .irq = MCFINT_VECBASE + MCFINT_PIT1,
+ .irq = MCF_IRQ_PIT1,
};
@@ -159,7 +159,7 @@ void hw_timer_init(irq_handler_t handler)
clockevent_delta2ns(0x3f, &cf_pit_clockevent);
clockevents_register_device(&cf_pit_clockevent);
- setup_irq(MCFINT_VECBASE + MCFINT_PIT1, &pit_irq);
+ setup_irq(MCF_IRQ_PIT1, &pit_irq);
clocksource_register_hz(&pit_clk, FREQ);
}
diff --git a/arch/m68k/platform/coldfire/timers.c b/arch/m68k/platform/coldfire/timers.c
index ed96ce50d79f..0a273e75408c 100644
--- a/arch/m68k/platform/coldfire/timers.c
+++ b/arch/m68k/platform/coldfire/timers.c
@@ -36,7 +36,7 @@
*/
void coldfire_profile_init(void);
-#if defined(CONFIG_M532x)
+#if defined(CONFIG_M532x) || defined(CONFIG_M5441x)
#define __raw_readtrr __raw_readl
#define __raw_writetrr __raw_writel
#else
diff --git a/arch/mips/kernel/kspd.c b/arch/mips/kernel/kspd.c
index 84d0639e4580..b77f56bbb477 100644
--- a/arch/mips/kernel/kspd.c
+++ b/arch/mips/kernel/kspd.c
@@ -323,7 +323,7 @@ static void sp_cleanup(void)
fdt = files_fdtable(files);
for (;;) {
unsigned long set;
- i = j * __NFDBITS;
+ i = j * BITS_PER_LONG;
if (i >= fdt->max_fds)
break;
set = fdt->open_fds[j++];
diff --git a/arch/powerpc/Makefile b/arch/powerpc/Makefile
index 950d1f7a5a39..159e94f4b22a 100644
--- a/arch/powerpc/Makefile
+++ b/arch/powerpc/Makefile
@@ -149,7 +149,6 @@ core-$(CONFIG_KVM) += arch/powerpc/kvm/
core-$(CONFIG_PERF_EVENTS) += arch/powerpc/perf/
drivers-$(CONFIG_OPROFILE) += arch/powerpc/oprofile/
-drivers-$(CONFIG_CRYPTO_DEV_NX) += drivers/crypto/nx/
# Default to zImage, override when needed
all: zImage
diff --git a/arch/s390/crypto/crypto_des.h b/arch/s390/crypto/crypto_des.h
deleted file mode 100644
index 6210457ceebb..000000000000
--- a/arch/s390/crypto/crypto_des.h
+++ /dev/null
@@ -1,18 +0,0 @@
-/*
- * Cryptographic API.
- *
- * Function for checking keys for the DES and Tripple DES Encryption
- * algorithms.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- */
-#ifndef __CRYPTO_DES_H__
-#define __CRYPTO_DES_H__
-
-extern int crypto_des_check_key(const u8*, unsigned int, u32*);
-
-#endif /*__CRYPTO_DES_H__*/
diff --git a/arch/sh/include/asm/siu.h b/arch/sh/include/asm/siu.h
index 1d95c78808d1..580b7ac228b7 100644
--- a/arch/sh/include/asm/siu.h
+++ b/arch/sh/include/asm/siu.h
@@ -14,7 +14,6 @@
struct device;
struct siu_platform {
- struct device *dma_dev;
unsigned int dma_slave_tx_a;
unsigned int dma_slave_rx_a;
unsigned int dma_slave_tx_b;
diff --git a/arch/sh/kernel/cpu/sh4a/setup-sh7722.c b/arch/sh/kernel/cpu/sh4a/setup-sh7722.c
index 0f5a21907da6..65786c7f5ded 100644
--- a/arch/sh/kernel/cpu/sh4a/setup-sh7722.c
+++ b/arch/sh/kernel/cpu/sh4a/setup-sh7722.c
@@ -512,7 +512,6 @@ static struct platform_device tmu2_device = {
};
static struct siu_platform siu_platform_data = {
- .dma_dev = &dma_device.dev,
.dma_slave_tx_a = SHDMA_SLAVE_SIUA_TX,
.dma_slave_rx_a = SHDMA_SLAVE_SIUA_RX,
.dma_slave_tx_b = SHDMA_SLAVE_SIUB_TX,
diff --git a/arch/sparc/include/asm/fixmap.h b/arch/sparc/include/asm/fixmap.h
deleted file mode 100644
index f18fc0755adf..000000000000
--- a/arch/sparc/include/asm/fixmap.h
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * fixmap.h: compile-time virtual memory allocation
- *
- * This file is subject to the terms and conditions of the GNU General Public
- * License. See the file "COPYING" in the main directory of this archive
- * for more details.
- *
- * Copyright (C) 1998 Ingo Molnar
- *
- * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
- */
-
-#ifndef _ASM_FIXMAP_H
-#define _ASM_FIXMAP_H
-
-#include <linux/kernel.h>
-#include <asm/page.h>
-#ifdef CONFIG_HIGHMEM
-#include <linux/threads.h>
-#include <asm/kmap_types.h>
-#endif
-
-/*
- * Here we define all the compile-time 'special' virtual
- * addresses. The point is to have a constant address at
- * compile time, but to set the physical address only
- * in the boot process. We allocate these special addresses
- * from the top of unused virtual memory (0xfd000000 - 1 page) backwards.
- * Also this lets us do fail-safe vmalloc(), we
- * can guarantee that these special addresses and
- * vmalloc()-ed addresses never overlap.
- *
- * these 'compile-time allocated' memory buffers are
- * fixed-size 4k pages. (or larger if used with an increment
- * highger than 1) use fixmap_set(idx,phys) to associate
- * physical memory with fixmap indices.
- *
- * TLB entries of such buffers will not be flushed across
- * task switches.
- */
-
-/*
- * on UP currently we will have no trace of the fixmap mechanism,
- * no page table allocations, etc. This might change in the
- * future, say framebuffers for the console driver(s) could be
- * fix-mapped?
- */
-enum fixed_addresses {
- FIX_HOLE,
-#ifdef CONFIG_HIGHMEM
- FIX_KMAP_BEGIN,
- FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*NR_CPUS)-1,
-#endif
- __end_of_fixed_addresses
-};
-
-extern void __set_fixmap (enum fixed_addresses idx,
- unsigned long phys, pgprot_t flags);
-
-#define set_fixmap(idx, phys) \
- __set_fixmap(idx, phys, PAGE_KERNEL)
-/*
- * Some hardware wants to get fixmapped without caching.
- */
-#define set_fixmap_nocache(idx, phys) \
- __set_fixmap(idx, phys, PAGE_KERNEL_NOCACHE)
-/*
- * used by vmalloc.c.
- *
- * Leave one empty page between IO pages at 0xfd000000 and
- * the start of the fixmap.
- */
-#define FIXADDR_TOP (0xfcfff000UL)
-#define FIXADDR_SIZE ((__end_of_fixed_addresses) << PAGE_SHIFT)
-#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE)
-
-#define __fix_to_virt(x) (FIXADDR_TOP - ((x) << PAGE_SHIFT))
-#define __virt_to_fix(x) ((FIXADDR_TOP - ((x)&PAGE_MASK)) >> PAGE_SHIFT)
-
-extern void __this_fixmap_does_not_exist(void);
-
-/*
- * 'index to address' translation. If anyone tries to use the idx
- * directly without tranlation, we catch the bug with a NULL-deference
- * kernel oops. Illegal ranges of incoming indices are caught too.
- */
-static inline unsigned long fix_to_virt(const unsigned int idx)
-{
- /*
- * this branch gets completely eliminated after inlining,
- * except when someone tries to use fixaddr indices in an
- * illegal way. (such as mixing up address types or using
- * out-of-range indices).
- *
- * If it doesn't get removed, the linker will complain
- * loudly with a reasonably clear error message..
- */
- if (idx >= __end_of_fixed_addresses)
- __this_fixmap_does_not_exist();
-
- return __fix_to_virt(idx);
-}
-
-static inline unsigned long virt_to_fix(const unsigned long vaddr)
-{
- BUG_ON(vaddr >= FIXADDR_TOP || vaddr < FIXADDR_START);
- return __virt_to_fix(vaddr);
-}
-
-#endif
diff --git a/arch/sparc/include/asm/highmem.h b/arch/sparc/include/asm/highmem.h
index 3b6e00dd96e5..4f9e15c757e2 100644
--- a/arch/sparc/include/asm/highmem.h
+++ b/arch/sparc/include/asm/highmem.h
@@ -21,7 +21,6 @@
#ifdef __KERNEL__
#include <linux/interrupt.h>
-#include <asm/fixmap.h>
#include <asm/vaddrs.h>
#include <asm/kmap_types.h>
#include <asm/pgtable.h>
@@ -29,7 +28,6 @@
/* declarations for highmem.c */
extern unsigned long highstart_pfn, highend_pfn;
-extern pte_t *kmap_pte;
extern pgprot_t kmap_prot;
extern pte_t *pkmap_page_table;
@@ -72,7 +70,6 @@ static inline void kunmap(struct page *page)
extern void *kmap_atomic(struct page *page);
extern void __kunmap_atomic(void *kvaddr);
-extern struct page *kmap_atomic_to_page(void *vaddr);
#define flush_cache_kmaps() flush_cache_all()
diff --git a/arch/sparc/include/asm/leon.h b/arch/sparc/include/asm/leon.h
index 3375c6293893..15a716934e4d 100644
--- a/arch/sparc/include/asm/leon.h
+++ b/arch/sparc/include/asm/leon.h
@@ -82,7 +82,6 @@ static inline unsigned long leon_load_reg(unsigned long paddr)
#define LEON_BYPASS_LOAD_PA(x) leon_load_reg((unsigned long)(x))
#define LEON_BYPASS_STORE_PA(x, v) leon_store_reg((unsigned long)(x), (unsigned long)(v))
-extern void leon_init(void);
extern void leon_switch_mm(void);
extern void leon_init_IRQ(void);
diff --git a/arch/sparc/include/asm/mmu_context_32.h b/arch/sparc/include/asm/mmu_context_32.h
index 01456c900720..2df2a9be8f6d 100644
--- a/arch/sparc/include/asm/mmu_context_32.h
+++ b/arch/sparc/include/asm/mmu_context_32.h
@@ -9,14 +9,12 @@ static inline void enter_lazy_tlb(struct mm_struct *mm, struct task_struct *tsk)
{
}
-/*
- * Initialize a new mmu context. This is invoked when a new
+/* Initialize a new mmu context. This is invoked when a new
* address space instance (unique or shared) is instantiated.
*/
-#define init_new_context(tsk, mm) (((mm)->context = NO_CONTEXT), 0)
+int init_new_context(struct task_struct *tsk, struct mm_struct *mm);
-/*
- * Destroy a dead context. This occurs when mmput drops the
+/* Destroy a dead context. This occurs when mmput drops the
* mm_users count to zero, the mmaps have been released, and
* all the page tables have been flushed. Our job is to destroy
* any remaining processor-specific state.
diff --git a/arch/sparc/include/asm/page_32.h b/arch/sparc/include/asm/page_32.h
index fab78a308ebf..f82a1f36b655 100644
--- a/arch/sparc/include/asm/page_32.h
+++ b/arch/sparc/include/asm/page_32.h
@@ -107,8 +107,7 @@ typedef unsigned long iopgprot_t;
typedef struct page *pgtable_t;
-extern unsigned long sparc_unmapped_base;
-#define TASK_UNMAPPED_BASE sparc_unmapped_base
+#define TASK_UNMAPPED_BASE 0x50000000
#else /* !(__ASSEMBLY__) */
diff --git a/arch/sparc/include/asm/pgalloc_32.h b/arch/sparc/include/asm/pgalloc_32.h
index e5b169b46d21..9b1c36de0f18 100644
--- a/arch/sparc/include/asm/pgalloc_32.h
+++ b/arch/sparc/include/asm/pgalloc_32.h
@@ -11,28 +11,15 @@
struct page;
-extern struct pgtable_cache_struct {
- unsigned long *pgd_cache;
- unsigned long *pte_cache;
- unsigned long pgtable_cache_sz;
- unsigned long pgd_cache_sz;
-} pgt_quicklists;
-
-unsigned long srmmu_get_nocache(int size, int align);
-void srmmu_free_nocache(unsigned long vaddr, int size);
-
-#define pgd_quicklist (pgt_quicklists.pgd_cache)
-#define pmd_quicklist ((unsigned long *)0)
-#define pte_quicklist (pgt_quicklists.pte_cache)
-#define pgtable_cache_size (pgt_quicklists.pgtable_cache_sz)
-#define pgd_cache_size (pgt_quicklists.pgd_cache_sz)
+void *srmmu_get_nocache(int size, int align);
+void srmmu_free_nocache(void *addr, int size);
#define check_pgt_cache() do { } while (0)
pgd_t *get_pgd_fast(void);
static inline void free_pgd_fast(pgd_t *pgd)
{
- srmmu_free_nocache((unsigned long)pgd, SRMMU_PGD_TABLE_SIZE);
+ srmmu_free_nocache(pgd, SRMMU_PGD_TABLE_SIZE);
}
#define pgd_free(mm, pgd) free_pgd_fast(pgd)
@@ -50,13 +37,13 @@ static inline void pgd_set(pgd_t * pgdp, pmd_t * pmdp)
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm,
unsigned long address)
{
- return (pmd_t *)srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE,
- SRMMU_PMD_TABLE_SIZE);
+ return srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE,
+ SRMMU_PMD_TABLE_SIZE);
}
static inline void free_pmd_fast(pmd_t * pmd)
{
- srmmu_free_nocache((unsigned long)pmd, SRMMU_PMD_TABLE_SIZE);
+ srmmu_free_nocache(pmd, SRMMU_PMD_TABLE_SIZE);
}
#define pmd_free(mm, pmd) free_pmd_fast(pmd)
@@ -73,13 +60,13 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address);
static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
unsigned long address)
{
- return (pte_t *)srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
+ return srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
}
static inline void free_pte_fast(pte_t *pte)
{
- srmmu_free_nocache((unsigned long)pte, PTE_SIZE);
+ srmmu_free_nocache(pte, PTE_SIZE);
}
#define pte_free_kernel(mm, pte) free_pte_fast(pte)
diff --git a/arch/sparc/include/asm/pgtable_32.h b/arch/sparc/include/asm/pgtable_32.h
index cbbbed5cb3aa..6fc13483f702 100644
--- a/arch/sparc/include/asm/pgtable_32.h
+++ b/arch/sparc/include/asm/pgtable_32.h
@@ -52,8 +52,9 @@ extern unsigned long calc_highpages(void);
#define PAGE_READONLY SRMMU_PAGE_RDONLY
#define PAGE_KERNEL SRMMU_PAGE_KERNEL
-/* Top-level page directory */
-extern pgd_t swapper_pg_dir[1024];
+/* Top-level page directory - dummy used by init-mm.
+ * srmmu.c will assign the real one (which is dynamically sized) */
+#define swapper_pg_dir NULL
extern void paging_init(void);
@@ -78,8 +79,6 @@ extern unsigned long ptr_in_current_pgd;
#define __S110 PAGE_SHARED
#define __S111 PAGE_SHARED
-extern int num_contexts;
-
/* First physical page can be anywhere, the following is needed so that
* va-->pa and vice versa conversions work properly without performance
* hit for all __pa()/__va() operations.
@@ -88,18 +87,11 @@ extern unsigned long phys_base;
extern unsigned long pfn_base;
/*
- * BAD_PAGETABLE is used when we need a bogus page-table, while
- * BAD_PAGE is used for a bogus page.
- *
* ZERO_PAGE is a global shared page that is always zero: used
* for zero-mapped memory areas etc..
*/
-extern pte_t * __bad_pagetable(void);
-extern pte_t __bad_page(void);
extern unsigned long empty_zero_page;
-#define BAD_PAGETABLE __bad_pagetable()
-#define BAD_PAGE __bad_page()
#define ZERO_PAGE(vaddr) (virt_to_page(&empty_zero_page))
/*
@@ -398,36 +390,6 @@ static inline pte_t pgoff_to_pte(unsigned long pgoff)
*/
#define PTE_FILE_MAX_BITS 24
-/*
- */
-struct ctx_list {
- struct ctx_list *next;
- struct ctx_list *prev;
- unsigned int ctx_number;
- struct mm_struct *ctx_mm;
-};
-
-extern struct ctx_list *ctx_list_pool; /* Dynamically allocated */
-extern struct ctx_list ctx_free; /* Head of free list */
-extern struct ctx_list ctx_used; /* Head of used contexts list */
-
-#define NO_CONTEXT -1
-
-static inline void remove_from_ctx_list(struct ctx_list *entry)
-{
- entry->next->prev = entry->prev;
- entry->prev->next = entry->next;
-}
-
-static inline void add_to_ctx_list(struct ctx_list *head, struct ctx_list *entry)
-{
- entry->next = head;
- (entry->prev = head->prev)->next = entry;
- head->prev = entry;
-}
-#define add_to_free_ctxlist(entry) add_to_ctx_list(&ctx_free, entry)
-#define add_to_used_ctxlist(entry) add_to_ctx_list(&ctx_used, entry)
-
static inline unsigned long
__get_phys (unsigned long addr)
{
diff --git a/arch/sparc/include/asm/vaddrs.h b/arch/sparc/include/asm/vaddrs.h
index da6535d88a72..c3dbcf902034 100644
--- a/arch/sparc/include/asm/vaddrs.h
+++ b/arch/sparc/include/asm/vaddrs.h
@@ -30,6 +30,28 @@
*/
#define SRMMU_NOCACHE_ALCRATIO 64 /* 256 pages per 64MB of system RAM */
+#ifndef __ASSEMBLY__
+#include <asm/kmap_types.h>
+
+enum fixed_addresses {
+ FIX_HOLE,
+#ifdef CONFIG_HIGHMEM
+ FIX_KMAP_BEGIN,
+ FIX_KMAP_END = (KM_TYPE_NR * NR_CPUS),
+#endif
+ __end_of_fixed_addresses
+};
+#endif
+
+/* Leave one empty page between IO pages at 0xfd000000 and
+ * the top of the fixmap.
+ */
+#define FIXADDR_TOP (0xfcfff000UL)
+#define FIXADDR_SIZE ((FIX_KMAP_END + 1) << PAGE_SHIFT)
+#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE)
+
+#define __fix_to_virt(x) (FIXADDR_TOP - ((x) << PAGE_SHIFT))
+
#define SUN4M_IOBASE_VADDR 0xfd000000 /* Base for mapping pages */
#define IOBASE_VADDR 0xfe000000
#define IOBASE_END 0xfe600000
diff --git a/arch/sparc/kernel/head_32.S b/arch/sparc/kernel/head_32.S
index afeb1d770303..3d92c0a8f6c4 100644
--- a/arch/sparc/kernel/head_32.S
+++ b/arch/sparc/kernel/head_32.S
@@ -58,8 +58,6 @@ sun4e_notsup:
/* This was the only reasonable way I could think of to properly align
* these page-table data structures.
*/
- .globl swapper_pg_dir
-swapper_pg_dir: .skip PAGE_SIZE
.globl empty_zero_page
empty_zero_page: .skip PAGE_SIZE
diff --git a/arch/sparc/kernel/leon_kernel.c b/arch/sparc/kernel/leon_kernel.c
index e34e2c40c060..f8b6eee40bde 100644
--- a/arch/sparc/kernel/leon_kernel.c
+++ b/arch/sparc/kernel/leon_kernel.c
@@ -486,17 +486,6 @@ void __init leon_trans_init(struct device_node *dp)
}
}
-void __initdata (*prom_amba_init)(struct device_node *dp, struct device_node ***nextp) = 0;
-
-void __init leon_node_init(struct device_node *dp, struct device_node ***nextp)
-{
- if (prom_amba_init &&
- strcmp(dp->type, "ambapp") == 0 &&
- strcmp(dp->name, "ambapp0") == 0) {
- prom_amba_init(dp, nextp);
- }
-}
-
#ifdef CONFIG_SMP
void leon_clear_profile_irq(int cpu)
{
@@ -522,8 +511,3 @@ void __init leon_init_IRQ(void)
sparc_config.clear_clock_irq = leon_clear_clock_irq;
sparc_config.load_profile_irq = leon_load_profile_irq;
}
-
-void __init leon_init(void)
-{
- of_pdt_build_more = &leon_node_init;
-}
diff --git a/arch/sparc/kernel/process_32.c b/arch/sparc/kernel/process_32.c
index cb36e82dcd5d..14006d8aca28 100644
--- a/arch/sparc/kernel/process_32.c
+++ b/arch/sparc/kernel/process_32.c
@@ -333,9 +333,6 @@ int copy_thread(unsigned long clone_flags, unsigned long sp,
put_psr(get_psr() | PSR_EF);
fpsave(&p->thread.float_regs[0], &p->thread.fsr,
&p->thread.fpqueue[0], &p->thread.fpqdepth);
-#ifdef CONFIG_SMP
- clear_thread_flag(TIF_USEDFPU);
-#endif
}
/*
@@ -413,6 +410,7 @@ int copy_thread(unsigned long clone_flags, unsigned long sp,
#ifdef CONFIG_SMP
/* FPU must be disabled on SMP. */
childregs->psr &= ~PSR_EF;
+ clear_tsk_thread_flag(p, TIF_USEDFPU);
#endif
/* Set the return value for the child. */
diff --git a/arch/sparc/kernel/setup_32.c b/arch/sparc/kernel/setup_32.c
index efe3e64bba38..38bf80a22f02 100644
--- a/arch/sparc/kernel/setup_32.c
+++ b/arch/sparc/kernel/setup_32.c
@@ -371,7 +371,6 @@ void __init setup_arch(char **cmdline_p)
(*(linux_dbvec->teach_debugger))();
}
- init_mm.context = (unsigned long) NO_CONTEXT;
init_task.thread.kregs = &fake_swapper_regs;
/* Run-time patch instructions to match the cpu model */
diff --git a/arch/sparc/kernel/sys_sparc_64.c b/arch/sparc/kernel/sys_sparc_64.c
index 275f74fd6f6a..c38e5aaae56f 100644
--- a/arch/sparc/kernel/sys_sparc_64.c
+++ b/arch/sparc/kernel/sys_sparc_64.c
@@ -66,23 +66,6 @@ static inline int invalid_64bit_range(unsigned long addr, unsigned long len)
return 0;
}
-/* Does start,end straddle the VA-space hole? */
-static inline int straddles_64bit_va_hole(unsigned long start, unsigned long end)
-{
- unsigned long va_exclude_start, va_exclude_end;
-
- va_exclude_start = VA_EXCLUDE_START;
- va_exclude_end = VA_EXCLUDE_END;
-
- if (likely(start < va_exclude_start && end < va_exclude_start))
- return 0;
-
- if (likely(start >= va_exclude_end && end >= va_exclude_end))
- return 0;
-
- return 1;
-}
-
/* These functions differ from the default implementations in
* mm/mmap.c in two ways:
*
diff --git a/arch/sparc/lib/NG2memcpy.S b/arch/sparc/lib/NG2memcpy.S
index 0aed75653b50..03eadf66b0d3 100644
--- a/arch/sparc/lib/NG2memcpy.S
+++ b/arch/sparc/lib/NG2memcpy.S
@@ -90,49 +90,49 @@
faligndata %x7, %x8, %f14;
#define FREG_MOVE_1(x0) \
- fmovd %x0, %f0;
+ fsrc2 %x0, %f0;
#define FREG_MOVE_2(x0, x1) \
- fmovd %x0, %f0; \
- fmovd %x1, %f2;
+ fsrc2 %x0, %f0; \
+ fsrc2 %x1, %f2;
#define FREG_MOVE_3(x0, x1, x2) \
- fmovd %x0, %f0; \
- fmovd %x1, %f2; \
- fmovd %x2, %f4;
+ fsrc2 %x0, %f0; \
+ fsrc2 %x1, %f2; \
+ fsrc2 %x2, %f4;
#define FREG_MOVE_4(x0, x1, x2, x3) \
- fmovd %x0, %f0; \
- fmovd %x1, %f2; \
- fmovd %x2, %f4; \
- fmovd %x3, %f6;
+ fsrc2 %x0, %f0; \
+ fsrc2 %x1, %f2; \
+ fsrc2 %x2, %f4; \
+ fsrc2 %x3, %f6;
#define FREG_MOVE_5(x0, x1, x2, x3, x4) \
- fmovd %x0, %f0; \
- fmovd %x1, %f2; \
- fmovd %x2, %f4; \
- fmovd %x3, %f6; \
- fmovd %x4, %f8;
+ fsrc2 %x0, %f0; \
+ fsrc2 %x1, %f2; \
+ fsrc2 %x2, %f4; \
+ fsrc2 %x3, %f6; \
+ fsrc2 %x4, %f8;
#define FREG_MOVE_6(x0, x1, x2, x3, x4, x5) \
- fmovd %x0, %f0; \
- fmovd %x1, %f2; \
- fmovd %x2, %f4; \
- fmovd %x3, %f6; \
- fmovd %x4, %f8; \
- fmovd %x5, %f10;
+ fsrc2 %x0, %f0; \
+ fsrc2 %x1, %f2; \
+ fsrc2 %x2, %f4; \
+ fsrc2 %x3, %f6; \
+ fsrc2 %x4, %f8; \
+ fsrc2 %x5, %f10;
#define FREG_MOVE_7(x0, x1, x2, x3, x4, x5, x6) \
- fmovd %x0, %f0; \
- fmovd %x1, %f2; \
- fmovd %x2, %f4; \
- fmovd %x3, %f6; \
- fmovd %x4, %f8; \
- fmovd %x5, %f10; \
- fmovd %x6, %f12;
+ fsrc2 %x0, %f0; \
+ fsrc2 %x1, %f2; \
+ fsrc2 %x2, %f4; \
+ fsrc2 %x3, %f6; \
+ fsrc2 %x4, %f8; \
+ fsrc2 %x5, %f10; \
+ fsrc2 %x6, %f12;
#define FREG_MOVE_8(x0, x1, x2, x3, x4, x5, x6, x7) \
- fmovd %x0, %f0; \
- fmovd %x1, %f2; \
- fmovd %x2, %f4; \
- fmovd %x3, %f6; \
- fmovd %x4, %f8; \
- fmovd %x5, %f10; \
- fmovd %x6, %f12; \
- fmovd %x7, %f14;
+ fsrc2 %x0, %f0; \
+ fsrc2 %x1, %f2; \
+ fsrc2 %x2, %f4; \
+ fsrc2 %x3, %f6; \
+ fsrc2 %x4, %f8; \
+ fsrc2 %x5, %f10; \
+ fsrc2 %x6, %f12; \
+ fsrc2 %x7, %f14;
#define FREG_LOAD_1(base, x0) \
EX_LD(LOAD(ldd, base + 0x00, %x0))
#define FREG_LOAD_2(base, x0, x1) \
diff --git a/arch/sparc/lib/U1memcpy.S b/arch/sparc/lib/U1memcpy.S
index bafd2fc07acb..b67142b7768e 100644
--- a/arch/sparc/lib/U1memcpy.S
+++ b/arch/sparc/lib/U1memcpy.S
@@ -109,7 +109,7 @@
#define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \
subcc %left, 8, %left; \
bl,pn %xcc, 95f; \
- fsrc1 %f0, %f1;
+ fsrc2 %f0, %f1;
#define UNEVEN_VISCHUNK(dest, f0, f1, left) \
UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \
@@ -201,7 +201,7 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
andn %o1, (0x40 - 1), %o1
and %g2, 7, %g2
andncc %g3, 0x7, %g3
- fmovd %f0, %f2
+ fsrc2 %f0, %f2
sub %g3, 0x8, %g3
sub %o2, %GLOBAL_SPARE, %o2
diff --git a/arch/sparc/lib/copy_page.S b/arch/sparc/lib/copy_page.S
index b243d3b606ba..4d2df328e514 100644
--- a/arch/sparc/lib/copy_page.S
+++ b/arch/sparc/lib/copy_page.S
@@ -34,10 +34,10 @@
#endif
#define TOUCH(reg0, reg1, reg2, reg3, reg4, reg5, reg6, reg7) \
- fmovd %reg0, %f48; fmovd %reg1, %f50; \
- fmovd %reg2, %f52; fmovd %reg3, %f54; \
- fmovd %reg4, %f56; fmovd %reg5, %f58; \
- fmovd %reg6, %f60; fmovd %reg7, %f62;
+ fsrc2 %reg0, %f48; fsrc2 %reg1, %f50; \
+ fsrc2 %reg2, %f52; fsrc2 %reg3, %f54; \
+ fsrc2 %reg4, %f56; fsrc2 %reg5, %f58; \
+ fsrc2 %reg6, %f60; fsrc2 %reg7, %f62;
.text
@@ -104,60 +104,60 @@ cheetah_copy_page_insn:
prefetch [%o1 + 0x140], #one_read
ldd [%o1 + 0x010], %f4
prefetch [%o1 + 0x180], #one_read
- fmovd %f0, %f16
+ fsrc2 %f0, %f16
ldd [%o1 + 0x018], %f6
- fmovd %f2, %f18
+ fsrc2 %f2, %f18
ldd [%o1 + 0x020], %f8
- fmovd %f4, %f20
+ fsrc2 %f4, %f20
ldd [%o1 + 0x028], %f10
- fmovd %f6, %f22
+ fsrc2 %f6, %f22
ldd [%o1 + 0x030], %f12
- fmovd %f8, %f24
+ fsrc2 %f8, %f24
ldd [%o1 + 0x038], %f14
- fmovd %f10, %f26
+ fsrc2 %f10, %f26
ldd [%o1 + 0x040], %f0
1: ldd [%o1 + 0x048], %f2
- fmovd %f12, %f28
+ fsrc2 %f12, %f28
ldd [%o1 + 0x050], %f4
- fmovd %f14, %f30
+ fsrc2 %f14, %f30
stda %f16, [%o0] ASI_BLK_P
ldd [%o1 + 0x058], %f6
- fmovd %f0, %f16
+ fsrc2 %f0, %f16
ldd [%o1 + 0x060], %f8
- fmovd %f2, %f18
+ fsrc2 %f2, %f18
ldd [%o1 + 0x068], %f10
- fmovd %f4, %f20
+ fsrc2 %f4, %f20
ldd [%o1 + 0x070], %f12
- fmovd %f6, %f22
+ fsrc2 %f6, %f22
ldd [%o1 + 0x078], %f14
- fmovd %f8, %f24
+ fsrc2 %f8, %f24
ldd [%o1 + 0x080], %f0
prefetch [%o1 + 0x180], #one_read
- fmovd %f10, %f26
+ fsrc2 %f10, %f26
subcc %o2, 1, %o2
add %o0, 0x40, %o0
bne,pt %xcc, 1b
add %o1, 0x40, %o1
ldd [%o1 + 0x048], %f2
- fmovd %f12, %f28
+ fsrc2 %f12, %f28
ldd [%o1 + 0x050], %f4
- fmovd %f14, %f30
+ fsrc2 %f14, %f30
stda %f16, [%o0] ASI_BLK_P
ldd [%o1 + 0x058], %f6
- fmovd %f0, %f16
+ fsrc2 %f0, %f16
ldd [%o1 + 0x060], %f8
- fmovd %f2, %f18
+ fsrc2 %f2, %f18
ldd [%o1 + 0x068], %f10
- fmovd %f4, %f20
+ fsrc2 %f4, %f20
ldd [%o1 + 0x070], %f12
- fmovd %f6, %f22
+ fsrc2 %f6, %f22
add %o0, 0x40, %o0
ldd [%o1 + 0x078], %f14
- fmovd %f8, %f24
- fmovd %f10, %f26
- fmovd %f12, %f28
- fmovd %f14, %f30
+ fsrc2 %f8, %f24
+ fsrc2 %f10, %f26
+ fsrc2 %f12, %f28
+ fsrc2 %f14, %f30
stda %f16, [%o0] ASI_BLK_P
membar #Sync
VISExitHalf
diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c
index f46cf6be3370..77ac917be152 100644
--- a/arch/sparc/mm/fault_32.c
+++ b/arch/sparc/mm/fault_32.c
@@ -32,24 +32,6 @@
int show_unhandled_signals = 1;
-/* At boot time we determine these two values necessary for setting
- * up the segment maps and page table entries (pte's).
- */
-
-int num_contexts;
-
-/* Return how much physical memory we have. */
-unsigned long probe_memory(void)
-{
- unsigned long total = 0;
- int i;
-
- for (i = 0; sp_banks[i].num_bytes; i++)
- total += sp_banks[i].num_bytes;
-
- return total;
-}
-
static void unhandled_fault(unsigned long, struct task_struct *,
struct pt_regs *) __attribute__ ((noreturn));
diff --git a/arch/sparc/mm/highmem.c b/arch/sparc/mm/highmem.c
index 055c66cf1bf4..449f864f0cef 100644
--- a/arch/sparc/mm/highmem.c
+++ b/arch/sparc/mm/highmem.c
@@ -22,13 +22,31 @@
* shared by CPUs, and so precious, and establishing them requires IPI.
* Atomic kmaps are lightweight and we may have NCPUS more of them.
*/
-#include <linux/mm.h>
#include <linux/highmem.h>
#include <linux/export.h>
-#include <asm/pgalloc.h>
+#include <linux/mm.h>
+
#include <asm/cacheflush.h>
#include <asm/tlbflush.h>
-#include <asm/fixmap.h>
+#include <asm/pgalloc.h>
+#include <asm/vaddrs.h>
+
+pgprot_t kmap_prot;
+
+static pte_t *kmap_pte;
+
+void __init kmap_init(void)
+{
+ unsigned long address;
+ pmd_t *dir;
+
+ address = __fix_to_virt(FIX_KMAP_BEGIN);
+ dir = pmd_offset(pgd_offset_k(address), address);
+
+ /* cache the first kmap pte */
+ kmap_pte = pte_offset_kernel(dir, address);
+ kmap_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV | SRMMU_CACHE);
+}
void *kmap_atomic(struct page *page)
{
@@ -110,21 +128,3 @@ void __kunmap_atomic(void *kvaddr)
pagefault_enable();
}
EXPORT_SYMBOL(__kunmap_atomic);
-
-/* We may be fed a pagetable here by ptep_to_xxx and others. */
-struct page *kmap_atomic_to_page(void *ptr)
-{
- unsigned long idx, vaddr = (unsigned long)ptr;
- pte_t *pte;
-
- if (vaddr < SRMMU_NOCACHE_VADDR)
- return virt_to_page(ptr);
- if (vaddr < PKMAP_BASE)
- return pfn_to_page(__nocache_pa(vaddr) >> PAGE_SHIFT);
- BUG_ON(vaddr < FIXADDR_START);
- BUG_ON(vaddr > FIXADDR_TOP);
-
- idx = virt_to_fix(vaddr);
- pte = kmap_pte - (idx - FIX_KMAP_BEGIN);
- return pte_page(*pte);
-}
diff --git a/arch/sparc/mm/init_32.c b/arch/sparc/mm/init_32.c
index ef5c779ec855..dde85ef1c56d 100644
--- a/arch/sparc/mm/init_32.c
+++ b/arch/sparc/mm/init_32.c
@@ -45,9 +45,6 @@ unsigned long pfn_base;
EXPORT_SYMBOL(pfn_base);
struct sparc_phys_banks sp_banks[SPARC_PHYS_BANKS+1];
-unsigned long sparc_unmapped_base;
-
-struct pgtable_cache_struct pgt_quicklists;
/* Initial ramdisk setup */
extern unsigned int sparc_ramdisk_image;
@@ -55,19 +52,6 @@ extern unsigned int sparc_ramdisk_size;
unsigned long highstart_pfn, highend_pfn;
-pte_t *kmap_pte;
-pgprot_t kmap_prot;
-
-#define kmap_get_fixmap_pte(vaddr) \
- pte_offset_kernel(pmd_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr))
-
-void __init kmap_init(void)
-{
- /* cache the first kmap pte */
- kmap_pte = kmap_get_fixmap_pte(__fix_to_virt(FIX_KMAP_BEGIN));
- kmap_prot = __pgprot(SRMMU_ET_PTE | SRMMU_PRIV | SRMMU_CACHE);
-}
-
void show_mem(unsigned int filter)
{
printk("Mem-info:\n");
@@ -76,33 +60,8 @@ void show_mem(unsigned int filter)
nr_swap_pages << (PAGE_SHIFT-10));
printk("%ld pages of RAM\n", totalram_pages);
printk("%ld free pages\n", nr_free_pages());
-#if 0 /* undefined pgtable_cache_size, pgd_cache_size */
- printk("%ld pages in page table cache\n",pgtable_cache_size);
-#ifndef CONFIG_SMP
- if (sparc_cpu_model == sun4m || sparc_cpu_model == sun4d)
- printk("%ld entries in page dir cache\n",pgd_cache_size);
-#endif
-#endif
}
-void __init sparc_context_init(int numctx)
-{
- int ctx;
-
- ctx_list_pool = __alloc_bootmem(numctx * sizeof(struct ctx_list), SMP_CACHE_BYTES, 0UL);
-
- for(ctx = 0; ctx < numctx; ctx++) {
- struct ctx_list *clist;
-
- clist = (ctx_list_pool + ctx);
- clist->ctx_number = ctx;
- clist->ctx_mm = NULL;
- }
- ctx_free.next = ctx_free.prev = &ctx_free;
- ctx_used.next = ctx_used.prev = &ctx_used;
- for(ctx = 0; ctx < numctx; ctx++)
- add_to_free_ctxlist(ctx_list_pool + ctx);
-}
extern unsigned long cmdline_memory_size;
unsigned long last_valid_pfn;
@@ -292,22 +251,7 @@ extern void device_scan(void);
void __init paging_init(void)
{
- switch(sparc_cpu_model) {
- case sparc_leon:
- leon_init();
- /* fall through */
- case sun4m:
- case sun4d:
- srmmu_paging_init();
- sparc_unmapped_base = 0x50000000;
- break;
- default:
- prom_printf("paging_init: Cannot init paging on this Sparc\n");
- prom_printf("paging_init: sparc_cpu_model = %d\n", sparc_cpu_model);
- prom_printf("paging_init: Halting...\n");
- prom_halt();
- }
-
+ srmmu_paging_init();
prom_build_devicetree();
of_fill_in_cpu_data();
device_scan();
diff --git a/arch/sparc/mm/srmmu.c b/arch/sparc/mm/srmmu.c
index 62e3f5773303..c38bb72e3e80 100644
--- a/arch/sparc/mm/srmmu.c
+++ b/arch/sparc/mm/srmmu.c
@@ -8,45 +8,45 @@
* Copyright (C) 1999,2000 Anton Blanchard (anton@samba.org)
*/
-#include <linux/kernel.h>
-#include <linux/mm.h>
-#include <linux/vmalloc.h>
-#include <linux/pagemap.h>
-#include <linux/init.h>
+#include <linux/seq_file.h>
#include <linux/spinlock.h>
#include <linux/bootmem.h>
-#include <linux/fs.h>
-#include <linux/seq_file.h>
+#include <linux/pagemap.h>
+#include <linux/vmalloc.h>
#include <linux/kdebug.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
#include <linux/log2.h>
#include <linux/gfp.h>
+#include <linux/fs.h>
+#include <linux/mm.h>
-#include <asm/bitext.h>
-#include <asm/page.h>
+#include <asm/mmu_context.h>
+#include <asm/cacheflush.h>
+#include <asm/tlbflush.h>
+#include <asm/io-unit.h>
#include <asm/pgalloc.h>
#include <asm/pgtable.h>
-#include <asm/io.h>
+#include <asm/bitext.h>
#include <asm/vaddrs.h>
-#include <asm/traps.h>
-#include <asm/smp.h>
-#include <asm/mbus.h>
#include <asm/cache.h>
+#include <asm/traps.h>
#include <asm/oplib.h>
+#include <asm/mbus.h>
+#include <asm/page.h>
#include <asm/asi.h>
#include <asm/msi.h>
-#include <asm/mmu_context.h>
-#include <asm/io-unit.h>
-#include <asm/cacheflush.h>
-#include <asm/tlbflush.h>
+#include <asm/smp.h>
+#include <asm/io.h>
/* Now the cpu specific definitions. */
-#include <asm/viking.h>
-#include <asm/mxcc.h>
-#include <asm/ross.h>
+#include <asm/turbosparc.h>
#include <asm/tsunami.h>
+#include <asm/viking.h>
#include <asm/swift.h>
-#include <asm/turbosparc.h>
#include <asm/leon.h>
+#include <asm/mxcc.h>
+#include <asm/ross.h>
#include "srmmu.h"
@@ -55,10 +55,6 @@ static unsigned int hwbug_bitmask;
int vac_cache_size;
int vac_line_size;
-struct ctx_list *ctx_list_pool;
-struct ctx_list ctx_free;
-struct ctx_list ctx_used;
-
extern struct resource sparc_iomap;
extern unsigned long last_valid_pfn;
@@ -136,8 +132,8 @@ void pmd_populate(struct mm_struct *mm, pmd_t *pmdp, struct page *ptep)
}
}
-/* Find an entry in the third-level page table.. */
-pte_t *pte_offset_kernel(pmd_t * dir, unsigned long address)
+/* Find an entry in the third-level page table.. */
+pte_t *pte_offset_kernel(pmd_t *dir, unsigned long address)
{
void *pte;
@@ -151,55 +147,61 @@ pte_t *pte_offset_kernel(pmd_t * dir, unsigned long address)
* align: bytes, number to align at.
* Returns the virtual address of the allocated area.
*/
-static unsigned long __srmmu_get_nocache(int size, int align)
+static void *__srmmu_get_nocache(int size, int align)
{
int offset;
+ unsigned long addr;
if (size < SRMMU_NOCACHE_BITMAP_SHIFT) {
- printk("Size 0x%x too small for nocache request\n", size);
+ printk(KERN_ERR "Size 0x%x too small for nocache request\n",
+ size);
size = SRMMU_NOCACHE_BITMAP_SHIFT;
}
- if (size & (SRMMU_NOCACHE_BITMAP_SHIFT-1)) {
- printk("Size 0x%x unaligned int nocache request\n", size);
- size += SRMMU_NOCACHE_BITMAP_SHIFT-1;
+ if (size & (SRMMU_NOCACHE_BITMAP_SHIFT - 1)) {
+ printk(KERN_ERR "Size 0x%x unaligned int nocache request\n",
+ size);
+ size += SRMMU_NOCACHE_BITMAP_SHIFT - 1;
}
BUG_ON(align > SRMMU_NOCACHE_ALIGN_MAX);
offset = bit_map_string_get(&srmmu_nocache_map,
- size >> SRMMU_NOCACHE_BITMAP_SHIFT,
- align >> SRMMU_NOCACHE_BITMAP_SHIFT);
+ size >> SRMMU_NOCACHE_BITMAP_SHIFT,
+ align >> SRMMU_NOCACHE_BITMAP_SHIFT);
if (offset == -1) {
- printk("srmmu: out of nocache %d: %d/%d\n",
- size, (int) srmmu_nocache_size,
- srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT);
+ printk(KERN_ERR "srmmu: out of nocache %d: %d/%d\n",
+ size, (int) srmmu_nocache_size,
+ srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT);
return 0;
}
- return (SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT));
+ addr = SRMMU_NOCACHE_VADDR + (offset << SRMMU_NOCACHE_BITMAP_SHIFT);
+ return (void *)addr;
}
-unsigned long srmmu_get_nocache(int size, int align)
+void *srmmu_get_nocache(int size, int align)
{
- unsigned long tmp;
+ void *tmp;
tmp = __srmmu_get_nocache(size, align);
if (tmp)
- memset((void *)tmp, 0, size);
+ memset(tmp, 0, size);
return tmp;
}
-void srmmu_free_nocache(unsigned long vaddr, int size)
+void srmmu_free_nocache(void *addr, int size)
{
+ unsigned long vaddr;
int offset;
+ vaddr = (unsigned long)addr;
if (vaddr < SRMMU_NOCACHE_VADDR) {
printk("Vaddr %lx is smaller than nocache base 0x%lx\n",
vaddr, (unsigned long)SRMMU_NOCACHE_VADDR);
BUG();
}
- if (vaddr+size > srmmu_nocache_end) {
+ if (vaddr + size > srmmu_nocache_end) {
printk("Vaddr %lx is bigger than nocache end 0x%lx\n",
vaddr, srmmu_nocache_end);
BUG();
@@ -212,7 +214,7 @@ void srmmu_free_nocache(unsigned long vaddr, int size)
printk("Size 0x%x is too small\n", size);
BUG();
}
- if (vaddr & (size-1)) {
+ if (vaddr & (size - 1)) {
printk("Vaddr %lx is not aligned to size 0x%x\n", vaddr, size);
BUG();
}
@@ -226,13 +228,23 @@ void srmmu_free_nocache(unsigned long vaddr, int size)
static void srmmu_early_allocate_ptable_skeleton(unsigned long start,
unsigned long end);
-extern unsigned long probe_memory(void); /* in fault.c */
+/* Return how much physical memory we have. */
+static unsigned long __init probe_memory(void)
+{
+ unsigned long total = 0;
+ int i;
+
+ for (i = 0; sp_banks[i].num_bytes; i++)
+ total += sp_banks[i].num_bytes;
+
+ return total;
+}
/*
* Reserve nocache dynamically proportionally to the amount of
* system RAM. -- Tomas Szepe <szepe@pinerecords.com>, June 2002
*/
-static void srmmu_nocache_calcsize(void)
+static void __init srmmu_nocache_calcsize(void)
{
unsigned long sysmemavail = probe_memory() / 1024;
int srmmu_nocache_npages;
@@ -271,7 +283,7 @@ static void __init srmmu_nocache_init(void)
srmmu_nocache_bitmap = __alloc_bootmem(bitmap_bits >> 3, SMP_CACHE_BYTES, 0UL);
bit_map_init(&srmmu_nocache_map, srmmu_nocache_bitmap, bitmap_bits);
- srmmu_swapper_pg_dir = (pgd_t *)__srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
+ srmmu_swapper_pg_dir = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
memset(__nocache_fix(srmmu_swapper_pg_dir), 0, SRMMU_PGD_TABLE_SIZE);
init_mm.pgd = srmmu_swapper_pg_dir;
@@ -304,7 +316,7 @@ pgd_t *get_pgd_fast(void)
{
pgd_t *pgd = NULL;
- pgd = (pgd_t *)__srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
+ pgd = __srmmu_get_nocache(SRMMU_PGD_TABLE_SIZE, SRMMU_PGD_TABLE_SIZE);
if (pgd) {
pgd_t *init = pgd_offset_k(0);
memset(pgd, 0, USER_PTRS_PER_PGD * sizeof(pgd_t));
@@ -330,7 +342,7 @@ pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
if ((pte = (unsigned long)pte_alloc_one_kernel(mm, address)) == 0)
return NULL;
- page = pfn_to_page( __nocache_pa(pte) >> PAGE_SHIFT );
+ page = pfn_to_page(__nocache_pa(pte) >> PAGE_SHIFT);
pgtable_page_ctor(page);
return page;
}
@@ -344,18 +356,50 @@ void pte_free(struct mm_struct *mm, pgtable_t pte)
if (p == 0)
BUG();
p = page_to_pfn(pte) << PAGE_SHIFT; /* Physical address */
- p = (unsigned long) __nocache_va(p); /* Nocached virtual */
- srmmu_free_nocache(p, PTE_SIZE);
+
+ /* free non cached virtual address*/
+ srmmu_free_nocache(__nocache_va(p), PTE_SIZE);
}
-/*
- */
+/* context handling - a dynamically sized pool is used */
+#define NO_CONTEXT -1
+
+struct ctx_list {
+ struct ctx_list *next;
+ struct ctx_list *prev;
+ unsigned int ctx_number;
+ struct mm_struct *ctx_mm;
+};
+
+static struct ctx_list *ctx_list_pool;
+static struct ctx_list ctx_free;
+static struct ctx_list ctx_used;
+
+/* At boot time we determine the number of contexts */
+static int num_contexts;
+
+static inline void remove_from_ctx_list(struct ctx_list *entry)
+{
+ entry->next->prev = entry->prev;
+ entry->prev->next = entry->next;
+}
+
+static inline void add_to_ctx_list(struct ctx_list *head, struct ctx_list *entry)
+{
+ entry->next = head;
+ (entry->prev = head->prev)->next = entry;
+ head->prev = entry;
+}
+#define add_to_free_ctxlist(entry) add_to_ctx_list(&ctx_free, entry)
+#define add_to_used_ctxlist(entry) add_to_ctx_list(&ctx_used, entry)
+
+
static inline void alloc_context(struct mm_struct *old_mm, struct mm_struct *mm)
{
struct ctx_list *ctxp;
ctxp = ctx_free.next;
- if(ctxp != &ctx_free) {
+ if (ctxp != &ctx_free) {
remove_from_ctx_list(ctxp);
add_to_used_ctxlist(ctxp);
mm->context = ctxp->ctx_number;
@@ -363,9 +407,9 @@ static inline void alloc_context(struct mm_struct *old_mm, struct mm_struct *mm)
return;
}
ctxp = ctx_used.next;
- if(ctxp->ctx_mm == old_mm)
+ if (ctxp->ctx_mm == old_mm)
ctxp = ctxp->next;
- if(ctxp == &ctx_used)
+ if (ctxp == &ctx_used)
panic("out of mmu contexts");
flush_cache_mm(ctxp->ctx_mm);
flush_tlb_mm(ctxp->ctx_mm);
@@ -385,11 +429,31 @@ static inline void free_context(int context)
add_to_free_ctxlist(ctx_old);
}
+static void __init sparc_context_init(int numctx)
+{
+ int ctx;
+ unsigned long size;
+
+ size = numctx * sizeof(struct ctx_list);
+ ctx_list_pool = __alloc_bootmem(size, SMP_CACHE_BYTES, 0UL);
+
+ for (ctx = 0; ctx < numctx; ctx++) {
+ struct ctx_list *clist;
+
+ clist = (ctx_list_pool + ctx);
+ clist->ctx_number = ctx;
+ clist->ctx_mm = NULL;
+ }
+ ctx_free.next = ctx_free.prev = &ctx_free;
+ ctx_used.next = ctx_used.prev = &ctx_used;
+ for (ctx = 0; ctx < numctx; ctx++)
+ add_to_free_ctxlist(ctx_list_pool + ctx);
+}
void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm,
struct task_struct *tsk)
{
- if(mm->context == NO_CONTEXT) {
+ if (mm->context == NO_CONTEXT) {
spin_lock(&srmmu_context_spinlock);
alloc_context(old_mm, mm);
spin_unlock(&srmmu_context_spinlock);
@@ -407,7 +471,7 @@ void switch_mm(struct mm_struct *old_mm, struct mm_struct *mm,
/* Low level IO area allocation on the SRMMU. */
static inline void srmmu_mapioaddr(unsigned long physaddr,
- unsigned long virt_addr, int bus_type)
+ unsigned long virt_addr, int bus_type)
{
pgd_t *pgdp;
pmd_t *pmdp;
@@ -420,8 +484,7 @@ static inline void srmmu_mapioaddr(unsigned long physaddr,
ptep = pte_offset_kernel(pmdp, virt_addr);
tmp = (physaddr >> 4) | SRMMU_ET_PTE;
- /*
- * I need to test whether this is consistent over all
+ /* I need to test whether this is consistent over all
* sun4m's. The bus_type represents the upper 4 bits of
* 36-bit physical address on the I/O space lines...
*/
@@ -591,10 +654,10 @@ static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start,
pmd_t *pmdp;
pte_t *ptep;
- while(start < end) {
+ while (start < end) {
pgdp = pgd_offset_k(start);
if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) {
- pmdp = (pmd_t *) __srmmu_get_nocache(
+ pmdp = __srmmu_get_nocache(
SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
if (pmdp == NULL)
early_pgtable_allocfail("pmd");
@@ -602,8 +665,8 @@ static void __init srmmu_early_allocate_ptable_skeleton(unsigned long start,
pgd_set(__nocache_fix(pgdp), pmdp);
}
pmdp = pmd_offset(__nocache_fix(pgdp), start);
- if(srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) {
- ptep = (pte_t *)__srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
+ if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) {
+ ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
if (ptep == NULL)
early_pgtable_allocfail("pte");
memset(__nocache_fix(ptep), 0, PTE_SIZE);
@@ -622,18 +685,18 @@ static void __init srmmu_allocate_ptable_skeleton(unsigned long start,
pmd_t *pmdp;
pte_t *ptep;
- while(start < end) {
+ while (start < end) {
pgdp = pgd_offset_k(start);
if (pgd_none(*pgdp)) {
- pmdp = (pmd_t *)__srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
+ pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
if (pmdp == NULL)
early_pgtable_allocfail("pmd");
memset(pmdp, 0, SRMMU_PMD_TABLE_SIZE);
pgd_set(pgdp, pmdp);
}
pmdp = pmd_offset(pgdp, start);
- if(srmmu_pmd_none(*pmdp)) {
- ptep = (pte_t *) __srmmu_get_nocache(PTE_SIZE,
+ if (srmmu_pmd_none(*pmdp)) {
+ ptep = __srmmu_get_nocache(PTE_SIZE,
PTE_SIZE);
if (ptep == NULL)
early_pgtable_allocfail("pte");
@@ -671,72 +734,76 @@ static inline unsigned long srmmu_probe(unsigned long vaddr)
static void __init srmmu_inherit_prom_mappings(unsigned long start,
unsigned long end)
{
+ unsigned long probed;
+ unsigned long addr;
pgd_t *pgdp;
pmd_t *pmdp;
pte_t *ptep;
- int what = 0; /* 0 = normal-pte, 1 = pmd-level pte, 2 = pgd-level pte */
- unsigned long prompte;
+ int what; /* 0 = normal-pte, 1 = pmd-level pte, 2 = pgd-level pte */
- while(start <= end) {
+ while (start <= end) {
if (start == 0)
break; /* probably wrap around */
- if(start == 0xfef00000)
+ if (start == 0xfef00000)
start = KADB_DEBUGGER_BEGVM;
- if(!(prompte = srmmu_probe(start))) {
+ probed = srmmu_probe(start);
+ if (!probed) {
+ /* continue probing until we find an entry */
start += PAGE_SIZE;
continue;
}
-
+
/* A red snapper, see what it really is. */
what = 0;
-
- if(!(start & ~(SRMMU_REAL_PMD_MASK))) {
- if(srmmu_probe((start-PAGE_SIZE) + SRMMU_REAL_PMD_SIZE) == prompte)
+ addr = start - PAGE_SIZE;
+
+ if (!(start & ~(SRMMU_REAL_PMD_MASK))) {
+ if (srmmu_probe(addr + SRMMU_REAL_PMD_SIZE) == probed)
what = 1;
}
-
- if(!(start & ~(SRMMU_PGDIR_MASK))) {
- if(srmmu_probe((start-PAGE_SIZE) + SRMMU_PGDIR_SIZE) ==
- prompte)
+
+ if (!(start & ~(SRMMU_PGDIR_MASK))) {
+ if (srmmu_probe(addr + SRMMU_PGDIR_SIZE) == probed)
what = 2;
}
-
+
pgdp = pgd_offset_k(start);
- if(what == 2) {
- *(pgd_t *)__nocache_fix(pgdp) = __pgd(prompte);
+ if (what == 2) {
+ *(pgd_t *)__nocache_fix(pgdp) = __pgd(probed);
start += SRMMU_PGDIR_SIZE;
continue;
}
if (pgd_none(*(pgd_t *)__nocache_fix(pgdp))) {
- pmdp = (pmd_t *)__srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE, SRMMU_PMD_TABLE_SIZE);
+ pmdp = __srmmu_get_nocache(SRMMU_PMD_TABLE_SIZE,
+ SRMMU_PMD_TABLE_SIZE);
if (pmdp == NULL)
early_pgtable_allocfail("pmd");
memset(__nocache_fix(pmdp), 0, SRMMU_PMD_TABLE_SIZE);
pgd_set(__nocache_fix(pgdp), pmdp);
}
pmdp = pmd_offset(__nocache_fix(pgdp), start);
- if(srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) {
- ptep = (pte_t *) __srmmu_get_nocache(PTE_SIZE,
- PTE_SIZE);
+ if (srmmu_pmd_none(*(pmd_t *)__nocache_fix(pmdp))) {
+ ptep = __srmmu_get_nocache(PTE_SIZE, PTE_SIZE);
if (ptep == NULL)
early_pgtable_allocfail("pte");
memset(__nocache_fix(ptep), 0, PTE_SIZE);
pmd_set(__nocache_fix(pmdp), ptep);
}
- if(what == 1) {
- /*
- * We bend the rule where all 16 PTPs in a pmd_t point
+ if (what == 1) {
+ /* We bend the rule where all 16 PTPs in a pmd_t point
* inside the same PTE page, and we leak a perfectly
* good hardware PTE piece. Alternatives seem worse.
*/
unsigned int x; /* Index of HW PMD in soft cluster */
+ unsigned long *val;
x = (start >> PMD_SHIFT) & 15;
- *(unsigned long *)__nocache_fix(&pmdp->pmdv[x]) = prompte;
+ val = &pmdp->pmdv[x];
+ *(unsigned long *)__nocache_fix(val) = probed;
start += SRMMU_REAL_PMD_SIZE;
continue;
}
ptep = pte_offset_kernel(__nocache_fix(pmdp), start);
- *(pte_t *)__nocache_fix(ptep) = __pte(prompte);
+ *(pte_t *)__nocache_fix(ptep) = __pte(probed);
start += PAGE_SIZE;
}
}
@@ -765,18 +832,18 @@ static unsigned long __init map_spbank(unsigned long vbase, int sp_entry)
if (vstart < min_vaddr || vstart >= max_vaddr)
return vstart;
-
+
if (vend > max_vaddr || vend < min_vaddr)
vend = max_vaddr;
- while(vstart < vend) {
+ while (vstart < vend) {
do_large_mapping(vstart, pstart);
vstart += SRMMU_PGDIR_SIZE; pstart += SRMMU_PGDIR_SIZE;
}
return vstart;
}
-static inline void map_kernel(void)
+static void __init map_kernel(void)
{
int i;
@@ -789,9 +856,6 @@ static inline void map_kernel(void)
}
}
-/* Paging initialization on the Sparc Reference MMU. */
-extern void sparc_context_init(int);
-
void (*poke_srmmu)(void) __cpuinitdata = NULL;
extern unsigned long bootmem_init(unsigned long *pages_avail);
@@ -806,6 +870,7 @@ void __init srmmu_paging_init(void)
pte_t *pte;
unsigned long pages_avail;
+ init_mm.context = (unsigned long) NO_CONTEXT;
sparc_iomap.start = SUN4M_IOBASE_VADDR; /* 16MB of IOSPACE on all sun4m's. */
if (sparc_cpu_model == sun4d)
@@ -814,9 +879,9 @@ void __init srmmu_paging_init(void)
/* Find the number of contexts on the srmmu. */
cpunode = prom_getchild(prom_root_node);
num_contexts = 0;
- while(cpunode != 0) {
+ while (cpunode != 0) {
prom_getstring(cpunode, "device_type", node_str, sizeof(node_str));
- if(!strcmp(node_str, "cpu")) {
+ if (!strcmp(node_str, "cpu")) {
num_contexts = prom_getintdefault(cpunode, "mmu-nctx", 0x8);
break;
}
@@ -824,7 +889,7 @@ void __init srmmu_paging_init(void)
}
}
- if(!num_contexts) {
+ if (!num_contexts) {
prom_printf("Something wrong, can't find cpu node in paging_init.\n");
prom_halt();
}
@@ -834,14 +899,14 @@ void __init srmmu_paging_init(void)
srmmu_nocache_calcsize();
srmmu_nocache_init();
- srmmu_inherit_prom_mappings(0xfe400000,(LINUX_OPPROM_ENDVM-PAGE_SIZE));
+ srmmu_inherit_prom_mappings(0xfe400000, (LINUX_OPPROM_ENDVM - PAGE_SIZE));
map_kernel();
/* ctx table has to be physically aligned to its size */
- srmmu_context_table = (ctxd_t *)__srmmu_get_nocache(num_contexts*sizeof(ctxd_t), num_contexts*sizeof(ctxd_t));
+ srmmu_context_table = __srmmu_get_nocache(num_contexts * sizeof(ctxd_t), num_contexts * sizeof(ctxd_t));
srmmu_ctx_table_phys = (ctxd_t *)__nocache_pa((unsigned long)srmmu_context_table);
- for(i = 0; i < num_contexts; i++)
+ for (i = 0; i < num_contexts; i++)
srmmu_ctxd_set((ctxd_t *)__nocache_fix(&srmmu_context_table[i]), srmmu_swapper_pg_dir);
flush_cache_all();
@@ -897,7 +962,7 @@ void __init srmmu_paging_init(void)
void mmu_info(struct seq_file *m)
{
- seq_printf(m,
+ seq_printf(m,
"MMU type\t: %s\n"
"contexts\t: %d\n"
"nocache total\t: %ld\n"
@@ -908,10 +973,16 @@ void mmu_info(struct seq_file *m)
srmmu_nocache_map.used << SRMMU_NOCACHE_BITMAP_SHIFT);
}
+int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
+{
+ mm->context = NO_CONTEXT;
+ return 0;
+}
+
void destroy_context(struct mm_struct *mm)
{
- if(mm->context != NO_CONTEXT) {
+ if (mm->context != NO_CONTEXT) {
flush_cache_mm(mm);
srmmu_ctxd_set(&srmmu_context_table[mm->context], srmmu_swapper_pg_dir);
flush_tlb_mm(mm);
@@ -941,13 +1012,12 @@ static void __init init_vac_layout(void)
#endif
nd = prom_getchild(prom_root_node);
- while((nd = prom_getsibling(nd)) != 0) {
+ while ((nd = prom_getsibling(nd)) != 0) {
prom_getstring(nd, "device_type", node_str, sizeof(node_str));
- if(!strcmp(node_str, "cpu")) {
+ if (!strcmp(node_str, "cpu")) {
vac_line_size = prom_getint(nd, "cache-line-size");
if (vac_line_size == -1) {
- prom_printf("can't determine cache-line-size, "
- "halting.\n");
+ prom_printf("can't determine cache-line-size, halting.\n");
prom_halt();
}
cache_lines = prom_getint(nd, "cache-nlines");
@@ -958,9 +1028,9 @@ static void __init init_vac_layout(void)
vac_cache_size = cache_lines * vac_line_size;
#ifdef CONFIG_SMP
- if(vac_cache_size > max_size)
+ if (vac_cache_size > max_size)
max_size = vac_cache_size;
- if(vac_line_size < min_line_size)
+ if (vac_line_size < min_line_size)
min_line_size = vac_line_size;
//FIXME: cpus not contiguous!!
cpu++;
@@ -971,7 +1041,7 @@ static void __init init_vac_layout(void)
#endif
}
}
- if(nd == 0) {
+ if (nd == 0) {
prom_printf("No CPU nodes found, halting.\n");
prom_halt();
}
@@ -1082,7 +1152,7 @@ static void __init init_swift(void)
"=r" (swift_rev) :
"r" (SWIFT_MASKID_ADDR), "i" (ASI_M_BYPASS));
srmmu_name = "Fujitsu Swift";
- switch(swift_rev) {
+ switch (swift_rev) {
case 0x11:
case 0x20:
case 0x23:
@@ -1222,10 +1292,11 @@ static void __cpuinit poke_turbosparc(void)
/* Clear any crap from the cache or else... */
turbosparc_flush_cache_all();
- mreg &= ~(TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* Temporarily disable I & D caches */
+ /* Temporarily disable I & D caches */
+ mreg &= ~(TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE);
mreg &= ~(TURBOSPARC_PCENABLE); /* Don't check parity */
srmmu_set_mmureg(mreg);
-
+
ccreg = turbosparc_get_ccreg();
#ifdef TURBOSPARC_WRITEBACK
@@ -1248,7 +1319,7 @@ static void __cpuinit poke_turbosparc(void)
default:
ccreg |= (TURBOSPARC_SCENABLE);
}
- turbosparc_set_ccreg (ccreg);
+ turbosparc_set_ccreg(ccreg);
mreg |= (TURBOSPARC_ICENABLE | TURBOSPARC_DCENABLE); /* I & D caches on */
mreg |= (TURBOSPARC_ICSNOOP); /* Icache snooping on */
@@ -1342,7 +1413,7 @@ static void __cpuinit poke_viking(void)
unsigned long bpreg;
mreg &= ~(VIKING_TCENABLE);
- if(smp_catch++) {
+ if (smp_catch++) {
/* Must disable mixed-cmd mode here for other cpu's. */
bpreg = viking_get_bpreg();
bpreg &= ~(VIKING_ACTION_MIX);
@@ -1411,7 +1482,7 @@ static void __init init_viking(void)
unsigned long mreg = srmmu_get_mmureg();
/* Ahhh, the viking. SRMMU VLSI abortion number two... */
- if(mreg & VIKING_MMODE) {
+ if (mreg & VIKING_MMODE) {
srmmu_name = "TI Viking";
viking_mxcc_present = 0;
msi_set_sync();
@@ -1467,8 +1538,8 @@ static void __init get_srmmu_type(void)
}
/* Second, check for HyperSparc or Cypress. */
- if(mod_typ == 1) {
- switch(mod_rev) {
+ if (mod_typ == 1) {
+ switch (mod_rev) {
case 7:
/* UP or MP Hypersparc */
init_hypersparc();
@@ -1488,9 +1559,8 @@ static void __init get_srmmu_type(void)
}
return;
}
-
- /*
- * Now Fujitsu TurboSparc. It might happen that it is
+
+ /* Now Fujitsu TurboSparc. It might happen that it is
* in Swift emulation mode, so we will check later...
*/
if (psr_typ == 0 && psr_vers == 5) {
@@ -1499,15 +1569,15 @@ static void __init get_srmmu_type(void)
}
/* Next check for Fujitsu Swift. */
- if(psr_typ == 0 && psr_vers == 4) {
+ if (psr_typ == 0 && psr_vers == 4) {
phandle cpunode;
char node_str[128];
/* Look if it is not a TurboSparc emulating Swift... */
cpunode = prom_getchild(prom_root_node);
- while((cpunode = prom_getsibling(cpunode)) != 0) {
+ while ((cpunode = prom_getsibling(cpunode)) != 0) {
prom_getstring(cpunode, "device_type", node_str, sizeof(node_str));
- if(!strcmp(node_str, "cpu")) {
+ if (!strcmp(node_str, "cpu")) {
if (!prom_getintdefault(cpunode, "psr-implementation", 1) &&
prom_getintdefault(cpunode, "psr-version", 1) == 5) {
init_turbosparc();
@@ -1516,13 +1586,13 @@ static void __init get_srmmu_type(void)
break;
}
}
-
+
init_swift();
return;
}
/* Now the Viking family of srmmu. */
- if(psr_typ == 4 &&
+ if (psr_typ == 4 &&
((psr_vers == 0) ||
((psr_vers == 1) && (mod_typ == 0) && (mod_rev == 0)))) {
init_viking();
@@ -1530,7 +1600,7 @@ static void __init get_srmmu_type(void)
}
/* Finally the Tsunami. */
- if(psr_typ == 4 && psr_vers == 1 && (mod_typ || mod_rev)) {
+ if (psr_typ == 4 && psr_vers == 1 && (mod_typ || mod_rev)) {
init_tsunami();
return;
}
diff --git a/arch/sparc/prom/init_32.c b/arch/sparc/prom/init_32.c
index 26c64cea3c9c..9ac30c2b7dba 100644
--- a/arch/sparc/prom/init_32.c
+++ b/arch/sparc/prom/init_32.c
@@ -27,13 +27,10 @@ EXPORT_SYMBOL(prom_root_node);
struct linux_nodeops *prom_nodeops;
/* You must call prom_init() before you attempt to use any of the
- * routines in the prom library. It returns 0 on success, 1 on
- * failure. It gets passed the pointer to the PROM vector.
+ * routines in the prom library.
+ * It gets passed the pointer to the PROM vector.
*/
-extern void prom_meminit(void);
-extern void prom_ranges_init(void);
-
void __init prom_init(struct linux_romvec *rp)
{
romvec = rp;
diff --git a/arch/sparc/prom/init_64.c b/arch/sparc/prom/init_64.c
index 5016c5e20575..d95db755828f 100644
--- a/arch/sparc/prom/init_64.c
+++ b/arch/sparc/prom/init_64.c
@@ -22,8 +22,8 @@ int prom_stdout;
phandle prom_chosen_node;
/* You must call prom_init() before you attempt to use any of the
- * routines in the prom library. It returns 0 on success, 1 on
- * failure. It gets passed the pointer to the PROM vector.
+ * routines in the prom library.
+ * It gets passed the pointer to the PROM vector.
*/
extern void prom_cif_init(void *, void *);
diff --git a/arch/tile/kernel/pci.c b/arch/tile/kernel/pci.c
index 0fdd99d0d8b7..33c10864d2f7 100644
--- a/arch/tile/kernel/pci.c
+++ b/arch/tile/kernel/pci.c
@@ -369,7 +369,7 @@ int __init pcibios_init(void)
*/
if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI &&
(PCI_SLOT(dev->devfn) == 0)) {
- next_bus = dev->busn_res.end;
+ next_bus = dev->subordinate;
controllers[i].mem_resources[0] =
*next_bus->resource[0];
controllers[i].mem_resources[1] =
diff --git a/arch/tile/kernel/pci_gx.c b/arch/tile/kernel/pci_gx.c
index fa75264a82ae..0e213e35ffc3 100644
--- a/arch/tile/kernel/pci_gx.c
+++ b/arch/tile/kernel/pci_gx.c
@@ -853,7 +853,7 @@ int __init pcibios_init(void)
bus = pci_scan_root_bus(NULL, next_busno, controller->ops,
controller, &resources);
controller->root_bus = bus;
- next_busno = bus->subordinate + 1;
+ next_busno = bus->busn_res.end + 1;
}
diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug
index e46c2147397f..b322f124ee3c 100644
--- a/arch/x86/Kconfig.debug
+++ b/arch/x86/Kconfig.debug
@@ -129,6 +129,25 @@ config DOUBLEFAULT
option saves about 4k and might cause you much additional grey
hair.
+config DEBUG_TLBFLUSH
+ bool "Set upper limit of TLB entries to flush one-by-one"
+ depends on DEBUG_KERNEL && (X86_64 || X86_INVLPG)
+ ---help---
+
+ X86-only for now.
+
+ This option allows the user to tune the amount of TLB entries the
+ kernel flushes one-by-one instead of doing a full TLB flush. In
+ certain situations, the former is cheaper. This is controlled by the
+ tlb_flushall_shift knob under /sys/kernel/debug/x86. If you set it
+ to -1, the code flushes the whole TLB unconditionally. Otherwise,
+ for positive values of it, the kernel will use single TLB entry
+ invalidating instructions according to the following formula:
+
+ flush_entries <= active_tlb_entries / 2^tlb_flushall_shift
+
+ If in doubt, say "N".
+
config IOMMU_DEBUG
bool "Enable IOMMU debugging"
depends on GART_IOMMU && DEBUG_KERNEL
diff --git a/arch/x86/boot/compressed/cmdline.c b/arch/x86/boot/compressed/cmdline.c
index cb62f786990d..10f6b1178c68 100644
--- a/arch/x86/boot/compressed/cmdline.c
+++ b/arch/x86/boot/compressed/cmdline.c
@@ -1,5 +1,7 @@
#include "misc.h"
+#ifdef CONFIG_EARLY_PRINTK
+
static unsigned long fs;
static inline void set_fs(unsigned long seg)
{
@@ -19,3 +21,5 @@ int cmdline_find_option_bool(const char *option)
{
return __cmdline_find_option_bool(real_mode->hdr.cmd_line_ptr, option);
}
+
+#endif
diff --git a/arch/x86/boot/compressed/early_serial_console.c b/arch/x86/boot/compressed/early_serial_console.c
index 261e81fb9582..d3d003cb5481 100644
--- a/arch/x86/boot/compressed/early_serial_console.c
+++ b/arch/x86/boot/compressed/early_serial_console.c
@@ -1,5 +1,9 @@
#include "misc.h"
+#ifdef CONFIG_EARLY_PRINTK
+
int early_serial_base;
#include "../early_serial_console.c"
+
+#endif
diff --git a/arch/x86/boot/compressed/eboot.c b/arch/x86/boot/compressed/eboot.c
index 4e85f5f85837..b3e0227df2c9 100644
--- a/arch/x86/boot/compressed/eboot.c
+++ b/arch/x86/boot/compressed/eboot.c
@@ -729,32 +729,68 @@ fail:
* need to create one ourselves (usually the bootloader would create
* one for us).
*/
-static efi_status_t make_boot_params(struct boot_params *boot_params,
- efi_loaded_image_t *image,
- void *handle)
+struct boot_params *make_boot_params(void *handle, efi_system_table_t *_table)
{
- struct efi_info *efi = &boot_params->efi_info;
- struct apm_bios_info *bi = &boot_params->apm_bios_info;
- struct sys_desc_table *sdt = &boot_params->sys_desc_table;
- struct e820entry *e820_map = &boot_params->e820_map[0];
- struct e820entry *prev = NULL;
- struct setup_header *hdr = &boot_params->hdr;
- unsigned long size, key, desc_size, _size;
- efi_memory_desc_t *mem_map;
- void *options = image->load_options;
- u32 load_options_size = image->load_options_size / 2; /* ASCII */
+ struct boot_params *boot_params;
+ struct sys_desc_table *sdt;
+ struct apm_bios_info *bi;
+ struct setup_header *hdr;
+ struct efi_info *efi;
+ efi_loaded_image_t *image;
+ void *options;
+ u32 load_options_size;
+ efi_guid_t proto = LOADED_IMAGE_PROTOCOL_GUID;
int options_size = 0;
efi_status_t status;
- __u32 desc_version;
unsigned long cmdline;
- u8 nr_entries;
u16 *s2;
u8 *s1;
int i;
+ sys_table = _table;
+
+ /* Check if we were booted by the EFI firmware */
+ if (sys_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE)
+ return NULL;
+
+ status = efi_call_phys3(sys_table->boottime->handle_protocol,
+ handle, &proto, (void *)&image);
+ if (status != EFI_SUCCESS) {
+ efi_printk("Failed to get handle for LOADED_IMAGE_PROTOCOL\n");
+ return NULL;
+ }
+
+ status = low_alloc(0x4000, 1, (unsigned long *)&boot_params);
+ if (status != EFI_SUCCESS) {
+ efi_printk("Failed to alloc lowmem for boot params\n");
+ return NULL;
+ }
+
+ memset(boot_params, 0x0, 0x4000);
+
+ hdr = &boot_params->hdr;
+ efi = &boot_params->efi_info;
+ bi = &boot_params->apm_bios_info;
+ sdt = &boot_params->sys_desc_table;
+
+ /* Copy the second sector to boot_params */
+ memcpy(&hdr->jump, image->image_base + 512, 512);
+
+ /*
+ * Fill out some of the header fields ourselves because the
+ * EFI firmware loader doesn't load the first sector.
+ */
+ hdr->root_flags = 1;
+ hdr->vid_mode = 0xffff;
+ hdr->boot_flag = 0xAA55;
+
+ hdr->code32_start = (__u64)(unsigned long)image->image_base;
+
hdr->type_of_loader = 0x21;
/* Convert unicode cmdline to ascii */
+ options = image->load_options;
+ load_options_size = image->load_options_size / 2; /* ASCII */
cmdline = 0;
s2 = (u16 *)options;
@@ -791,18 +827,36 @@ static efi_status_t make_boot_params(struct boot_params *boot_params,
hdr->ramdisk_image = 0;
hdr->ramdisk_size = 0;
- status = handle_ramdisks(image, hdr);
- if (status != EFI_SUCCESS)
- goto free_cmdline;
-
- setup_graphics(boot_params);
-
/* Clear APM BIOS info */
memset(bi, 0, sizeof(*bi));
memset(sdt, 0, sizeof(*sdt));
- memcpy(&efi->efi_loader_signature, EFI_LOADER_SIGNATURE, sizeof(__u32));
+ status = handle_ramdisks(image, hdr);
+ if (status != EFI_SUCCESS)
+ goto fail2;
+
+ return boot_params;
+fail2:
+ if (options_size)
+ low_free(options_size, hdr->cmd_line_ptr);
+fail:
+ low_free(0x4000, (unsigned long)boot_params);
+ return NULL;
+}
+
+static efi_status_t exit_boot(struct boot_params *boot_params,
+ void *handle)
+{
+ struct efi_info *efi = &boot_params->efi_info;
+ struct e820entry *e820_map = &boot_params->e820_map[0];
+ struct e820entry *prev = NULL;
+ unsigned long size, key, desc_size, _size;
+ efi_memory_desc_t *mem_map;
+ efi_status_t status;
+ __u32 desc_version;
+ u8 nr_entries;
+ int i;
size = sizeof(*mem_map) * 32;
@@ -811,7 +865,7 @@ again:
_size = size;
status = low_alloc(size, 1, (unsigned long *)&mem_map);
if (status != EFI_SUCCESS)
- goto free_cmdline;
+ return status;
status = efi_call_phys5(sys_table->boottime->get_memory_map, &size,
mem_map, &key, &desc_size, &desc_version);
@@ -823,6 +877,7 @@ again:
if (status != EFI_SUCCESS)
goto free_mem_map;
+ memcpy(&efi->efi_loader_signature, EFI_LOADER_SIGNATURE, sizeof(__u32));
efi->efi_systab = (unsigned long)sys_table;
efi->efi_memdesc_size = desc_size;
efi->efi_memdesc_version = desc_version;
@@ -906,61 +961,13 @@ again:
free_mem_map:
low_free(_size, (unsigned long)mem_map);
-free_cmdline:
- if (options_size)
- low_free(options_size, hdr->cmd_line_ptr);
-fail:
return status;
}
-/*
- * On success we return a pointer to a boot_params structure, and NULL
- * on failure.
- */
-struct boot_params *efi_main(void *handle, efi_system_table_t *_table)
+static efi_status_t relocate_kernel(struct setup_header *hdr)
{
- struct boot_params *boot_params;
unsigned long start, nr_pages;
- struct desc_ptr *gdt, *idt;
- efi_loaded_image_t *image;
- struct setup_header *hdr;
efi_status_t status;
- efi_guid_t proto = LOADED_IMAGE_PROTOCOL_GUID;
- struct desc_struct *desc;
-
- sys_table = _table;
-
- /* Check if we were booted by the EFI firmware */
- if (sys_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE)
- goto fail;
-
- status = efi_call_phys3(sys_table->boottime->handle_protocol,
- handle, &proto, (void *)&image);
- if (status != EFI_SUCCESS) {
- efi_printk("Failed to get handle for LOADED_IMAGE_PROTOCOL\n");
- goto fail;
- }
-
- status = low_alloc(0x4000, 1, (unsigned long *)&boot_params);
- if (status != EFI_SUCCESS) {
- efi_printk("Failed to alloc lowmem for boot params\n");
- goto fail;
- }
-
- memset(boot_params, 0x0, 0x4000);
-
- hdr = &boot_params->hdr;
-
- /* Copy the second sector to boot_params */
- memcpy(&hdr->jump, image->image_base + 512, 512);
-
- /*
- * Fill out some of the header fields ourselves because the
- * EFI firmware loader doesn't load the first sector.
- */
- hdr->root_flags = 1;
- hdr->vid_mode = 0xffff;
- hdr->boot_flag = 0xAA55;
/*
* The EFI firmware loader could have placed the kernel image
@@ -978,16 +985,40 @@ struct boot_params *efi_main(void *handle, efi_system_table_t *_table)
if (status != EFI_SUCCESS) {
status = low_alloc(hdr->init_size, hdr->kernel_alignment,
&start);
- if (status != EFI_SUCCESS) {
+ if (status != EFI_SUCCESS)
efi_printk("Failed to alloc mem for kernel\n");
- goto fail;
- }
}
+ if (status == EFI_SUCCESS)
+ memcpy((void *)start, (void *)(unsigned long)hdr->code32_start,
+ hdr->init_size);
+
+ hdr->pref_address = hdr->code32_start;
hdr->code32_start = (__u32)start;
- hdr->pref_address = (__u64)(unsigned long)image->image_base;
- memcpy((void *)start, image->image_base, image->image_size);
+ return status;
+}
+
+/*
+ * On success we return a pointer to a boot_params structure, and NULL
+ * on failure.
+ */
+struct boot_params *efi_main(void *handle, efi_system_table_t *_table,
+ struct boot_params *boot_params)
+{
+ struct desc_ptr *gdt, *idt;
+ efi_loaded_image_t *image;
+ struct setup_header *hdr = &boot_params->hdr;
+ efi_status_t status;
+ struct desc_struct *desc;
+
+ sys_table = _table;
+
+ /* Check if we were booted by the EFI firmware */
+ if (sys_table->hdr.signature != EFI_SYSTEM_TABLE_SIGNATURE)
+ goto fail;
+
+ setup_graphics(boot_params);
status = efi_call_phys3(sys_table->boottime->allocate_pool,
EFI_LOADER_DATA, sizeof(*gdt),
@@ -1015,7 +1046,18 @@ struct boot_params *efi_main(void *handle, efi_system_table_t *_table)
idt->size = 0;
idt->address = 0;
- status = make_boot_params(boot_params, image, handle);
+ /*
+ * If the kernel isn't already loaded at the preferred load
+ * address, relocate it.
+ */
+ if (hdr->pref_address != hdr->code32_start) {
+ status = relocate_kernel(hdr);
+
+ if (status != EFI_SUCCESS)
+ goto fail;
+ }
+
+ status = exit_boot(boot_params, handle);
if (status != EFI_SUCCESS)
goto fail;
diff --git a/arch/x86/boot/compressed/head_32.S b/arch/x86/boot/compressed/head_32.S
index c85e3ac99bba..aa4aaf1b2380 100644
--- a/arch/x86/boot/compressed/head_32.S
+++ b/arch/x86/boot/compressed/head_32.S
@@ -42,6 +42,16 @@ ENTRY(startup_32)
*/
add $0x4, %esp
+ call make_boot_params
+ cmpl $0, %eax
+ je 1f
+ movl 0x4(%esp), %esi
+ movl (%esp), %ecx
+ pushl %eax
+ pushl %esi
+ pushl %ecx
+
+ .org 0x30,0x90
call efi_main
cmpl $0, %eax
movl %eax, %esi
diff --git a/arch/x86/boot/compressed/head_64.S b/arch/x86/boot/compressed/head_64.S
index 87e03a13d8e3..2c4b171eec33 100644
--- a/arch/x86/boot/compressed/head_64.S
+++ b/arch/x86/boot/compressed/head_64.S
@@ -209,6 +209,16 @@ ENTRY(startup_64)
.org 0x210
mov %rcx, %rdi
mov %rdx, %rsi
+ pushq %rdi
+ pushq %rsi
+ call make_boot_params
+ cmpq $0,%rax
+ je 1f
+ mov %rax, %rdx
+ popq %rsi
+ popq %rdi
+
+ .org 0x230,0x90
call efi_main
movq %rax,%rsi
cmpq $0,%rax
diff --git a/arch/x86/boot/compressed/misc.c b/arch/x86/boot/compressed/misc.c
index 7116dcba0c9e..88f7ff6da404 100644
--- a/arch/x86/boot/compressed/misc.c
+++ b/arch/x86/boot/compressed/misc.c
@@ -108,8 +108,6 @@ static void error(char *m);
* This is set up by the setup-routine at boot-time
*/
struct boot_params *real_mode; /* Pointer to real-mode data */
-static int quiet;
-static int debug;
void *memset(void *s, int c, size_t n);
void *memcpy(void *dest, const void *src, size_t n);
@@ -170,15 +168,11 @@ static void serial_putchar(int ch)
outb(ch, early_serial_base + TXR);
}
-void __putstr(int error, const char *s)
+void __putstr(const char *s)
{
int x, y, pos;
char c;
-#ifndef CONFIG_X86_VERBOSE_BOOTUP
- if (!error)
- return;
-#endif
if (early_serial_base) {
const char *str = s;
while (*str) {
@@ -265,9 +259,9 @@ void *memcpy(void *dest, const void *src, size_t n)
static void error(char *x)
{
- __putstr(1, "\n\n");
- __putstr(1, x);
- __putstr(1, "\n\n -- System halted");
+ error_putstr("\n\n");
+ error_putstr(x);
+ error_putstr("\n\n -- System halted");
while (1)
asm("hlt");
@@ -294,8 +288,7 @@ static void parse_elf(void *output)
return;
}
- if (!quiet)
- putstr("Parsing ELF... ");
+ debug_putstr("Parsing ELF... ");
phdrs = malloc(sizeof(*phdrs) * ehdr.e_phnum);
if (!phdrs)
@@ -332,11 +325,6 @@ asmlinkage void decompress_kernel(void *rmode, memptr heap,
{
real_mode = rmode;
- if (cmdline_find_option_bool("quiet"))
- quiet = 1;
- if (cmdline_find_option_bool("debug"))
- debug = 1;
-
if (real_mode->screen_info.orig_video_mode == 7) {
vidmem = (char *) 0xb0000;
vidport = 0x3b4;
@@ -349,8 +337,7 @@ asmlinkage void decompress_kernel(void *rmode, memptr heap,
cols = real_mode->screen_info.orig_video_cols;
console_init();
- if (debug)
- putstr("early console in decompress_kernel\n");
+ debug_putstr("early console in decompress_kernel\n");
free_mem_ptr = heap; /* Heap */
free_mem_end_ptr = heap + BOOT_HEAP_SIZE;
@@ -369,11 +356,9 @@ asmlinkage void decompress_kernel(void *rmode, memptr heap,
error("Wrong destination address");
#endif
- if (!quiet)
- putstr("\nDecompressing Linux... ");
+ debug_putstr("\nDecompressing Linux... ");
decompress(input_data, input_len, NULL, NULL, output, NULL, error);
parse_elf(output);
- if (!quiet)
- putstr("done.\nBooting the kernel.\n");
+ debug_putstr("done.\nBooting the kernel.\n");
return;
}
diff --git a/arch/x86/boot/compressed/misc.h b/arch/x86/boot/compressed/misc.h
index 3f19c81a6203..0e6dc0ee0eea 100644
--- a/arch/x86/boot/compressed/misc.h
+++ b/arch/x86/boot/compressed/misc.h
@@ -24,9 +24,21 @@
/* misc.c */
extern struct boot_params *real_mode; /* Pointer to real-mode data */
-void __putstr(int error, const char *s);
-#define putstr(__x) __putstr(0, __x)
-#define puts(__x) __putstr(0, __x)
+void __putstr(const char *s);
+#define error_putstr(__x) __putstr(__x)
+
+#ifdef CONFIG_X86_VERBOSE_BOOTUP
+
+#define debug_putstr(__x) __putstr(__x)
+
+#else
+
+static inline void debug_putstr(const char *s)
+{ }
+
+#endif
+
+#ifdef CONFIG_EARLY_PRINTK
/* cmdline.c */
int cmdline_find_option(const char *option, char *buffer, int bufsize);
@@ -36,4 +48,13 @@ int cmdline_find_option_bool(const char *option);
extern int early_serial_base;
void console_init(void);
+#else
+
+/* early_serial_console.c */
+static const int early_serial_base;
+static inline void console_init(void)
+{ }
+
+#endif
+
#endif
diff --git a/arch/x86/boot/header.S b/arch/x86/boot/header.S
index efe5acfc79c3..b4e15dd6786a 100644
--- a/arch/x86/boot/header.S
+++ b/arch/x86/boot/header.S
@@ -283,7 +283,7 @@ _start:
# Part 2 of the header, from the old setup.S
.ascii "HdrS" # header signature
- .word 0x020a # header version number (>= 0x0105)
+ .word 0x020b # header version number (>= 0x0105)
# or else old loadlin-1.5 will fail)
.globl realmode_swtch
realmode_swtch: .word 0, 0 # default_switch, SETUPSEG
@@ -401,18 +401,13 @@ pref_address: .quad LOAD_PHYSICAL_ADDR # preferred load addr
#define INIT_SIZE VO_INIT_SIZE
#endif
init_size: .long INIT_SIZE # kernel initialization size
+handover_offset: .long 0x30 # offset to the handover
+ # protocol entry point
# End of setup header #####################################################
.section ".entrytext", "ax"
start_of_setup:
-#ifdef SAFE_RESET_DISK_CONTROLLER
-# Reset the disk controller.
- movw $0x0000, %ax # Reset disk controller
- movb $0x80, %dl # All disks
- int $0x13
-#endif
-
# Force %es = %ds
movw %ds, %ax
movw %ax, %es
diff --git a/arch/x86/crypto/Makefile b/arch/x86/crypto/Makefile
index e191ac048b59..e908e5de82d3 100644
--- a/arch/x86/crypto/Makefile
+++ b/arch/x86/crypto/Makefile
@@ -2,6 +2,9 @@
# Arch-specific CryptoAPI modules.
#
+obj-$(CONFIG_CRYPTO_ABLK_HELPER_X86) += ablk_helper.o
+obj-$(CONFIG_CRYPTO_GLUE_HELPER_X86) += glue_helper.o
+
obj-$(CONFIG_CRYPTO_AES_586) += aes-i586.o
obj-$(CONFIG_CRYPTO_TWOFISH_586) += twofish-i586.o
obj-$(CONFIG_CRYPTO_SALSA20_586) += salsa20-i586.o
@@ -12,8 +15,10 @@ obj-$(CONFIG_CRYPTO_CAMELLIA_X86_64) += camellia-x86_64.o
obj-$(CONFIG_CRYPTO_BLOWFISH_X86_64) += blowfish-x86_64.o
obj-$(CONFIG_CRYPTO_TWOFISH_X86_64) += twofish-x86_64.o
obj-$(CONFIG_CRYPTO_TWOFISH_X86_64_3WAY) += twofish-x86_64-3way.o
+obj-$(CONFIG_CRYPTO_TWOFISH_AVX_X86_64) += twofish-avx-x86_64.o
obj-$(CONFIG_CRYPTO_SALSA20_X86_64) += salsa20-x86_64.o
obj-$(CONFIG_CRYPTO_SERPENT_SSE2_X86_64) += serpent-sse2-x86_64.o
+obj-$(CONFIG_CRYPTO_SERPENT_AVX_X86_64) += serpent-avx-x86_64.o
obj-$(CONFIG_CRYPTO_AES_NI_INTEL) += aesni-intel.o
obj-$(CONFIG_CRYPTO_GHASH_CLMUL_NI_INTEL) += ghash-clmulni-intel.o
@@ -30,16 +35,11 @@ camellia-x86_64-y := camellia-x86_64-asm_64.o camellia_glue.o
blowfish-x86_64-y := blowfish-x86_64-asm_64.o blowfish_glue.o
twofish-x86_64-y := twofish-x86_64-asm_64.o twofish_glue.o
twofish-x86_64-3way-y := twofish-x86_64-asm_64-3way.o twofish_glue_3way.o
+twofish-avx-x86_64-y := twofish-avx-x86_64-asm_64.o twofish_avx_glue.o
salsa20-x86_64-y := salsa20-x86_64-asm_64.o salsa20_glue.o
serpent-sse2-x86_64-y := serpent-sse2-x86_64-asm_64.o serpent_sse2_glue.o
+serpent-avx-x86_64-y := serpent-avx-x86_64-asm_64.o serpent_avx_glue.o
aesni-intel-y := aesni-intel_asm.o aesni-intel_glue.o fpu.o
-
ghash-clmulni-intel-y := ghash-clmulni-intel_asm.o ghash-clmulni-intel_glue.o
-
-# enable AVX support only when $(AS) can actually assemble the instructions
-ifeq ($(call as-instr,vpxor %xmm0$(comma)%xmm1$(comma)%xmm2,yes,no),yes)
-AFLAGS_sha1_ssse3_asm.o += -DSHA1_ENABLE_AVX_SUPPORT
-CFLAGS_sha1_ssse3_glue.o += -DSHA1_ENABLE_AVX_SUPPORT
-endif
sha1-ssse3-y := sha1_ssse3_asm.o sha1_ssse3_glue.o
diff --git a/arch/x86/crypto/ablk_helper.c b/arch/x86/crypto/ablk_helper.c
new file mode 100644
index 000000000000..43282fe04a8b
--- /dev/null
+++ b/arch/x86/crypto/ablk_helper.c
@@ -0,0 +1,149 @@
+/*
+ * Shared async block cipher helpers
+ *
+ * Copyright (c) 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
+ *
+ * Based on aesni-intel_glue.c by:
+ * Copyright (C) 2008, Intel Corp.
+ * Author: Huang Ying <ying.huang@intel.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA
+ *
+ */
+
+#include <linux/kernel.h>
+#include <linux/crypto.h>
+#include <linux/init.h>
+#include <linux/module.h>
+#include <crypto/algapi.h>
+#include <crypto/cryptd.h>
+#include <asm/i387.h>
+#include <asm/crypto/ablk_helper.h>
+
+int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
+ unsigned int key_len)
+{
+ struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+ struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base;
+ int err;
+
+ crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
+ crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm)
+ & CRYPTO_TFM_REQ_MASK);
+ err = crypto_ablkcipher_setkey(child, key, key_len);
+ crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child)
+ & CRYPTO_TFM_RES_MASK);
+ return err;
+}
+EXPORT_SYMBOL_GPL(ablk_set_key);
+
+int __ablk_encrypt(struct ablkcipher_request *req)
+{
+ struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+ struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+ struct blkcipher_desc desc;
+
+ desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
+ desc.info = req->info;
+ desc.flags = 0;
+
+ return crypto_blkcipher_crt(desc.tfm)->encrypt(
+ &desc, req->dst, req->src, req->nbytes);
+}
+EXPORT_SYMBOL_GPL(__ablk_encrypt);
+
+int ablk_encrypt(struct ablkcipher_request *req)
+{
+ struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+ struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+
+ if (!irq_fpu_usable()) {
+ struct ablkcipher_request *cryptd_req =
+ ablkcipher_request_ctx(req);
+
+ memcpy(cryptd_req, req, sizeof(*req));
+ ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
+
+ return crypto_ablkcipher_encrypt(cryptd_req);
+ } else {
+ return __ablk_encrypt(req);
+ }
+}
+EXPORT_SYMBOL_GPL(ablk_encrypt);
+
+int ablk_decrypt(struct ablkcipher_request *req)
+{
+ struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
+ struct async_helper_ctx *ctx = crypto_ablkcipher_ctx(tfm);
+
+ if (!irq_fpu_usable()) {
+ struct ablkcipher_request *cryptd_req =
+ ablkcipher_request_ctx(req);
+
+ memcpy(cryptd_req, req, sizeof(*req));
+ ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
+
+ return crypto_ablkcipher_decrypt(cryptd_req);
+ } else {
+ struct blkcipher_desc desc;
+
+ desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
+ desc.info = req->info;
+ desc.flags = 0;
+
+ return crypto_blkcipher_crt(desc.tfm)->decrypt(
+ &desc, req->dst, req->src, req->nbytes);
+ }
+}
+EXPORT_SYMBOL_GPL(ablk_decrypt);
+
+void ablk_exit(struct crypto_tfm *tfm)
+{
+ struct async_helper_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ cryptd_free_ablkcipher(ctx->cryptd_tfm);
+}
+EXPORT_SYMBOL_GPL(ablk_exit);
+
+int ablk_init_common(struct crypto_tfm *tfm, const char *drv_name)
+{
+ struct async_helper_ctx *ctx = crypto_tfm_ctx(tfm);
+ struct cryptd_ablkcipher *cryptd_tfm;
+
+ cryptd_tfm = cryptd_alloc_ablkcipher(drv_name, 0, 0);
+ if (IS_ERR(cryptd_tfm))
+ return PTR_ERR(cryptd_tfm);
+
+ ctx->cryptd_tfm = cryptd_tfm;
+ tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
+ crypto_ablkcipher_reqsize(&cryptd_tfm->base);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(ablk_init_common);
+
+int ablk_init(struct crypto_tfm *tfm)
+{
+ char drv_name[CRYPTO_MAX_ALG_NAME];
+
+ snprintf(drv_name, sizeof(drv_name), "__driver-%s",
+ crypto_tfm_alg_driver_name(tfm));
+
+ return ablk_init_common(tfm, drv_name);
+}
+EXPORT_SYMBOL_GPL(ablk_init);
+
+MODULE_LICENSE("GPL");
diff --git a/arch/x86/crypto/aes_glue.c b/arch/x86/crypto/aes_glue.c
index 8efcf42a9d7e..59b37deb8c8d 100644
--- a/arch/x86/crypto/aes_glue.c
+++ b/arch/x86/crypto/aes_glue.c
@@ -5,7 +5,7 @@
#include <linux/module.h>
#include <crypto/aes.h>
-#include <asm/aes.h>
+#include <asm/crypto/aes.h>
asmlinkage void aes_enc_blk(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in);
asmlinkage void aes_dec_blk(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in);
diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c
index ac7f5cd019e8..34fdcff4d2c8 100644
--- a/arch/x86/crypto/aesni-intel_glue.c
+++ b/arch/x86/crypto/aesni-intel_glue.c
@@ -30,7 +30,8 @@
#include <crypto/ctr.h>
#include <asm/cpu_device_id.h>
#include <asm/i387.h>
-#include <asm/aes.h>
+#include <asm/crypto/aes.h>
+#include <asm/crypto/ablk_helper.h>
#include <crypto/scatterwalk.h>
#include <crypto/internal/aead.h>
#include <linux/workqueue.h>
@@ -52,10 +53,6 @@
#define HAS_XTS
#endif
-struct async_aes_ctx {
- struct cryptd_ablkcipher *cryptd_tfm;
-};
-
/* This data is stored at the end of the crypto_tfm struct.
* It's a type of per "session" data storage location.
* This needs to be 16 byte aligned.
@@ -377,87 +374,6 @@ static int ctr_crypt(struct blkcipher_desc *desc,
}
#endif
-static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
- unsigned int key_len)
-{
- struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
- struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base;
- int err;
-
- crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
- crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm)
- & CRYPTO_TFM_REQ_MASK);
- err = crypto_ablkcipher_setkey(child, key, key_len);
- crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child)
- & CRYPTO_TFM_RES_MASK);
- return err;
-}
-
-static int ablk_encrypt(struct ablkcipher_request *req)
-{
- struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
- struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
-
- if (!irq_fpu_usable()) {
- struct ablkcipher_request *cryptd_req =
- ablkcipher_request_ctx(req);
- memcpy(cryptd_req, req, sizeof(*req));
- ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
- return crypto_ablkcipher_encrypt(cryptd_req);
- } else {
- struct blkcipher_desc desc;
- desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
- desc.info = req->info;
- desc.flags = 0;
- return crypto_blkcipher_crt(desc.tfm)->encrypt(
- &desc, req->dst, req->src, req->nbytes);
- }
-}
-
-static int ablk_decrypt(struct ablkcipher_request *req)
-{
- struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
- struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm);
-
- if (!irq_fpu_usable()) {
- struct ablkcipher_request *cryptd_req =
- ablkcipher_request_ctx(req);
- memcpy(cryptd_req, req, sizeof(*req));
- ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
- return crypto_ablkcipher_decrypt(cryptd_req);
- } else {
- struct blkcipher_desc desc;
- desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
- desc.info = req->info;
- desc.flags = 0;
- return crypto_blkcipher_crt(desc.tfm)->decrypt(
- &desc, req->dst, req->src, req->nbytes);
- }
-}
-
-static void ablk_exit(struct crypto_tfm *tfm)
-{
- struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
-
- cryptd_free_ablkcipher(ctx->cryptd_tfm);
-}
-
-static int ablk_init_common(struct crypto_tfm *tfm, const char *drv_name)
-{
- struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm);
- struct cryptd_ablkcipher *cryptd_tfm;
-
- cryptd_tfm = cryptd_alloc_ablkcipher(drv_name, 0, 0);
- if (IS_ERR(cryptd_tfm))
- return PTR_ERR(cryptd_tfm);
-
- ctx->cryptd_tfm = cryptd_tfm;
- tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
- crypto_ablkcipher_reqsize(&cryptd_tfm->base);
-
- return 0;
-}
-
static int ablk_ecb_init(struct crypto_tfm *tfm)
{
return ablk_init_common(tfm, "__driver-ecb-aes-aesni");
@@ -613,7 +529,7 @@ static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
struct crypto_aead *cryptd_child = cryptd_aead_child(ctx->cryptd_tfm);
struct aesni_rfc4106_gcm_ctx *child_ctx =
aesni_rfc4106_gcm_ctx_get(cryptd_child);
- u8 *new_key_mem = NULL;
+ u8 *new_key_align, *new_key_mem = NULL;
if (key_len < 4) {
crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN);
@@ -637,9 +553,9 @@ static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key,
if (!new_key_mem)
return -ENOMEM;
- new_key_mem = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
- memcpy(new_key_mem, key, key_len);
- key = new_key_mem;
+ new_key_align = PTR_ALIGN(new_key_mem, AESNI_ALIGN);
+ memcpy(new_key_align, key, key_len);
+ key = new_key_align;
}
if (!irq_fpu_usable())
@@ -968,7 +884,7 @@ static struct crypto_alg aesni_algs[] = { {
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
.cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct async_aes_ctx),
+ .cra_ctxsize = sizeof(struct async_helper_ctx),
.cra_alignmask = 0,
.cra_type = &crypto_ablkcipher_type,
.cra_module = THIS_MODULE,
@@ -989,7 +905,7 @@ static struct crypto_alg aesni_algs[] = { {
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
.cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct async_aes_ctx),
+ .cra_ctxsize = sizeof(struct async_helper_ctx),
.cra_alignmask = 0,
.cra_type = &crypto_ablkcipher_type,
.cra_module = THIS_MODULE,
@@ -1033,7 +949,7 @@ static struct crypto_alg aesni_algs[] = { {
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
.cra_blocksize = 1,
- .cra_ctxsize = sizeof(struct async_aes_ctx),
+ .cra_ctxsize = sizeof(struct async_helper_ctx),
.cra_alignmask = 0,
.cra_type = &crypto_ablkcipher_type,
.cra_module = THIS_MODULE,
@@ -1098,7 +1014,7 @@ static struct crypto_alg aesni_algs[] = { {
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
.cra_blocksize = 1,
- .cra_ctxsize = sizeof(struct async_aes_ctx),
+ .cra_ctxsize = sizeof(struct async_helper_ctx),
.cra_alignmask = 0,
.cra_type = &crypto_ablkcipher_type,
.cra_module = THIS_MODULE,
@@ -1126,7 +1042,7 @@ static struct crypto_alg aesni_algs[] = { {
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
.cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct async_aes_ctx),
+ .cra_ctxsize = sizeof(struct async_helper_ctx),
.cra_alignmask = 0,
.cra_type = &crypto_ablkcipher_type,
.cra_module = THIS_MODULE,
@@ -1150,7 +1066,7 @@ static struct crypto_alg aesni_algs[] = { {
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
.cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct async_aes_ctx),
+ .cra_ctxsize = sizeof(struct async_helper_ctx),
.cra_alignmask = 0,
.cra_type = &crypto_ablkcipher_type,
.cra_module = THIS_MODULE,
@@ -1174,7 +1090,7 @@ static struct crypto_alg aesni_algs[] = { {
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
.cra_blocksize = AES_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct async_aes_ctx),
+ .cra_ctxsize = sizeof(struct async_helper_ctx),
.cra_alignmask = 0,
.cra_type = &crypto_ablkcipher_type,
.cra_module = THIS_MODULE,
diff --git a/arch/x86/crypto/camellia_glue.c b/arch/x86/crypto/camellia_glue.c
index 3306dc0b139e..eeb2b3b743e9 100644
--- a/arch/x86/crypto/camellia_glue.c
+++ b/arch/x86/crypto/camellia_glue.c
@@ -5,10 +5,6 @@
*
* Camellia parts based on code by:
* Copyright (C) 2006 NTT (Nippon Telegraph and Telephone Corporation)
- * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
- * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
- * CTR part based on code (crypto/ctr.c) by:
- * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@@ -34,9 +30,9 @@
#include <linux/module.h>
#include <linux/types.h>
#include <crypto/algapi.h>
-#include <crypto/b128ops.h>
#include <crypto/lrw.h>
#include <crypto/xts.h>
+#include <asm/crypto/glue_helper.h>
#define CAMELLIA_MIN_KEY_SIZE 16
#define CAMELLIA_MAX_KEY_SIZE 32
@@ -1312,307 +1308,128 @@ static int camellia_setkey(struct crypto_tfm *tfm, const u8 *in_key,
&tfm->crt_flags);
}
-static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
- void (*fn)(struct camellia_ctx *, u8 *, const u8 *),
- void (*fn_2way)(struct camellia_ctx *, u8 *, const u8 *))
+static void camellia_decrypt_cbc_2way(void *ctx, u128 *dst, const u128 *src)
{
- struct camellia_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- unsigned int bsize = CAMELLIA_BLOCK_SIZE;
- unsigned int nbytes;
- int err;
-
- err = blkcipher_walk_virt(desc, walk);
-
- while ((nbytes = walk->nbytes)) {
- u8 *wsrc = walk->src.virt.addr;
- u8 *wdst = walk->dst.virt.addr;
-
- /* Process two block batch */
- if (nbytes >= bsize * 2) {
- do {
- fn_2way(ctx, wdst, wsrc);
-
- wsrc += bsize * 2;
- wdst += bsize * 2;
- nbytes -= bsize * 2;
- } while (nbytes >= bsize * 2);
-
- if (nbytes < bsize)
- goto done;
- }
-
- /* Handle leftovers */
- do {
- fn(ctx, wdst, wsrc);
-
- wsrc += bsize;
- wdst += bsize;
- nbytes -= bsize;
- } while (nbytes >= bsize);
-
-done:
- err = blkcipher_walk_done(desc, walk, nbytes);
- }
-
- return err;
-}
-
-static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
- struct scatterlist *src, unsigned int nbytes)
-{
- struct blkcipher_walk walk;
-
- blkcipher_walk_init(&walk, dst, src, nbytes);
- return ecb_crypt(desc, &walk, camellia_enc_blk, camellia_enc_blk_2way);
-}
+ u128 iv = *src;
-static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
- struct scatterlist *src, unsigned int nbytes)
-{
- struct blkcipher_walk walk;
-
- blkcipher_walk_init(&walk, dst, src, nbytes);
- return ecb_crypt(desc, &walk, camellia_dec_blk, camellia_dec_blk_2way);
-}
+ camellia_dec_blk_2way(ctx, (u8 *)dst, (u8 *)src);
-static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
- struct blkcipher_walk *walk)
-{
- struct camellia_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- unsigned int bsize = CAMELLIA_BLOCK_SIZE;
- unsigned int nbytes = walk->nbytes;
- u128 *src = (u128 *)walk->src.virt.addr;
- u128 *dst = (u128 *)walk->dst.virt.addr;
- u128 *iv = (u128 *)walk->iv;
-
- do {
- u128_xor(dst, src, iv);
- camellia_enc_blk(ctx, (u8 *)dst, (u8 *)dst);
- iv = dst;
-
- src += 1;
- dst += 1;
- nbytes -= bsize;
- } while (nbytes >= bsize);
-
- u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv);
- return nbytes;
+ u128_xor(&dst[1], &dst[1], &iv);
}
-static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
- struct scatterlist *src, unsigned int nbytes)
+static void camellia_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)
{
- struct blkcipher_walk walk;
- int err;
+ be128 ctrblk;
- blkcipher_walk_init(&walk, dst, src, nbytes);
- err = blkcipher_walk_virt(desc, &walk);
+ if (dst != src)
+ *dst = *src;
- while ((nbytes = walk.nbytes)) {
- nbytes = __cbc_encrypt(desc, &walk);
- err = blkcipher_walk_done(desc, &walk, nbytes);
- }
+ u128_to_be128(&ctrblk, iv);
+ u128_inc(iv);
- return err;
+ camellia_enc_blk_xor(ctx, (u8 *)dst, (u8 *)&ctrblk);
}
-static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
- struct blkcipher_walk *walk)
+static void camellia_crypt_ctr_2way(void *ctx, u128 *dst, const u128 *src,
+ u128 *iv)
{
- struct camellia_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- unsigned int bsize = CAMELLIA_BLOCK_SIZE;
- unsigned int nbytes = walk->nbytes;
- u128 *src = (u128 *)walk->src.virt.addr;
- u128 *dst = (u128 *)walk->dst.virt.addr;
- u128 ivs[2 - 1];
- u128 last_iv;
+ be128 ctrblks[2];
- /* Start of the last block. */
- src += nbytes / bsize - 1;
- dst += nbytes / bsize - 1;
-
- last_iv = *src;
-
- /* Process two block batch */
- if (nbytes >= bsize * 2) {
- do {
- nbytes -= bsize * (2 - 1);
- src -= 2 - 1;
- dst -= 2 - 1;
-
- ivs[0] = src[0];
-
- camellia_dec_blk_2way(ctx, (u8 *)dst, (u8 *)src);
-
- u128_xor(dst + 1, dst + 1, ivs + 0);
-
- nbytes -= bsize;
- if (nbytes < bsize)
- goto done;
-
- u128_xor(dst, dst, src - 1);
- src -= 1;
- dst -= 1;
- } while (nbytes >= bsize * 2);
-
- if (nbytes < bsize)
- goto done;
+ if (dst != src) {
+ dst[0] = src[0];
+ dst[1] = src[1];
}
- /* Handle leftovers */
- for (;;) {
- camellia_dec_blk(ctx, (u8 *)dst, (u8 *)src);
-
- nbytes -= bsize;
- if (nbytes < bsize)
- break;
+ u128_to_be128(&ctrblks[0], iv);
+ u128_inc(iv);
+ u128_to_be128(&ctrblks[1], iv);
+ u128_inc(iv);
- u128_xor(dst, dst, src - 1);
- src -= 1;
- dst -= 1;
- }
-
-done:
- u128_xor(dst, dst, (u128 *)walk->iv);
- *(u128 *)walk->iv = last_iv;
-
- return nbytes;
+ camellia_enc_blk_xor_2way(ctx, (u8 *)dst, (u8 *)ctrblks);
}
-static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
- struct scatterlist *src, unsigned int nbytes)
-{
- struct blkcipher_walk walk;
- int err;
-
- blkcipher_walk_init(&walk, dst, src, nbytes);
- err = blkcipher_walk_virt(desc, &walk);
+static const struct common_glue_ctx camellia_enc = {
+ .num_funcs = 2,
+ .fpu_blocks_limit = -1,
+
+ .funcs = { {
+ .num_blocks = 2,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk_2way) }
+ }, {
+ .num_blocks = 1,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_enc_blk) }
+ } }
+};
- while ((nbytes = walk.nbytes)) {
- nbytes = __cbc_decrypt(desc, &walk);
- err = blkcipher_walk_done(desc, &walk, nbytes);
- }
+static const struct common_glue_ctx camellia_ctr = {
+ .num_funcs = 2,
+ .fpu_blocks_limit = -1,
+
+ .funcs = { {
+ .num_blocks = 2,
+ .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr_2way) }
+ }, {
+ .num_blocks = 1,
+ .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(camellia_crypt_ctr) }
+ } }
+};
- return err;
-}
+static const struct common_glue_ctx camellia_dec = {
+ .num_funcs = 2,
+ .fpu_blocks_limit = -1,
+
+ .funcs = { {
+ .num_blocks = 2,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk_2way) }
+ }, {
+ .num_blocks = 1,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(camellia_dec_blk) }
+ } }
+};
-static inline void u128_to_be128(be128 *dst, const u128 *src)
-{
- dst->a = cpu_to_be64(src->a);
- dst->b = cpu_to_be64(src->b);
-}
+static const struct common_glue_ctx camellia_dec_cbc = {
+ .num_funcs = 2,
+ .fpu_blocks_limit = -1,
+
+ .funcs = { {
+ .num_blocks = 2,
+ .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_decrypt_cbc_2way) }
+ }, {
+ .num_blocks = 1,
+ .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(camellia_dec_blk) }
+ } }
+};
-static inline void be128_to_u128(u128 *dst, const be128 *src)
+static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
{
- dst->a = be64_to_cpu(src->a);
- dst->b = be64_to_cpu(src->b);
+ return glue_ecb_crypt_128bit(&camellia_enc, desc, dst, src, nbytes);
}
-static inline void u128_inc(u128 *i)
+static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
{
- i->b++;
- if (!i->b)
- i->a++;
+ return glue_ecb_crypt_128bit(&camellia_dec, desc, dst, src, nbytes);
}
-static void ctr_crypt_final(struct blkcipher_desc *desc,
- struct blkcipher_walk *walk)
+static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
{
- struct camellia_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- u8 keystream[CAMELLIA_BLOCK_SIZE];
- u8 *src = walk->src.virt.addr;
- u8 *dst = walk->dst.virt.addr;
- unsigned int nbytes = walk->nbytes;
- u128 ctrblk;
-
- memcpy(keystream, src, nbytes);
- camellia_enc_blk_xor(ctx, keystream, walk->iv);
- memcpy(dst, keystream, nbytes);
-
- be128_to_u128(&ctrblk, (be128 *)walk->iv);
- u128_inc(&ctrblk);
- u128_to_be128((be128 *)walk->iv, &ctrblk);
+ return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(camellia_enc_blk), desc,
+ dst, src, nbytes);
}
-static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
- struct blkcipher_walk *walk)
+static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
{
- struct camellia_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- unsigned int bsize = CAMELLIA_BLOCK_SIZE;
- unsigned int nbytes = walk->nbytes;
- u128 *src = (u128 *)walk->src.virt.addr;
- u128 *dst = (u128 *)walk->dst.virt.addr;
- u128 ctrblk;
- be128 ctrblocks[2];
-
- be128_to_u128(&ctrblk, (be128 *)walk->iv);
-
- /* Process two block batch */
- if (nbytes >= bsize * 2) {
- do {
- if (dst != src) {
- dst[0] = src[0];
- dst[1] = src[1];
- }
-
- /* create ctrblks for parallel encrypt */
- u128_to_be128(&ctrblocks[0], &ctrblk);
- u128_inc(&ctrblk);
- u128_to_be128(&ctrblocks[1], &ctrblk);
- u128_inc(&ctrblk);
-
- camellia_enc_blk_xor_2way(ctx, (u8 *)dst,
- (u8 *)ctrblocks);
-
- src += 2;
- dst += 2;
- nbytes -= bsize * 2;
- } while (nbytes >= bsize * 2);
-
- if (nbytes < bsize)
- goto done;
- }
-
- /* Handle leftovers */
- do {
- if (dst != src)
- *dst = *src;
-
- u128_to_be128(&ctrblocks[0], &ctrblk);
- u128_inc(&ctrblk);
-
- camellia_enc_blk_xor(ctx, (u8 *)dst, (u8 *)ctrblocks);
-
- src += 1;
- dst += 1;
- nbytes -= bsize;
- } while (nbytes >= bsize);
-
-done:
- u128_to_be128((be128 *)walk->iv, &ctrblk);
- return nbytes;
+ return glue_cbc_decrypt_128bit(&camellia_dec_cbc, desc, dst, src,
+ nbytes);
}
static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
- struct blkcipher_walk walk;
- int err;
-
- blkcipher_walk_init(&walk, dst, src, nbytes);
- err = blkcipher_walk_virt_block(desc, &walk, CAMELLIA_BLOCK_SIZE);
-
- while ((nbytes = walk.nbytes) >= CAMELLIA_BLOCK_SIZE) {
- nbytes = __ctr_crypt(desc, &walk);
- err = blkcipher_walk_done(desc, &walk, nbytes);
- }
-
- if (walk.nbytes) {
- ctr_crypt_final(desc, &walk);
- err = blkcipher_walk_done(desc, &walk, 0);
- }
-
- return err;
+ return glue_ctr_crypt_128bit(&camellia_ctr, desc, dst, src, nbytes);
}
static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
diff --git a/arch/x86/crypto/glue_helper.c b/arch/x86/crypto/glue_helper.c
new file mode 100644
index 000000000000..4854f0f31e4f
--- /dev/null
+++ b/arch/x86/crypto/glue_helper.c
@@ -0,0 +1,307 @@
+/*
+ * Shared glue code for 128bit block ciphers
+ *
+ * Copyright (c) 2012 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
+ *
+ * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
+ * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
+ * CTR part based on code (crypto/ctr.c) by:
+ * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA
+ *
+ */
+
+#include <linux/module.h>
+#include <crypto/b128ops.h>
+#include <crypto/lrw.h>
+#include <crypto/xts.h>
+#include <asm/crypto/glue_helper.h>
+#include <crypto/scatterwalk.h>
+
+static int __glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx,
+ struct blkcipher_desc *desc,
+ struct blkcipher_walk *walk)
+{
+ void *ctx = crypto_blkcipher_ctx(desc->tfm);
+ const unsigned int bsize = 128 / 8;
+ unsigned int nbytes, i, func_bytes;
+ bool fpu_enabled = false;
+ int err;
+
+ err = blkcipher_walk_virt(desc, walk);
+
+ while ((nbytes = walk->nbytes)) {
+ u8 *wsrc = walk->src.virt.addr;
+ u8 *wdst = walk->dst.virt.addr;
+
+ fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
+ desc, fpu_enabled, nbytes);
+
+ for (i = 0; i < gctx->num_funcs; i++) {
+ func_bytes = bsize * gctx->funcs[i].num_blocks;
+
+ /* Process multi-block batch */
+ if (nbytes >= func_bytes) {
+ do {
+ gctx->funcs[i].fn_u.ecb(ctx, wdst,
+ wsrc);
+
+ wsrc += func_bytes;
+ wdst += func_bytes;
+ nbytes -= func_bytes;
+ } while (nbytes >= func_bytes);
+
+ if (nbytes < bsize)
+ goto done;
+ }
+ }
+
+done:
+ err = blkcipher_walk_done(desc, walk, nbytes);
+ }
+
+ glue_fpu_end(fpu_enabled);
+ return err;
+}
+
+int glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx,
+ struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ struct blkcipher_walk walk;
+
+ blkcipher_walk_init(&walk, dst, src, nbytes);
+ return __glue_ecb_crypt_128bit(gctx, desc, &walk);
+}
+EXPORT_SYMBOL_GPL(glue_ecb_crypt_128bit);
+
+static unsigned int __glue_cbc_encrypt_128bit(const common_glue_func_t fn,
+ struct blkcipher_desc *desc,
+ struct blkcipher_walk *walk)
+{
+ void *ctx = crypto_blkcipher_ctx(desc->tfm);
+ const unsigned int bsize = 128 / 8;
+ unsigned int nbytes = walk->nbytes;
+ u128 *src = (u128 *)walk->src.virt.addr;
+ u128 *dst = (u128 *)walk->dst.virt.addr;
+ u128 *iv = (u128 *)walk->iv;
+
+ do {
+ u128_xor(dst, src, iv);
+ fn(ctx, (u8 *)dst, (u8 *)dst);
+ iv = dst;
+
+ src += 1;
+ dst += 1;
+ nbytes -= bsize;
+ } while (nbytes >= bsize);
+
+ u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv);
+ return nbytes;
+}
+
+int glue_cbc_encrypt_128bit(const common_glue_func_t fn,
+ struct blkcipher_desc *desc,
+ struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ struct blkcipher_walk walk;
+ int err;
+
+ blkcipher_walk_init(&walk, dst, src, nbytes);
+ err = blkcipher_walk_virt(desc, &walk);
+
+ while ((nbytes = walk.nbytes)) {
+ nbytes = __glue_cbc_encrypt_128bit(fn, desc, &walk);
+ err = blkcipher_walk_done(desc, &walk, nbytes);
+ }
+
+ return err;
+}
+EXPORT_SYMBOL_GPL(glue_cbc_encrypt_128bit);
+
+static unsigned int
+__glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx,
+ struct blkcipher_desc *desc,
+ struct blkcipher_walk *walk)
+{
+ void *ctx = crypto_blkcipher_ctx(desc->tfm);
+ const unsigned int bsize = 128 / 8;
+ unsigned int nbytes = walk->nbytes;
+ u128 *src = (u128 *)walk->src.virt.addr;
+ u128 *dst = (u128 *)walk->dst.virt.addr;
+ u128 last_iv;
+ unsigned int num_blocks, func_bytes;
+ unsigned int i;
+
+ /* Start of the last block. */
+ src += nbytes / bsize - 1;
+ dst += nbytes / bsize - 1;
+
+ last_iv = *src;
+
+ for (i = 0; i < gctx->num_funcs; i++) {
+ num_blocks = gctx->funcs[i].num_blocks;
+ func_bytes = bsize * num_blocks;
+
+ /* Process multi-block batch */
+ if (nbytes >= func_bytes) {
+ do {
+ nbytes -= func_bytes - bsize;
+ src -= num_blocks - 1;
+ dst -= num_blocks - 1;
+
+ gctx->funcs[i].fn_u.cbc(ctx, dst, src);
+
+ nbytes -= bsize;
+ if (nbytes < bsize)
+ goto done;
+
+ u128_xor(dst, dst, src - 1);
+ src -= 1;
+ dst -= 1;
+ } while (nbytes >= func_bytes);
+
+ if (nbytes < bsize)
+ goto done;
+ }
+ }
+
+done:
+ u128_xor(dst, dst, (u128 *)walk->iv);
+ *(u128 *)walk->iv = last_iv;
+
+ return nbytes;
+}
+
+int glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx,
+ struct blkcipher_desc *desc,
+ struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ const unsigned int bsize = 128 / 8;
+ bool fpu_enabled = false;
+ struct blkcipher_walk walk;
+ int err;
+
+ blkcipher_walk_init(&walk, dst, src, nbytes);
+ err = blkcipher_walk_virt(desc, &walk);
+
+ while ((nbytes = walk.nbytes)) {
+ fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
+ desc, fpu_enabled, nbytes);
+ nbytes = __glue_cbc_decrypt_128bit(gctx, desc, &walk);
+ err = blkcipher_walk_done(desc, &walk, nbytes);
+ }
+
+ glue_fpu_end(fpu_enabled);
+ return err;
+}
+EXPORT_SYMBOL_GPL(glue_cbc_decrypt_128bit);
+
+static void glue_ctr_crypt_final_128bit(const common_glue_ctr_func_t fn_ctr,
+ struct blkcipher_desc *desc,
+ struct blkcipher_walk *walk)
+{
+ void *ctx = crypto_blkcipher_ctx(desc->tfm);
+ u8 *src = (u8 *)walk->src.virt.addr;
+ u8 *dst = (u8 *)walk->dst.virt.addr;
+ unsigned int nbytes = walk->nbytes;
+ u128 ctrblk;
+ u128 tmp;
+
+ be128_to_u128(&ctrblk, (be128 *)walk->iv);
+
+ memcpy(&tmp, src, nbytes);
+ fn_ctr(ctx, &tmp, &tmp, &ctrblk);
+ memcpy(dst, &tmp, nbytes);
+
+ u128_to_be128((be128 *)walk->iv, &ctrblk);
+}
+EXPORT_SYMBOL_GPL(glue_ctr_crypt_final_128bit);
+
+static unsigned int __glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx,
+ struct blkcipher_desc *desc,
+ struct blkcipher_walk *walk)
+{
+ const unsigned int bsize = 128 / 8;
+ void *ctx = crypto_blkcipher_ctx(desc->tfm);
+ unsigned int nbytes = walk->nbytes;
+ u128 *src = (u128 *)walk->src.virt.addr;
+ u128 *dst = (u128 *)walk->dst.virt.addr;
+ u128 ctrblk;
+ unsigned int num_blocks, func_bytes;
+ unsigned int i;
+
+ be128_to_u128(&ctrblk, (be128 *)walk->iv);
+
+ /* Process multi-block batch */
+ for (i = 0; i < gctx->num_funcs; i++) {
+ num_blocks = gctx->funcs[i].num_blocks;
+ func_bytes = bsize * num_blocks;
+
+ if (nbytes >= func_bytes) {
+ do {
+ gctx->funcs[i].fn_u.ctr(ctx, dst, src, &ctrblk);
+
+ src += num_blocks;
+ dst += num_blocks;
+ nbytes -= func_bytes;
+ } while (nbytes >= func_bytes);
+
+ if (nbytes < bsize)
+ goto done;
+ }
+ }
+
+done:
+ u128_to_be128((be128 *)walk->iv, &ctrblk);
+ return nbytes;
+}
+
+int glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx,
+ struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ const unsigned int bsize = 128 / 8;
+ bool fpu_enabled = false;
+ struct blkcipher_walk walk;
+ int err;
+
+ blkcipher_walk_init(&walk, dst, src, nbytes);
+ err = blkcipher_walk_virt_block(desc, &walk, bsize);
+
+ while ((nbytes = walk.nbytes) >= bsize) {
+ fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
+ desc, fpu_enabled, nbytes);
+ nbytes = __glue_ctr_crypt_128bit(gctx, desc, &walk);
+ err = blkcipher_walk_done(desc, &walk, nbytes);
+ }
+
+ glue_fpu_end(fpu_enabled);
+
+ if (walk.nbytes) {
+ glue_ctr_crypt_final_128bit(
+ gctx->funcs[gctx->num_funcs - 1].fn_u.ctr, desc, &walk);
+ err = blkcipher_walk_done(desc, &walk, 0);
+ }
+
+ return err;
+}
+EXPORT_SYMBOL_GPL(glue_ctr_crypt_128bit);
+
+MODULE_LICENSE("GPL");
diff --git a/arch/x86/crypto/serpent-avx-x86_64-asm_64.S b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
new file mode 100644
index 000000000000..504106bf04a2
--- /dev/null
+++ b/arch/x86/crypto/serpent-avx-x86_64-asm_64.S
@@ -0,0 +1,704 @@
+/*
+ * Serpent Cipher 8-way parallel algorithm (x86_64/AVX)
+ *
+ * Copyright (C) 2012 Johannes Goetzfried
+ * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
+ *
+ * Based on arch/x86/crypto/serpent-sse2-x86_64-asm_64.S by
+ * Copyright (C) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA
+ *
+ */
+
+.file "serpent-avx-x86_64-asm_64.S"
+.text
+
+#define CTX %rdi
+
+/**********************************************************************
+ 8-way AVX serpent
+ **********************************************************************/
+#define RA1 %xmm0
+#define RB1 %xmm1
+#define RC1 %xmm2
+#define RD1 %xmm3
+#define RE1 %xmm4
+
+#define tp %xmm5
+
+#define RA2 %xmm6
+#define RB2 %xmm7
+#define RC2 %xmm8
+#define RD2 %xmm9
+#define RE2 %xmm10
+
+#define RNOT %xmm11
+
+#define RK0 %xmm12
+#define RK1 %xmm13
+#define RK2 %xmm14
+#define RK3 %xmm15
+
+
+#define S0_1(x0, x1, x2, x3, x4) \
+ vpor x0, x3, tp; \
+ vpxor x3, x0, x0; \
+ vpxor x2, x3, x4; \
+ vpxor RNOT, x4, x4; \
+ vpxor x1, tp, x3; \
+ vpand x0, x1, x1; \
+ vpxor x4, x1, x1; \
+ vpxor x0, x2, x2;
+#define S0_2(x0, x1, x2, x3, x4) \
+ vpxor x3, x0, x0; \
+ vpor x0, x4, x4; \
+ vpxor x2, x0, x0; \
+ vpand x1, x2, x2; \
+ vpxor x2, x3, x3; \
+ vpxor RNOT, x1, x1; \
+ vpxor x4, x2, x2; \
+ vpxor x2, x1, x1;
+
+#define S1_1(x0, x1, x2, x3, x4) \
+ vpxor x0, x1, tp; \
+ vpxor x3, x0, x0; \
+ vpxor RNOT, x3, x3; \
+ vpand tp, x1, x4; \
+ vpor tp, x0, x0; \
+ vpxor x2, x3, x3; \
+ vpxor x3, x0, x0; \
+ vpxor x3, tp, x1;
+#define S1_2(x0, x1, x2, x3, x4) \
+ vpxor x4, x3, x3; \
+ vpor x4, x1, x1; \
+ vpxor x2, x4, x4; \
+ vpand x0, x2, x2; \
+ vpxor x1, x2, x2; \
+ vpor x0, x1, x1; \
+ vpxor RNOT, x0, x0; \
+ vpxor x2, x0, x0; \
+ vpxor x1, x4, x4;
+
+#define S2_1(x0, x1, x2, x3, x4) \
+ vpxor RNOT, x3, x3; \
+ vpxor x0, x1, x1; \
+ vpand x2, x0, tp; \
+ vpxor x3, tp, tp; \
+ vpor x0, x3, x3; \
+ vpxor x1, x2, x2; \
+ vpxor x1, x3, x3; \
+ vpand tp, x1, x1;
+#define S2_2(x0, x1, x2, x3, x4) \
+ vpxor x2, tp, tp; \
+ vpand x3, x2, x2; \
+ vpor x1, x3, x3; \
+ vpxor RNOT, tp, tp; \
+ vpxor tp, x3, x3; \
+ vpxor tp, x0, x4; \
+ vpxor x2, tp, x0; \
+ vpor x2, x1, x1;
+
+#define S3_1(x0, x1, x2, x3, x4) \
+ vpxor x3, x1, tp; \
+ vpor x0, x3, x3; \
+ vpand x0, x1, x4; \
+ vpxor x2, x0, x0; \
+ vpxor tp, x2, x2; \
+ vpand x3, tp, x1; \
+ vpxor x3, x2, x2; \
+ vpor x4, x0, x0; \
+ vpxor x3, x4, x4;
+#define S3_2(x0, x1, x2, x3, x4) \
+ vpxor x0, x1, x1; \
+ vpand x3, x0, x0; \
+ vpand x4, x3, x3; \
+ vpxor x2, x3, x3; \
+ vpor x1, x4, x4; \
+ vpand x1, x2, x2; \
+ vpxor x3, x4, x4; \
+ vpxor x3, x0, x0; \
+ vpxor x2, x3, x3;
+
+#define S4_1(x0, x1, x2, x3, x4) \
+ vpand x0, x3, tp; \
+ vpxor x3, x0, x0; \
+ vpxor x2, tp, tp; \
+ vpor x3, x2, x2; \
+ vpxor x1, x0, x0; \
+ vpxor tp, x3, x4; \
+ vpor x0, x2, x2; \
+ vpxor x1, x2, x2;
+#define S4_2(x0, x1, x2, x3, x4) \
+ vpand x0, x1, x1; \
+ vpxor x4, x1, x1; \
+ vpand x2, x4, x4; \
+ vpxor tp, x2, x2; \
+ vpxor x0, x4, x4; \
+ vpor x1, tp, x3; \
+ vpxor RNOT, x1, x1; \
+ vpxor x0, x3, x3;
+
+#define S5_1(x0, x1, x2, x3, x4) \
+ vpor x0, x1, tp; \
+ vpxor tp, x2, x2; \
+ vpxor RNOT, x3, x3; \
+ vpxor x0, x1, x4; \
+ vpxor x2, x0, x0; \
+ vpand x4, tp, x1; \
+ vpor x3, x4, x4; \
+ vpxor x0, x4, x4;
+#define S5_2(x0, x1, x2, x3, x4) \
+ vpand x3, x0, x0; \
+ vpxor x3, x1, x1; \
+ vpxor x2, x3, x3; \
+ vpxor x1, x0, x0; \
+ vpand x4, x2, x2; \
+ vpxor x2, x1, x1; \
+ vpand x0, x2, x2; \
+ vpxor x2, x3, x3;
+
+#define S6_1(x0, x1, x2, x3, x4) \
+ vpxor x0, x3, x3; \
+ vpxor x2, x1, tp; \
+ vpxor x0, x2, x2; \
+ vpand x3, x0, x0; \
+ vpor x3, tp, tp; \
+ vpxor RNOT, x1, x4; \
+ vpxor tp, x0, x0; \
+ vpxor x2, tp, x1;
+#define S6_2(x0, x1, x2, x3, x4) \
+ vpxor x4, x3, x3; \
+ vpxor x0, x4, x4; \
+ vpand x0, x2, x2; \
+ vpxor x1, x4, x4; \
+ vpxor x3, x2, x2; \
+ vpand x1, x3, x3; \
+ vpxor x0, x3, x3; \
+ vpxor x2, x1, x1;
+
+#define S7_1(x0, x1, x2, x3, x4) \
+ vpxor RNOT, x1, tp; \
+ vpxor RNOT, x0, x0; \
+ vpand x2, tp, x1; \
+ vpxor x3, x1, x1; \
+ vpor tp, x3, x3; \
+ vpxor x2, tp, x4; \
+ vpxor x3, x2, x2; \
+ vpxor x0, x3, x3; \
+ vpor x1, x0, x0;
+#define S7_2(x0, x1, x2, x3, x4) \
+ vpand x0, x2, x2; \
+ vpxor x4, x0, x0; \
+ vpxor x3, x4, x4; \
+ vpand x0, x3, x3; \
+ vpxor x1, x4, x4; \
+ vpxor x4, x2, x2; \
+ vpxor x1, x3, x3; \
+ vpor x0, x4, x4; \
+ vpxor x1, x4, x4;
+
+#define SI0_1(x0, x1, x2, x3, x4) \
+ vpxor x0, x1, x1; \
+ vpor x1, x3, tp; \
+ vpxor x1, x3, x4; \
+ vpxor RNOT, x0, x0; \
+ vpxor tp, x2, x2; \
+ vpxor x0, tp, x3; \
+ vpand x1, x0, x0; \
+ vpxor x2, x0, x0;
+#define SI0_2(x0, x1, x2, x3, x4) \
+ vpand x3, x2, x2; \
+ vpxor x4, x3, x3; \
+ vpxor x3, x2, x2; \
+ vpxor x3, x1, x1; \
+ vpand x0, x3, x3; \
+ vpxor x0, x1, x1; \
+ vpxor x2, x0, x0; \
+ vpxor x3, x4, x4;
+
+#define SI1_1(x0, x1, x2, x3, x4) \
+ vpxor x3, x1, x1; \
+ vpxor x2, x0, tp; \
+ vpxor RNOT, x2, x2; \
+ vpor x1, x0, x4; \
+ vpxor x3, x4, x4; \
+ vpand x1, x3, x3; \
+ vpxor x2, x1, x1; \
+ vpand x4, x2, x2;
+#define SI1_2(x0, x1, x2, x3, x4) \
+ vpxor x1, x4, x4; \
+ vpor x3, x1, x1; \
+ vpxor tp, x3, x3; \
+ vpxor tp, x2, x2; \
+ vpor x4, tp, x0; \
+ vpxor x4, x2, x2; \
+ vpxor x0, x1, x1; \
+ vpxor x1, x4, x4;
+
+#define SI2_1(x0, x1, x2, x3, x4) \
+ vpxor x1, x2, x2; \
+ vpxor RNOT, x3, tp; \
+ vpor x2, tp, tp; \
+ vpxor x3, x2, x2; \
+ vpxor x0, x3, x4; \
+ vpxor x1, tp, x3; \
+ vpor x2, x1, x1; \
+ vpxor x0, x2, x2;
+#define SI2_2(x0, x1, x2, x3, x4) \
+ vpxor x4, x1, x1; \
+ vpor x3, x4, x4; \
+ vpxor x3, x2, x2; \
+ vpxor x2, x4, x4; \
+ vpand x1, x2, x2; \
+ vpxor x3, x2, x2; \
+ vpxor x4, x3, x3; \
+ vpxor x0, x4, x4;
+
+#define SI3_1(x0, x1, x2, x3, x4) \
+ vpxor x1, x2, x2; \
+ vpand x2, x1, tp; \
+ vpxor x0, tp, tp; \
+ vpor x1, x0, x0; \
+ vpxor x3, x1, x4; \
+ vpxor x3, x0, x0; \
+ vpor tp, x3, x3; \
+ vpxor x2, tp, x1;
+#define SI3_2(x0, x1, x2, x3, x4) \
+ vpxor x3, x1, x1; \
+ vpxor x2, x0, x0; \
+ vpxor x3, x2, x2; \
+ vpand x1, x3, x3; \
+ vpxor x0, x1, x1; \
+ vpand x2, x0, x0; \
+ vpxor x3, x4, x4; \
+ vpxor x0, x3, x3; \
+ vpxor x1, x0, x0;
+
+#define SI4_1(x0, x1, x2, x3, x4) \
+ vpxor x3, x2, x2; \
+ vpand x1, x0, tp; \
+ vpxor x2, tp, tp; \
+ vpor x3, x2, x2; \
+ vpxor RNOT, x0, x4; \
+ vpxor tp, x1, x1; \
+ vpxor x2, tp, x0; \
+ vpand x4, x2, x2;
+#define SI4_2(x0, x1, x2, x3, x4) \
+ vpxor x0, x2, x2; \
+ vpor x4, x0, x0; \
+ vpxor x3, x0, x0; \
+ vpand x2, x3, x3; \
+ vpxor x3, x4, x4; \
+ vpxor x1, x3, x3; \
+ vpand x0, x1, x1; \
+ vpxor x1, x4, x4; \
+ vpxor x3, x0, x0;
+
+#define SI5_1(x0, x1, x2, x3, x4) \
+ vpor x2, x1, tp; \
+ vpxor x1, x2, x2; \
+ vpxor x3, tp, tp; \
+ vpand x1, x3, x3; \
+ vpxor x3, x2, x2; \
+ vpor x0, x3, x3; \
+ vpxor RNOT, x0, x0; \
+ vpxor x2, x3, x3; \
+ vpor x0, x2, x2;
+#define SI5_2(x0, x1, x2, x3, x4) \
+ vpxor tp, x1, x4; \
+ vpxor x4, x2, x2; \
+ vpand x0, x4, x4; \
+ vpxor tp, x0, x0; \
+ vpxor x3, tp, x1; \
+ vpand x2, x0, x0; \
+ vpxor x3, x2, x2; \
+ vpxor x2, x0, x0; \
+ vpxor x4, x2, x2; \
+ vpxor x3, x4, x4;
+
+#define SI6_1(x0, x1, x2, x3, x4) \
+ vpxor x2, x0, x0; \
+ vpand x3, x0, tp; \
+ vpxor x3, x2, x2; \
+ vpxor x2, tp, tp; \
+ vpxor x1, x3, x3; \
+ vpor x0, x2, x2; \
+ vpxor x3, x2, x2; \
+ vpand tp, x3, x3;
+#define SI6_2(x0, x1, x2, x3, x4) \
+ vpxor RNOT, tp, tp; \
+ vpxor x1, x3, x3; \
+ vpand x2, x1, x1; \
+ vpxor tp, x0, x4; \
+ vpxor x4, x3, x3; \
+ vpxor x2, x4, x4; \
+ vpxor x1, tp, x0; \
+ vpxor x0, x2, x2;
+
+#define SI7_1(x0, x1, x2, x3, x4) \
+ vpand x0, x3, tp; \
+ vpxor x2, x0, x0; \
+ vpor x3, x2, x2; \
+ vpxor x1, x3, x4; \
+ vpxor RNOT, x0, x0; \
+ vpor tp, x1, x1; \
+ vpxor x0, x4, x4; \
+ vpand x2, x0, x0; \
+ vpxor x1, x0, x0;
+#define SI7_2(x0, x1, x2, x3, x4) \
+ vpand x2, x1, x1; \
+ vpxor x2, tp, x3; \
+ vpxor x3, x4, x4; \
+ vpand x3, x2, x2; \
+ vpor x0, x3, x3; \
+ vpxor x4, x1, x1; \
+ vpxor x4, x3, x3; \
+ vpand x0, x4, x4; \
+ vpxor x2, x4, x4;
+
+#define get_key(i, j, t) \
+ vbroadcastss (4*(i)+(j))*4(CTX), t;
+
+#define K2(x0, x1, x2, x3, x4, i) \
+ get_key(i, 0, RK0); \
+ get_key(i, 1, RK1); \
+ get_key(i, 2, RK2); \
+ get_key(i, 3, RK3); \
+ vpxor RK0, x0 ## 1, x0 ## 1; \
+ vpxor RK1, x1 ## 1, x1 ## 1; \
+ vpxor RK2, x2 ## 1, x2 ## 1; \
+ vpxor RK3, x3 ## 1, x3 ## 1; \
+ vpxor RK0, x0 ## 2, x0 ## 2; \
+ vpxor RK1, x1 ## 2, x1 ## 2; \
+ vpxor RK2, x2 ## 2, x2 ## 2; \
+ vpxor RK3, x3 ## 2, x3 ## 2;
+
+#define LK2(x0, x1, x2, x3, x4, i) \
+ vpslld $13, x0 ## 1, x4 ## 1; \
+ vpsrld $(32 - 13), x0 ## 1, x0 ## 1; \
+ vpor x4 ## 1, x0 ## 1, x0 ## 1; \
+ vpxor x0 ## 1, x1 ## 1, x1 ## 1; \
+ vpslld $3, x2 ## 1, x4 ## 1; \
+ vpsrld $(32 - 3), x2 ## 1, x2 ## 1; \
+ vpor x4 ## 1, x2 ## 1, x2 ## 1; \
+ vpxor x2 ## 1, x1 ## 1, x1 ## 1; \
+ vpslld $13, x0 ## 2, x4 ## 2; \
+ vpsrld $(32 - 13), x0 ## 2, x0 ## 2; \
+ vpor x4 ## 2, x0 ## 2, x0 ## 2; \
+ vpxor x0 ## 2, x1 ## 2, x1 ## 2; \
+ vpslld $3, x2 ## 2, x4 ## 2; \
+ vpsrld $(32 - 3), x2 ## 2, x2 ## 2; \
+ vpor x4 ## 2, x2 ## 2, x2 ## 2; \
+ vpxor x2 ## 2, x1 ## 2, x1 ## 2; \
+ vpslld $1, x1 ## 1, x4 ## 1; \
+ vpsrld $(32 - 1), x1 ## 1, x1 ## 1; \
+ vpor x4 ## 1, x1 ## 1, x1 ## 1; \
+ vpslld $3, x0 ## 1, x4 ## 1; \
+ vpxor x2 ## 1, x3 ## 1, x3 ## 1; \
+ vpxor x4 ## 1, x3 ## 1, x3 ## 1; \
+ get_key(i, 1, RK1); \
+ vpslld $1, x1 ## 2, x4 ## 2; \
+ vpsrld $(32 - 1), x1 ## 2, x1 ## 2; \
+ vpor x4 ## 2, x1 ## 2, x1 ## 2; \
+ vpslld $3, x0 ## 2, x4 ## 2; \
+ vpxor x2 ## 2, x3 ## 2, x3 ## 2; \
+ vpxor x4 ## 2, x3 ## 2, x3 ## 2; \
+ get_key(i, 3, RK3); \
+ vpslld $7, x3 ## 1, x4 ## 1; \
+ vpsrld $(32 - 7), x3 ## 1, x3 ## 1; \
+ vpor x4 ## 1, x3 ## 1, x3 ## 1; \
+ vpslld $7, x1 ## 1, x4 ## 1; \
+ vpxor x1 ## 1, x0 ## 1, x0 ## 1; \
+ vpxor x3 ## 1, x0 ## 1, x0 ## 1; \
+ vpxor x3 ## 1, x2 ## 1, x2 ## 1; \
+ vpxor x4 ## 1, x2 ## 1, x2 ## 1; \
+ get_key(i, 0, RK0); \
+ vpslld $7, x3 ## 2, x4 ## 2; \
+ vpsrld $(32 - 7), x3 ## 2, x3 ## 2; \
+ vpor x4 ## 2, x3 ## 2, x3 ## 2; \
+ vpslld $7, x1 ## 2, x4 ## 2; \
+ vpxor x1 ## 2, x0 ## 2, x0 ## 2; \
+ vpxor x3 ## 2, x0 ## 2, x0 ## 2; \
+ vpxor x3 ## 2, x2 ## 2, x2 ## 2; \
+ vpxor x4 ## 2, x2 ## 2, x2 ## 2; \
+ get_key(i, 2, RK2); \
+ vpxor RK1, x1 ## 1, x1 ## 1; \
+ vpxor RK3, x3 ## 1, x3 ## 1; \
+ vpslld $5, x0 ## 1, x4 ## 1; \
+ vpsrld $(32 - 5), x0 ## 1, x0 ## 1; \
+ vpor x4 ## 1, x0 ## 1, x0 ## 1; \
+ vpslld $22, x2 ## 1, x4 ## 1; \
+ vpsrld $(32 - 22), x2 ## 1, x2 ## 1; \
+ vpor x4 ## 1, x2 ## 1, x2 ## 1; \
+ vpxor RK0, x0 ## 1, x0 ## 1; \
+ vpxor RK2, x2 ## 1, x2 ## 1; \
+ vpxor RK1, x1 ## 2, x1 ## 2; \
+ vpxor RK3, x3 ## 2, x3 ## 2; \
+ vpslld $5, x0 ## 2, x4 ## 2; \
+ vpsrld $(32 - 5), x0 ## 2, x0 ## 2; \
+ vpor x4 ## 2, x0 ## 2, x0 ## 2; \
+ vpslld $22, x2 ## 2, x4 ## 2; \
+ vpsrld $(32 - 22), x2 ## 2, x2 ## 2; \
+ vpor x4 ## 2, x2 ## 2, x2 ## 2; \
+ vpxor RK0, x0 ## 2, x0 ## 2; \
+ vpxor RK2, x2 ## 2, x2 ## 2;
+
+#define KL2(x0, x1, x2, x3, x4, i) \
+ vpxor RK0, x0 ## 1, x0 ## 1; \
+ vpxor RK2, x2 ## 1, x2 ## 1; \
+ vpsrld $5, x0 ## 1, x4 ## 1; \
+ vpslld $(32 - 5), x0 ## 1, x0 ## 1; \
+ vpor x4 ## 1, x0 ## 1, x0 ## 1; \
+ vpxor RK3, x3 ## 1, x3 ## 1; \
+ vpxor RK1, x1 ## 1, x1 ## 1; \
+ vpsrld $22, x2 ## 1, x4 ## 1; \
+ vpslld $(32 - 22), x2 ## 1, x2 ## 1; \
+ vpor x4 ## 1, x2 ## 1, x2 ## 1; \
+ vpxor x3 ## 1, x2 ## 1, x2 ## 1; \
+ vpxor RK0, x0 ## 2, x0 ## 2; \
+ vpxor RK2, x2 ## 2, x2 ## 2; \
+ vpsrld $5, x0 ## 2, x4 ## 2; \
+ vpslld $(32 - 5), x0 ## 2, x0 ## 2; \
+ vpor x4 ## 2, x0 ## 2, x0 ## 2; \
+ vpxor RK3, x3 ## 2, x3 ## 2; \
+ vpxor RK1, x1 ## 2, x1 ## 2; \
+ vpsrld $22, x2 ## 2, x4 ## 2; \
+ vpslld $(32 - 22), x2 ## 2, x2 ## 2; \
+ vpor x4 ## 2, x2 ## 2, x2 ## 2; \
+ vpxor x3 ## 2, x2 ## 2, x2 ## 2; \
+ vpxor x3 ## 1, x0 ## 1, x0 ## 1; \
+ vpslld $7, x1 ## 1, x4 ## 1; \
+ vpxor x1 ## 1, x0 ## 1, x0 ## 1; \
+ vpxor x4 ## 1, x2 ## 1, x2 ## 1; \
+ vpsrld $1, x1 ## 1, x4 ## 1; \
+ vpslld $(32 - 1), x1 ## 1, x1 ## 1; \
+ vpor x4 ## 1, x1 ## 1, x1 ## 1; \
+ vpxor x3 ## 2, x0 ## 2, x0 ## 2; \
+ vpslld $7, x1 ## 2, x4 ## 2; \
+ vpxor x1 ## 2, x0 ## 2, x0 ## 2; \
+ vpxor x4 ## 2, x2 ## 2, x2 ## 2; \
+ vpsrld $1, x1 ## 2, x4 ## 2; \
+ vpslld $(32 - 1), x1 ## 2, x1 ## 2; \
+ vpor x4 ## 2, x1 ## 2, x1 ## 2; \
+ vpsrld $7, x3 ## 1, x4 ## 1; \
+ vpslld $(32 - 7), x3 ## 1, x3 ## 1; \
+ vpor x4 ## 1, x3 ## 1, x3 ## 1; \
+ vpxor x0 ## 1, x1 ## 1, x1 ## 1; \
+ vpslld $3, x0 ## 1, x4 ## 1; \
+ vpxor x4 ## 1, x3 ## 1, x3 ## 1; \
+ vpsrld $7, x3 ## 2, x4 ## 2; \
+ vpslld $(32 - 7), x3 ## 2, x3 ## 2; \
+ vpor x4 ## 2, x3 ## 2, x3 ## 2; \
+ vpxor x0 ## 2, x1 ## 2, x1 ## 2; \
+ vpslld $3, x0 ## 2, x4 ## 2; \
+ vpxor x4 ## 2, x3 ## 2, x3 ## 2; \
+ vpsrld $13, x0 ## 1, x4 ## 1; \
+ vpslld $(32 - 13), x0 ## 1, x0 ## 1; \
+ vpor x4 ## 1, x0 ## 1, x0 ## 1; \
+ vpxor x2 ## 1, x1 ## 1, x1 ## 1; \
+ vpxor x2 ## 1, x3 ## 1, x3 ## 1; \
+ vpsrld $3, x2 ## 1, x4 ## 1; \
+ vpslld $(32 - 3), x2 ## 1, x2 ## 1; \
+ vpor x4 ## 1, x2 ## 1, x2 ## 1; \
+ vpsrld $13, x0 ## 2, x4 ## 2; \
+ vpslld $(32 - 13), x0 ## 2, x0 ## 2; \
+ vpor x4 ## 2, x0 ## 2, x0 ## 2; \
+ vpxor x2 ## 2, x1 ## 2, x1 ## 2; \
+ vpxor x2 ## 2, x3 ## 2, x3 ## 2; \
+ vpsrld $3, x2 ## 2, x4 ## 2; \
+ vpslld $(32 - 3), x2 ## 2, x2 ## 2; \
+ vpor x4 ## 2, x2 ## 2, x2 ## 2;
+
+#define S(SBOX, x0, x1, x2, x3, x4) \
+ SBOX ## _1(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \
+ SBOX ## _2(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \
+ SBOX ## _1(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2); \
+ SBOX ## _2(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2);
+
+#define SP(SBOX, x0, x1, x2, x3, x4, i) \
+ get_key(i, 0, RK0); \
+ SBOX ## _1(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \
+ get_key(i, 2, RK2); \
+ SBOX ## _2(x0 ## 1, x1 ## 1, x2 ## 1, x3 ## 1, x4 ## 1); \
+ get_key(i, 3, RK3); \
+ SBOX ## _1(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2); \
+ get_key(i, 1, RK1); \
+ SBOX ## _2(x0 ## 2, x1 ## 2, x2 ## 2, x3 ## 2, x4 ## 2); \
+
+#define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
+ vpunpckldq x1, x0, t0; \
+ vpunpckhdq x1, x0, t2; \
+ vpunpckldq x3, x2, t1; \
+ vpunpckhdq x3, x2, x3; \
+ \
+ vpunpcklqdq t1, t0, x0; \
+ vpunpckhqdq t1, t0, x1; \
+ vpunpcklqdq x3, t2, x2; \
+ vpunpckhqdq x3, t2, x3;
+
+#define read_blocks(in, x0, x1, x2, x3, t0, t1, t2) \
+ vmovdqu (0*4*4)(in), x0; \
+ vmovdqu (1*4*4)(in), x1; \
+ vmovdqu (2*4*4)(in), x2; \
+ vmovdqu (3*4*4)(in), x3; \
+ \
+ transpose_4x4(x0, x1, x2, x3, t0, t1, t2)
+
+#define write_blocks(out, x0, x1, x2, x3, t0, t1, t2) \
+ transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
+ \
+ vmovdqu x0, (0*4*4)(out); \
+ vmovdqu x1, (1*4*4)(out); \
+ vmovdqu x2, (2*4*4)(out); \
+ vmovdqu x3, (3*4*4)(out);
+
+#define xor_blocks(out, x0, x1, x2, x3, t0, t1, t2) \
+ transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
+ \
+ vpxor (0*4*4)(out), x0, x0; \
+ vmovdqu x0, (0*4*4)(out); \
+ vpxor (1*4*4)(out), x1, x1; \
+ vmovdqu x1, (1*4*4)(out); \
+ vpxor (2*4*4)(out), x2, x2; \
+ vmovdqu x2, (2*4*4)(out); \
+ vpxor (3*4*4)(out), x3, x3; \
+ vmovdqu x3, (3*4*4)(out);
+
+.align 8
+.global __serpent_enc_blk_8way_avx
+.type __serpent_enc_blk_8way_avx,@function;
+
+__serpent_enc_blk_8way_avx:
+ /* input:
+ * %rdi: ctx, CTX
+ * %rsi: dst
+ * %rdx: src
+ * %rcx: bool, if true: xor output
+ */
+
+ vpcmpeqd RNOT, RNOT, RNOT;
+
+ leaq (4*4*4)(%rdx), %rax;
+ read_blocks(%rdx, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
+ read_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
+
+ K2(RA, RB, RC, RD, RE, 0);
+ S(S0, RA, RB, RC, RD, RE); LK2(RC, RB, RD, RA, RE, 1);
+ S(S1, RC, RB, RD, RA, RE); LK2(RE, RD, RA, RC, RB, 2);
+ S(S2, RE, RD, RA, RC, RB); LK2(RB, RD, RE, RC, RA, 3);
+ S(S3, RB, RD, RE, RC, RA); LK2(RC, RA, RD, RB, RE, 4);
+ S(S4, RC, RA, RD, RB, RE); LK2(RA, RD, RB, RE, RC, 5);
+ S(S5, RA, RD, RB, RE, RC); LK2(RC, RA, RD, RE, RB, 6);
+ S(S6, RC, RA, RD, RE, RB); LK2(RD, RB, RA, RE, RC, 7);
+ S(S7, RD, RB, RA, RE, RC); LK2(RC, RA, RE, RD, RB, 8);
+ S(S0, RC, RA, RE, RD, RB); LK2(RE, RA, RD, RC, RB, 9);
+ S(S1, RE, RA, RD, RC, RB); LK2(RB, RD, RC, RE, RA, 10);
+ S(S2, RB, RD, RC, RE, RA); LK2(RA, RD, RB, RE, RC, 11);
+ S(S3, RA, RD, RB, RE, RC); LK2(RE, RC, RD, RA, RB, 12);
+ S(S4, RE, RC, RD, RA, RB); LK2(RC, RD, RA, RB, RE, 13);
+ S(S5, RC, RD, RA, RB, RE); LK2(RE, RC, RD, RB, RA, 14);
+ S(S6, RE, RC, RD, RB, RA); LK2(RD, RA, RC, RB, RE, 15);
+ S(S7, RD, RA, RC, RB, RE); LK2(RE, RC, RB, RD, RA, 16);
+ S(S0, RE, RC, RB, RD, RA); LK2(RB, RC, RD, RE, RA, 17);
+ S(S1, RB, RC, RD, RE, RA); LK2(RA, RD, RE, RB, RC, 18);
+ S(S2, RA, RD, RE, RB, RC); LK2(RC, RD, RA, RB, RE, 19);
+ S(S3, RC, RD, RA, RB, RE); LK2(RB, RE, RD, RC, RA, 20);
+ S(S4, RB, RE, RD, RC, RA); LK2(RE, RD, RC, RA, RB, 21);
+ S(S5, RE, RD, RC, RA, RB); LK2(RB, RE, RD, RA, RC, 22);
+ S(S6, RB, RE, RD, RA, RC); LK2(RD, RC, RE, RA, RB, 23);
+ S(S7, RD, RC, RE, RA, RB); LK2(RB, RE, RA, RD, RC, 24);
+ S(S0, RB, RE, RA, RD, RC); LK2(RA, RE, RD, RB, RC, 25);
+ S(S1, RA, RE, RD, RB, RC); LK2(RC, RD, RB, RA, RE, 26);
+ S(S2, RC, RD, RB, RA, RE); LK2(RE, RD, RC, RA, RB, 27);
+ S(S3, RE, RD, RC, RA, RB); LK2(RA, RB, RD, RE, RC, 28);
+ S(S4, RA, RB, RD, RE, RC); LK2(RB, RD, RE, RC, RA, 29);
+ S(S5, RB, RD, RE, RC, RA); LK2(RA, RB, RD, RC, RE, 30);
+ S(S6, RA, RB, RD, RC, RE); LK2(RD, RE, RB, RC, RA, 31);
+ S(S7, RD, RE, RB, RC, RA); K2(RA, RB, RC, RD, RE, 32);
+
+ leaq (4*4*4)(%rsi), %rax;
+
+ testb %cl, %cl;
+ jnz __enc_xor8;
+
+ write_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
+ write_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
+
+ ret;
+
+__enc_xor8:
+ xor_blocks(%rsi, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
+ xor_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
+
+ ret;
+
+.align 8
+.global serpent_dec_blk_8way_avx
+.type serpent_dec_blk_8way_avx,@function;
+
+serpent_dec_blk_8way_avx:
+ /* input:
+ * %rdi: ctx, CTX
+ * %rsi: dst
+ * %rdx: src
+ */
+
+ vpcmpeqd RNOT, RNOT, RNOT;
+
+ leaq (4*4*4)(%rdx), %rax;
+ read_blocks(%rdx, RA1, RB1, RC1, RD1, RK0, RK1, RK2);
+ read_blocks(%rax, RA2, RB2, RC2, RD2, RK0, RK1, RK2);
+
+ K2(RA, RB, RC, RD, RE, 32);
+ SP(SI7, RA, RB, RC, RD, RE, 31); KL2(RB, RD, RA, RE, RC, 31);
+ SP(SI6, RB, RD, RA, RE, RC, 30); KL2(RA, RC, RE, RB, RD, 30);
+ SP(SI5, RA, RC, RE, RB, RD, 29); KL2(RC, RD, RA, RE, RB, 29);
+ SP(SI4, RC, RD, RA, RE, RB, 28); KL2(RC, RA, RB, RE, RD, 28);
+ SP(SI3, RC, RA, RB, RE, RD, 27); KL2(RB, RC, RD, RE, RA, 27);
+ SP(SI2, RB, RC, RD, RE, RA, 26); KL2(RC, RA, RE, RD, RB, 26);
+ SP(SI1, RC, RA, RE, RD, RB, 25); KL2(RB, RA, RE, RD, RC, 25);
+ SP(SI0, RB, RA, RE, RD, RC, 24); KL2(RE, RC, RA, RB, RD, 24);
+ SP(SI7, RE, RC, RA, RB, RD, 23); KL2(RC, RB, RE, RD, RA, 23);
+ SP(SI6, RC, RB, RE, RD, RA, 22); KL2(RE, RA, RD, RC, RB, 22);
+ SP(SI5, RE, RA, RD, RC, RB, 21); KL2(RA, RB, RE, RD, RC, 21);
+ SP(SI4, RA, RB, RE, RD, RC, 20); KL2(RA, RE, RC, RD, RB, 20);
+ SP(SI3, RA, RE, RC, RD, RB, 19); KL2(RC, RA, RB, RD, RE, 19);
+ SP(SI2, RC, RA, RB, RD, RE, 18); KL2(RA, RE, RD, RB, RC, 18);
+ SP(SI1, RA, RE, RD, RB, RC, 17); KL2(RC, RE, RD, RB, RA, 17);
+ SP(SI0, RC, RE, RD, RB, RA, 16); KL2(RD, RA, RE, RC, RB, 16);
+ SP(SI7, RD, RA, RE, RC, RB, 15); KL2(RA, RC, RD, RB, RE, 15);
+ SP(SI6, RA, RC, RD, RB, RE, 14); KL2(RD, RE, RB, RA, RC, 14);
+ SP(SI5, RD, RE, RB, RA, RC, 13); KL2(RE, RC, RD, RB, RA, 13);
+ SP(SI4, RE, RC, RD, RB, RA, 12); KL2(RE, RD, RA, RB, RC, 12);
+ SP(SI3, RE, RD, RA, RB, RC, 11); KL2(RA, RE, RC, RB, RD, 11);
+ SP(SI2, RA, RE, RC, RB, RD, 10); KL2(RE, RD, RB, RC, RA, 10);
+ SP(SI1, RE, RD, RB, RC, RA, 9); KL2(RA, RD, RB, RC, RE, 9);
+ SP(SI0, RA, RD, RB, RC, RE, 8); KL2(RB, RE, RD, RA, RC, 8);
+ SP(SI7, RB, RE, RD, RA, RC, 7); KL2(RE, RA, RB, RC, RD, 7);
+ SP(SI6, RE, RA, RB, RC, RD, 6); KL2(RB, RD, RC, RE, RA, 6);
+ SP(SI5, RB, RD, RC, RE, RA, 5); KL2(RD, RA, RB, RC, RE, 5);
+ SP(SI4, RD, RA, RB, RC, RE, 4); KL2(RD, RB, RE, RC, RA, 4);
+ SP(SI3, RD, RB, RE, RC, RA, 3); KL2(RE, RD, RA, RC, RB, 3);
+ SP(SI2, RE, RD, RA, RC, RB, 2); KL2(RD, RB, RC, RA, RE, 2);
+ SP(SI1, RD, RB, RC, RA, RE, 1); KL2(RE, RB, RC, RA, RD, 1);
+ S(SI0, RE, RB, RC, RA, RD); K2(RC, RD, RB, RE, RA, 0);
+
+ leaq (4*4*4)(%rsi), %rax;
+ write_blocks(%rsi, RC1, RD1, RB1, RE1, RK0, RK1, RK2);
+ write_blocks(%rax, RC2, RD2, RB2, RE2, RK0, RK1, RK2);
+
+ ret;
diff --git a/arch/x86/crypto/serpent_avx_glue.c b/arch/x86/crypto/serpent_avx_glue.c
new file mode 100644
index 000000000000..b36bdac237eb
--- /dev/null
+++ b/arch/x86/crypto/serpent_avx_glue.c
@@ -0,0 +1,636 @@
+/*
+ * Glue Code for AVX assembler versions of Serpent Cipher
+ *
+ * Copyright (C) 2012 Johannes Goetzfried
+ * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
+ *
+ * Glue code based on serpent_sse2_glue.c by:
+ * Copyright (C) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/hardirq.h>
+#include <linux/types.h>
+#include <linux/crypto.h>
+#include <linux/err.h>
+#include <crypto/algapi.h>
+#include <crypto/serpent.h>
+#include <crypto/cryptd.h>
+#include <crypto/b128ops.h>
+#include <crypto/ctr.h>
+#include <crypto/lrw.h>
+#include <crypto/xts.h>
+#include <asm/xcr.h>
+#include <asm/xsave.h>
+#include <asm/crypto/serpent-avx.h>
+#include <asm/crypto/ablk_helper.h>
+#include <asm/crypto/glue_helper.h>
+
+static void serpent_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src)
+{
+ u128 ivs[SERPENT_PARALLEL_BLOCKS - 1];
+ unsigned int j;
+
+ for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
+ ivs[j] = src[j];
+
+ serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
+
+ for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
+ u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);
+}
+
+static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)
+{
+ be128 ctrblk;
+
+ u128_to_be128(&ctrblk, iv);
+ u128_inc(iv);
+
+ __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
+ u128_xor(dst, src, (u128 *)&ctrblk);
+}
+
+static void serpent_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src,
+ u128 *iv)
+{
+ be128 ctrblks[SERPENT_PARALLEL_BLOCKS];
+ unsigned int i;
+
+ for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) {
+ if (dst != src)
+ dst[i] = src[i];
+
+ u128_to_be128(&ctrblks[i], iv);
+ u128_inc(iv);
+ }
+
+ serpent_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks);
+}
+
+static const struct common_glue_ctx serpent_enc = {
+ .num_funcs = 2,
+ .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
+
+ .funcs = { {
+ .num_blocks = SERPENT_PARALLEL_BLOCKS,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_enc_blk_xway) }
+ }, {
+ .num_blocks = 1,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
+ } }
+};
+
+static const struct common_glue_ctx serpent_ctr = {
+ .num_funcs = 2,
+ .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
+
+ .funcs = { {
+ .num_blocks = SERPENT_PARALLEL_BLOCKS,
+ .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr_xway) }
+ }, {
+ .num_blocks = 1,
+ .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr) }
+ } }
+};
+
+static const struct common_glue_ctx serpent_dec = {
+ .num_funcs = 2,
+ .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
+
+ .funcs = { {
+ .num_blocks = SERPENT_PARALLEL_BLOCKS,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_dec_blk_xway) }
+ }, {
+ .num_blocks = 1,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
+ } }
+};
+
+static const struct common_glue_ctx serpent_dec_cbc = {
+ .num_funcs = 2,
+ .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
+
+ .funcs = { {
+ .num_blocks = SERPENT_PARALLEL_BLOCKS,
+ .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_decrypt_cbc_xway) }
+ }, {
+ .num_blocks = 1,
+ .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
+ } }
+};
+
+static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ return glue_ecb_crypt_128bit(&serpent_enc, desc, dst, src, nbytes);
+}
+
+static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ return glue_ecb_crypt_128bit(&serpent_dec, desc, dst, src, nbytes);
+}
+
+static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__serpent_encrypt), desc,
+ dst, src, nbytes);
+}
+
+static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ return glue_cbc_decrypt_128bit(&serpent_dec_cbc, desc, dst, src,
+ nbytes);
+}
+
+static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ return glue_ctr_crypt_128bit(&serpent_ctr, desc, dst, src, nbytes);
+}
+
+static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes)
+{
+ return glue_fpu_begin(SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS,
+ NULL, fpu_enabled, nbytes);
+}
+
+static inline void serpent_fpu_end(bool fpu_enabled)
+{
+ glue_fpu_end(fpu_enabled);
+}
+
+struct crypt_priv {
+ struct serpent_ctx *ctx;
+ bool fpu_enabled;
+};
+
+static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
+{
+ const unsigned int bsize = SERPENT_BLOCK_SIZE;
+ struct crypt_priv *ctx = priv;
+ int i;
+
+ ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
+
+ if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
+ serpent_enc_blk_xway(ctx->ctx, srcdst, srcdst);
+ return;
+ }
+
+ for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
+ __serpent_encrypt(ctx->ctx, srcdst, srcdst);
+}
+
+static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
+{
+ const unsigned int bsize = SERPENT_BLOCK_SIZE;
+ struct crypt_priv *ctx = priv;
+ int i;
+
+ ctx->fpu_enabled = serpent_fpu_begin(ctx->fpu_enabled, nbytes);
+
+ if (nbytes == bsize * SERPENT_PARALLEL_BLOCKS) {
+ serpent_dec_blk_xway(ctx->ctx, srcdst, srcdst);
+ return;
+ }
+
+ for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
+ __serpent_decrypt(ctx->ctx, srcdst, srcdst);
+}
+
+struct serpent_lrw_ctx {
+ struct lrw_table_ctx lrw_table;
+ struct serpent_ctx serpent_ctx;
+};
+
+static int lrw_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
+ int err;
+
+ err = __serpent_setkey(&ctx->serpent_ctx, key, keylen -
+ SERPENT_BLOCK_SIZE);
+ if (err)
+ return err;
+
+ return lrw_init_table(&ctx->lrw_table, key + keylen -
+ SERPENT_BLOCK_SIZE);
+}
+
+static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+ be128 buf[SERPENT_PARALLEL_BLOCKS];
+ struct crypt_priv crypt_ctx = {
+ .ctx = &ctx->serpent_ctx,
+ .fpu_enabled = false,
+ };
+ struct lrw_crypt_req req = {
+ .tbuf = buf,
+ .tbuflen = sizeof(buf),
+
+ .table_ctx = &ctx->lrw_table,
+ .crypt_ctx = &crypt_ctx,
+ .crypt_fn = encrypt_callback,
+ };
+ int ret;
+
+ desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+ ret = lrw_crypt(desc, dst, src, nbytes, &req);
+ serpent_fpu_end(crypt_ctx.fpu_enabled);
+
+ return ret;
+}
+
+static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ struct serpent_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+ be128 buf[SERPENT_PARALLEL_BLOCKS];
+ struct crypt_priv crypt_ctx = {
+ .ctx = &ctx->serpent_ctx,
+ .fpu_enabled = false,
+ };
+ struct lrw_crypt_req req = {
+ .tbuf = buf,
+ .tbuflen = sizeof(buf),
+
+ .table_ctx = &ctx->lrw_table,
+ .crypt_ctx = &crypt_ctx,
+ .crypt_fn = decrypt_callback,
+ };
+ int ret;
+
+ desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+ ret = lrw_crypt(desc, dst, src, nbytes, &req);
+ serpent_fpu_end(crypt_ctx.fpu_enabled);
+
+ return ret;
+}
+
+static void lrw_exit_tfm(struct crypto_tfm *tfm)
+{
+ struct serpent_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
+
+ lrw_free_table(&ctx->lrw_table);
+}
+
+struct serpent_xts_ctx {
+ struct serpent_ctx tweak_ctx;
+ struct serpent_ctx crypt_ctx;
+};
+
+static int xts_serpent_setkey(struct crypto_tfm *tfm, const u8 *key,
+ unsigned int keylen)
+{
+ struct serpent_xts_ctx *ctx = crypto_tfm_ctx(tfm);
+ u32 *flags = &tfm->crt_flags;
+ int err;
+
+ /* key consists of keys of equal size concatenated, therefore
+ * the length must be even
+ */
+ if (keylen % 2) {
+ *flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
+ return -EINVAL;
+ }
+
+ /* first half of xts-key is for crypt */
+ err = __serpent_setkey(&ctx->crypt_ctx, key, keylen / 2);
+ if (err)
+ return err;
+
+ /* second half of xts-key is for tweak */
+ return __serpent_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2);
+}
+
+static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+ be128 buf[SERPENT_PARALLEL_BLOCKS];
+ struct crypt_priv crypt_ctx = {
+ .ctx = &ctx->crypt_ctx,
+ .fpu_enabled = false,
+ };
+ struct xts_crypt_req req = {
+ .tbuf = buf,
+ .tbuflen = sizeof(buf),
+
+ .tweak_ctx = &ctx->tweak_ctx,
+ .tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt),
+ .crypt_ctx = &crypt_ctx,
+ .crypt_fn = encrypt_callback,
+ };
+ int ret;
+
+ desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+ ret = xts_crypt(desc, dst, src, nbytes, &req);
+ serpent_fpu_end(crypt_ctx.fpu_enabled);
+
+ return ret;
+}
+
+static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ struct serpent_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+ be128 buf[SERPENT_PARALLEL_BLOCKS];
+ struct crypt_priv crypt_ctx = {
+ .ctx = &ctx->crypt_ctx,
+ .fpu_enabled = false,
+ };
+ struct xts_crypt_req req = {
+ .tbuf = buf,
+ .tbuflen = sizeof(buf),
+
+ .tweak_ctx = &ctx->tweak_ctx,
+ .tweak_fn = XTS_TWEAK_CAST(__serpent_encrypt),
+ .crypt_ctx = &crypt_ctx,
+ .crypt_fn = decrypt_callback,
+ };
+ int ret;
+
+ desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+ ret = xts_crypt(desc, dst, src, nbytes, &req);
+ serpent_fpu_end(crypt_ctx.fpu_enabled);
+
+ return ret;
+}
+
+static struct crypto_alg serpent_algs[10] = { {
+ .cra_name = "__ecb-serpent-avx",
+ .cra_driver_name = "__driver-ecb-serpent-avx",
+ .cra_priority = 0,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
+ .cra_blocksize = SERPENT_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct serpent_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_blkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(serpent_algs[0].cra_list),
+ .cra_u = {
+ .blkcipher = {
+ .min_keysize = SERPENT_MIN_KEY_SIZE,
+ .max_keysize = SERPENT_MAX_KEY_SIZE,
+ .setkey = serpent_setkey,
+ .encrypt = ecb_encrypt,
+ .decrypt = ecb_decrypt,
+ },
+ },
+}, {
+ .cra_name = "__cbc-serpent-avx",
+ .cra_driver_name = "__driver-cbc-serpent-avx",
+ .cra_priority = 0,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
+ .cra_blocksize = SERPENT_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct serpent_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_blkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(serpent_algs[1].cra_list),
+ .cra_u = {
+ .blkcipher = {
+ .min_keysize = SERPENT_MIN_KEY_SIZE,
+ .max_keysize = SERPENT_MAX_KEY_SIZE,
+ .setkey = serpent_setkey,
+ .encrypt = cbc_encrypt,
+ .decrypt = cbc_decrypt,
+ },
+ },
+}, {
+ .cra_name = "__ctr-serpent-avx",
+ .cra_driver_name = "__driver-ctr-serpent-avx",
+ .cra_priority = 0,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct serpent_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_blkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(serpent_algs[2].cra_list),
+ .cra_u = {
+ .blkcipher = {
+ .min_keysize = SERPENT_MIN_KEY_SIZE,
+ .max_keysize = SERPENT_MAX_KEY_SIZE,
+ .ivsize = SERPENT_BLOCK_SIZE,
+ .setkey = serpent_setkey,
+ .encrypt = ctr_crypt,
+ .decrypt = ctr_crypt,
+ },
+ },
+}, {
+ .cra_name = "__lrw-serpent-avx",
+ .cra_driver_name = "__driver-lrw-serpent-avx",
+ .cra_priority = 0,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
+ .cra_blocksize = SERPENT_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct serpent_lrw_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_blkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(serpent_algs[3].cra_list),
+ .cra_exit = lrw_exit_tfm,
+ .cra_u = {
+ .blkcipher = {
+ .min_keysize = SERPENT_MIN_KEY_SIZE +
+ SERPENT_BLOCK_SIZE,
+ .max_keysize = SERPENT_MAX_KEY_SIZE +
+ SERPENT_BLOCK_SIZE,
+ .ivsize = SERPENT_BLOCK_SIZE,
+ .setkey = lrw_serpent_setkey,
+ .encrypt = lrw_encrypt,
+ .decrypt = lrw_decrypt,
+ },
+ },
+}, {
+ .cra_name = "__xts-serpent-avx",
+ .cra_driver_name = "__driver-xts-serpent-avx",
+ .cra_priority = 0,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
+ .cra_blocksize = SERPENT_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct serpent_xts_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_blkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(serpent_algs[4].cra_list),
+ .cra_u = {
+ .blkcipher = {
+ .min_keysize = SERPENT_MIN_KEY_SIZE * 2,
+ .max_keysize = SERPENT_MAX_KEY_SIZE * 2,
+ .ivsize = SERPENT_BLOCK_SIZE,
+ .setkey = xts_serpent_setkey,
+ .encrypt = xts_encrypt,
+ .decrypt = xts_decrypt,
+ },
+ },
+}, {
+ .cra_name = "ecb(serpent)",
+ .cra_driver_name = "ecb-serpent-avx",
+ .cra_priority = 500,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+ .cra_blocksize = SERPENT_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct async_helper_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(serpent_algs[5].cra_list),
+ .cra_init = ablk_init,
+ .cra_exit = ablk_exit,
+ .cra_u = {
+ .ablkcipher = {
+ .min_keysize = SERPENT_MIN_KEY_SIZE,
+ .max_keysize = SERPENT_MAX_KEY_SIZE,
+ .setkey = ablk_set_key,
+ .encrypt = ablk_encrypt,
+ .decrypt = ablk_decrypt,
+ },
+ },
+}, {
+ .cra_name = "cbc(serpent)",
+ .cra_driver_name = "cbc-serpent-avx",
+ .cra_priority = 500,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+ .cra_blocksize = SERPENT_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct async_helper_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(serpent_algs[6].cra_list),
+ .cra_init = ablk_init,
+ .cra_exit = ablk_exit,
+ .cra_u = {
+ .ablkcipher = {
+ .min_keysize = SERPENT_MIN_KEY_SIZE,
+ .max_keysize = SERPENT_MAX_KEY_SIZE,
+ .ivsize = SERPENT_BLOCK_SIZE,
+ .setkey = ablk_set_key,
+ .encrypt = __ablk_encrypt,
+ .decrypt = ablk_decrypt,
+ },
+ },
+}, {
+ .cra_name = "ctr(serpent)",
+ .cra_driver_name = "ctr-serpent-avx",
+ .cra_priority = 500,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct async_helper_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(serpent_algs[7].cra_list),
+ .cra_init = ablk_init,
+ .cra_exit = ablk_exit,
+ .cra_u = {
+ .ablkcipher = {
+ .min_keysize = SERPENT_MIN_KEY_SIZE,
+ .max_keysize = SERPENT_MAX_KEY_SIZE,
+ .ivsize = SERPENT_BLOCK_SIZE,
+ .setkey = ablk_set_key,
+ .encrypt = ablk_encrypt,
+ .decrypt = ablk_encrypt,
+ .geniv = "chainiv",
+ },
+ },
+}, {
+ .cra_name = "lrw(serpent)",
+ .cra_driver_name = "lrw-serpent-avx",
+ .cra_priority = 500,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+ .cra_blocksize = SERPENT_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct async_helper_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(serpent_algs[8].cra_list),
+ .cra_init = ablk_init,
+ .cra_exit = ablk_exit,
+ .cra_u = {
+ .ablkcipher = {
+ .min_keysize = SERPENT_MIN_KEY_SIZE +
+ SERPENT_BLOCK_SIZE,
+ .max_keysize = SERPENT_MAX_KEY_SIZE +
+ SERPENT_BLOCK_SIZE,
+ .ivsize = SERPENT_BLOCK_SIZE,
+ .setkey = ablk_set_key,
+ .encrypt = ablk_encrypt,
+ .decrypt = ablk_decrypt,
+ },
+ },
+}, {
+ .cra_name = "xts(serpent)",
+ .cra_driver_name = "xts-serpent-avx",
+ .cra_priority = 500,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+ .cra_blocksize = SERPENT_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct async_helper_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(serpent_algs[9].cra_list),
+ .cra_init = ablk_init,
+ .cra_exit = ablk_exit,
+ .cra_u = {
+ .ablkcipher = {
+ .min_keysize = SERPENT_MIN_KEY_SIZE * 2,
+ .max_keysize = SERPENT_MAX_KEY_SIZE * 2,
+ .ivsize = SERPENT_BLOCK_SIZE,
+ .setkey = ablk_set_key,
+ .encrypt = ablk_encrypt,
+ .decrypt = ablk_decrypt,
+ },
+ },
+} };
+
+static int __init serpent_init(void)
+{
+ u64 xcr0;
+
+ if (!cpu_has_avx || !cpu_has_osxsave) {
+ printk(KERN_INFO "AVX instructions are not detected.\n");
+ return -ENODEV;
+ }
+
+ xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
+ if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
+ printk(KERN_INFO "AVX detected but unusable.\n");
+ return -ENODEV;
+ }
+
+ return crypto_register_algs(serpent_algs, ARRAY_SIZE(serpent_algs));
+}
+
+static void __exit serpent_exit(void)
+{
+ crypto_unregister_algs(serpent_algs, ARRAY_SIZE(serpent_algs));
+}
+
+module_init(serpent_init);
+module_exit(serpent_exit);
+
+MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX optimized");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("serpent");
diff --git a/arch/x86/crypto/serpent_sse2_glue.c b/arch/x86/crypto/serpent_sse2_glue.c
index 4b21be85e0a1..d679c8675f4a 100644
--- a/arch/x86/crypto/serpent_sse2_glue.c
+++ b/arch/x86/crypto/serpent_sse2_glue.c
@@ -41,358 +41,145 @@
#include <crypto/ctr.h>
#include <crypto/lrw.h>
#include <crypto/xts.h>
-#include <asm/i387.h>
-#include <asm/serpent.h>
-#include <crypto/scatterwalk.h>
-#include <linux/workqueue.h>
-#include <linux/spinlock.h>
-
-struct async_serpent_ctx {
- struct cryptd_ablkcipher *cryptd_tfm;
-};
+#include <asm/crypto/serpent-sse2.h>
+#include <asm/crypto/ablk_helper.h>
+#include <asm/crypto/glue_helper.h>
-static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes)
-{
- if (fpu_enabled)
- return true;
-
- /* SSE2 is only used when chunk to be processed is large enough, so
- * do not enable FPU until it is necessary.
- */
- if (nbytes < SERPENT_BLOCK_SIZE * SERPENT_PARALLEL_BLOCKS)
- return false;
-
- kernel_fpu_begin();
- return true;
-}
-
-static inline void serpent_fpu_end(bool fpu_enabled)
+static void serpent_decrypt_cbc_xway(void *ctx, u128 *dst, const u128 *src)
{
- if (fpu_enabled)
- kernel_fpu_end();
-}
-
-static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
- bool enc)
-{
- bool fpu_enabled = false;
- struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- const unsigned int bsize = SERPENT_BLOCK_SIZE;
- unsigned int nbytes;
- int err;
-
- err = blkcipher_walk_virt(desc, walk);
- desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
-
- while ((nbytes = walk->nbytes)) {
- u8 *wsrc = walk->src.virt.addr;
- u8 *wdst = walk->dst.virt.addr;
-
- fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes);
-
- /* Process multi-block batch */
- if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) {
- do {
- if (enc)
- serpent_enc_blk_xway(ctx, wdst, wsrc);
- else
- serpent_dec_blk_xway(ctx, wdst, wsrc);
-
- wsrc += bsize * SERPENT_PARALLEL_BLOCKS;
- wdst += bsize * SERPENT_PARALLEL_BLOCKS;
- nbytes -= bsize * SERPENT_PARALLEL_BLOCKS;
- } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS);
-
- if (nbytes < bsize)
- goto done;
- }
-
- /* Handle leftovers */
- do {
- if (enc)
- __serpent_encrypt(ctx, wdst, wsrc);
- else
- __serpent_decrypt(ctx, wdst, wsrc);
-
- wsrc += bsize;
- wdst += bsize;
- nbytes -= bsize;
- } while (nbytes >= bsize);
-
-done:
- err = blkcipher_walk_done(desc, walk, nbytes);
- }
+ u128 ivs[SERPENT_PARALLEL_BLOCKS - 1];
+ unsigned int j;
- serpent_fpu_end(fpu_enabled);
- return err;
-}
+ for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
+ ivs[j] = src[j];
-static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
- struct scatterlist *src, unsigned int nbytes)
-{
- struct blkcipher_walk walk;
+ serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
- blkcipher_walk_init(&walk, dst, src, nbytes);
- return ecb_crypt(desc, &walk, true);
+ for (j = 0; j < SERPENT_PARALLEL_BLOCKS - 1; j++)
+ u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);
}
-static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
- struct scatterlist *src, unsigned int nbytes)
+static void serpent_crypt_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)
{
- struct blkcipher_walk walk;
+ be128 ctrblk;
- blkcipher_walk_init(&walk, dst, src, nbytes);
- return ecb_crypt(desc, &walk, false);
-}
+ u128_to_be128(&ctrblk, iv);
+ u128_inc(iv);
-static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
- struct blkcipher_walk *walk)
-{
- struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- const unsigned int bsize = SERPENT_BLOCK_SIZE;
- unsigned int nbytes = walk->nbytes;
- u128 *src = (u128 *)walk->src.virt.addr;
- u128 *dst = (u128 *)walk->dst.virt.addr;
- u128 *iv = (u128 *)walk->iv;
-
- do {
- u128_xor(dst, src, iv);
- __serpent_encrypt(ctx, (u8 *)dst, (u8 *)dst);
- iv = dst;
-
- src += 1;
- dst += 1;
- nbytes -= bsize;
- } while (nbytes >= bsize);
-
- u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv);
- return nbytes;
+ __serpent_encrypt(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
+ u128_xor(dst, src, (u128 *)&ctrblk);
}
-static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
- struct scatterlist *src, unsigned int nbytes)
+static void serpent_crypt_ctr_xway(void *ctx, u128 *dst, const u128 *src,
+ u128 *iv)
{
- struct blkcipher_walk walk;
- int err;
+ be128 ctrblks[SERPENT_PARALLEL_BLOCKS];
+ unsigned int i;
- blkcipher_walk_init(&walk, dst, src, nbytes);
- err = blkcipher_walk_virt(desc, &walk);
+ for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) {
+ if (dst != src)
+ dst[i] = src[i];
- while ((nbytes = walk.nbytes)) {
- nbytes = __cbc_encrypt(desc, &walk);
- err = blkcipher_walk_done(desc, &walk, nbytes);
+ u128_to_be128(&ctrblks[i], iv);
+ u128_inc(iv);
}
- return err;
+ serpent_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks);
}
-static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
- struct blkcipher_walk *walk)
-{
- struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- const unsigned int bsize = SERPENT_BLOCK_SIZE;
- unsigned int nbytes = walk->nbytes;
- u128 *src = (u128 *)walk->src.virt.addr;
- u128 *dst = (u128 *)walk->dst.virt.addr;
- u128 ivs[SERPENT_PARALLEL_BLOCKS - 1];
- u128 last_iv;
- int i;
-
- /* Start of the last block. */
- src += nbytes / bsize - 1;
- dst += nbytes / bsize - 1;
-
- last_iv = *src;
-
- /* Process multi-block batch */
- if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) {
- do {
- nbytes -= bsize * (SERPENT_PARALLEL_BLOCKS - 1);
- src -= SERPENT_PARALLEL_BLOCKS - 1;
- dst -= SERPENT_PARALLEL_BLOCKS - 1;
-
- for (i = 0; i < SERPENT_PARALLEL_BLOCKS - 1; i++)
- ivs[i] = src[i];
-
- serpent_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
-
- for (i = 0; i < SERPENT_PARALLEL_BLOCKS - 1; i++)
- u128_xor(dst + (i + 1), dst + (i + 1), ivs + i);
-
- nbytes -= bsize;
- if (nbytes < bsize)
- goto done;
+static const struct common_glue_ctx serpent_enc = {
+ .num_funcs = 2,
+ .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
- u128_xor(dst, dst, src - 1);
- src -= 1;
- dst -= 1;
- } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS);
-
- if (nbytes < bsize)
- goto done;
- }
-
- /* Handle leftovers */
- for (;;) {
- __serpent_decrypt(ctx, (u8 *)dst, (u8 *)src);
-
- nbytes -= bsize;
- if (nbytes < bsize)
- break;
+ .funcs = { {
+ .num_blocks = SERPENT_PARALLEL_BLOCKS,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_enc_blk_xway) }
+ }, {
+ .num_blocks = 1,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_encrypt) }
+ } }
+};
- u128_xor(dst, dst, src - 1);
- src -= 1;
- dst -= 1;
- }
+static const struct common_glue_ctx serpent_ctr = {
+ .num_funcs = 2,
+ .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
+
+ .funcs = { {
+ .num_blocks = SERPENT_PARALLEL_BLOCKS,
+ .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr_xway) }
+ }, {
+ .num_blocks = 1,
+ .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(serpent_crypt_ctr) }
+ } }
+};
-done:
- u128_xor(dst, dst, (u128 *)walk->iv);
- *(u128 *)walk->iv = last_iv;
+static const struct common_glue_ctx serpent_dec = {
+ .num_funcs = 2,
+ .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
+
+ .funcs = { {
+ .num_blocks = SERPENT_PARALLEL_BLOCKS,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(serpent_dec_blk_xway) }
+ }, {
+ .num_blocks = 1,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(__serpent_decrypt) }
+ } }
+};
- return nbytes;
-}
+static const struct common_glue_ctx serpent_dec_cbc = {
+ .num_funcs = 2,
+ .fpu_blocks_limit = SERPENT_PARALLEL_BLOCKS,
+
+ .funcs = { {
+ .num_blocks = SERPENT_PARALLEL_BLOCKS,
+ .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(serpent_decrypt_cbc_xway) }
+ }, {
+ .num_blocks = 1,
+ .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(__serpent_decrypt) }
+ } }
+};
-static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
- bool fpu_enabled = false;
- struct blkcipher_walk walk;
- int err;
-
- blkcipher_walk_init(&walk, dst, src, nbytes);
- err = blkcipher_walk_virt(desc, &walk);
- desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
-
- while ((nbytes = walk.nbytes)) {
- fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes);
- nbytes = __cbc_decrypt(desc, &walk);
- err = blkcipher_walk_done(desc, &walk, nbytes);
- }
-
- serpent_fpu_end(fpu_enabled);
- return err;
+ return glue_ecb_crypt_128bit(&serpent_enc, desc, dst, src, nbytes);
}
-static inline void u128_to_be128(be128 *dst, const u128 *src)
+static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
{
- dst->a = cpu_to_be64(src->a);
- dst->b = cpu_to_be64(src->b);
+ return glue_ecb_crypt_128bit(&serpent_dec, desc, dst, src, nbytes);
}
-static inline void be128_to_u128(u128 *dst, const be128 *src)
+static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
{
- dst->a = be64_to_cpu(src->a);
- dst->b = be64_to_cpu(src->b);
+ return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(__serpent_encrypt), desc,
+ dst, src, nbytes);
}
-static inline void u128_inc(u128 *i)
+static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
{
- i->b++;
- if (!i->b)
- i->a++;
+ return glue_cbc_decrypt_128bit(&serpent_dec_cbc, desc, dst, src,
+ nbytes);
}
-static void ctr_crypt_final(struct blkcipher_desc *desc,
- struct blkcipher_walk *walk)
+static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
{
- struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- u8 *ctrblk = walk->iv;
- u8 keystream[SERPENT_BLOCK_SIZE];
- u8 *src = walk->src.virt.addr;
- u8 *dst = walk->dst.virt.addr;
- unsigned int nbytes = walk->nbytes;
-
- __serpent_encrypt(ctx, keystream, ctrblk);
- crypto_xor(keystream, src, nbytes);
- memcpy(dst, keystream, nbytes);
-
- crypto_inc(ctrblk, SERPENT_BLOCK_SIZE);
+ return glue_ctr_crypt_128bit(&serpent_ctr, desc, dst, src, nbytes);
}
-static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
- struct blkcipher_walk *walk)
+static inline bool serpent_fpu_begin(bool fpu_enabled, unsigned int nbytes)
{
- struct serpent_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- const unsigned int bsize = SERPENT_BLOCK_SIZE;
- unsigned int nbytes = walk->nbytes;
- u128 *src = (u128 *)walk->src.virt.addr;
- u128 *dst = (u128 *)walk->dst.virt.addr;
- u128 ctrblk;
- be128 ctrblocks[SERPENT_PARALLEL_BLOCKS];
- int i;
-
- be128_to_u128(&ctrblk, (be128 *)walk->iv);
-
- /* Process multi-block batch */
- if (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS) {
- do {
- /* create ctrblks for parallel encrypt */
- for (i = 0; i < SERPENT_PARALLEL_BLOCKS; i++) {
- if (dst != src)
- dst[i] = src[i];
-
- u128_to_be128(&ctrblocks[i], &ctrblk);
- u128_inc(&ctrblk);
- }
-
- serpent_enc_blk_xway_xor(ctx, (u8 *)dst,
- (u8 *)ctrblocks);
-
- src += SERPENT_PARALLEL_BLOCKS;
- dst += SERPENT_PARALLEL_BLOCKS;
- nbytes -= bsize * SERPENT_PARALLEL_BLOCKS;
- } while (nbytes >= bsize * SERPENT_PARALLEL_BLOCKS);
-
- if (nbytes < bsize)
- goto done;
- }
-
- /* Handle leftovers */
- do {
- if (dst != src)
- *dst = *src;
-
- u128_to_be128(&ctrblocks[0], &ctrblk);
- u128_inc(&ctrblk);
-
- __serpent_encrypt(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks);
- u128_xor(dst, dst, (u128 *)ctrblocks);
-
- src += 1;
- dst += 1;
- nbytes -= bsize;
- } while (nbytes >= bsize);
-
-done:
- u128_to_be128((be128 *)walk->iv, &ctrblk);
- return nbytes;
+ return glue_fpu_begin(SERPENT_BLOCK_SIZE, SERPENT_PARALLEL_BLOCKS,
+ NULL, fpu_enabled, nbytes);
}
-static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
- struct scatterlist *src, unsigned int nbytes)
+static inline void serpent_fpu_end(bool fpu_enabled)
{
- bool fpu_enabled = false;
- struct blkcipher_walk walk;
- int err;
-
- blkcipher_walk_init(&walk, dst, src, nbytes);
- err = blkcipher_walk_virt_block(desc, &walk, SERPENT_BLOCK_SIZE);
- desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
-
- while ((nbytes = walk.nbytes) >= SERPENT_BLOCK_SIZE) {
- fpu_enabled = serpent_fpu_begin(fpu_enabled, nbytes);
- nbytes = __ctr_crypt(desc, &walk);
- err = blkcipher_walk_done(desc, &walk, nbytes);
- }
-
- serpent_fpu_end(fpu_enabled);
-
- if (walk.nbytes) {
- ctr_crypt_final(desc, &walk);
- err = blkcipher_walk_done(desc, &walk, 0);
- }
-
- return err;
+ glue_fpu_end(fpu_enabled);
}
struct crypt_priv {
@@ -596,106 +383,6 @@ static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
return ret;
}
-static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
- unsigned int key_len)
-{
- struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
- struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base;
- int err;
-
- crypto_ablkcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
- crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm)
- & CRYPTO_TFM_REQ_MASK);
- err = crypto_ablkcipher_setkey(child, key, key_len);
- crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child)
- & CRYPTO_TFM_RES_MASK);
- return err;
-}
-
-static int __ablk_encrypt(struct ablkcipher_request *req)
-{
- struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
- struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
- struct blkcipher_desc desc;
-
- desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
- desc.info = req->info;
- desc.flags = 0;
-
- return crypto_blkcipher_crt(desc.tfm)->encrypt(
- &desc, req->dst, req->src, req->nbytes);
-}
-
-static int ablk_encrypt(struct ablkcipher_request *req)
-{
- struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
- struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
-
- if (!irq_fpu_usable()) {
- struct ablkcipher_request *cryptd_req =
- ablkcipher_request_ctx(req);
-
- memcpy(cryptd_req, req, sizeof(*req));
- ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
-
- return crypto_ablkcipher_encrypt(cryptd_req);
- } else {
- return __ablk_encrypt(req);
- }
-}
-
-static int ablk_decrypt(struct ablkcipher_request *req)
-{
- struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req);
- struct async_serpent_ctx *ctx = crypto_ablkcipher_ctx(tfm);
-
- if (!irq_fpu_usable()) {
- struct ablkcipher_request *cryptd_req =
- ablkcipher_request_ctx(req);
-
- memcpy(cryptd_req, req, sizeof(*req));
- ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base);
-
- return crypto_ablkcipher_decrypt(cryptd_req);
- } else {
- struct blkcipher_desc desc;
-
- desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm);
- desc.info = req->info;
- desc.flags = 0;
-
- return crypto_blkcipher_crt(desc.tfm)->decrypt(
- &desc, req->dst, req->src, req->nbytes);
- }
-}
-
-static void ablk_exit(struct crypto_tfm *tfm)
-{
- struct async_serpent_ctx *ctx = crypto_tfm_ctx(tfm);
-
- cryptd_free_ablkcipher(ctx->cryptd_tfm);
-}
-
-static int ablk_init(struct crypto_tfm *tfm)
-{
- struct async_serpent_ctx *ctx = crypto_tfm_ctx(tfm);
- struct cryptd_ablkcipher *cryptd_tfm;
- char drv_name[CRYPTO_MAX_ALG_NAME];
-
- snprintf(drv_name, sizeof(drv_name), "__driver-%s",
- crypto_tfm_alg_driver_name(tfm));
-
- cryptd_tfm = cryptd_alloc_ablkcipher(drv_name, 0, 0);
- if (IS_ERR(cryptd_tfm))
- return PTR_ERR(cryptd_tfm);
-
- ctx->cryptd_tfm = cryptd_tfm;
- tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) +
- crypto_ablkcipher_reqsize(&cryptd_tfm->base);
-
- return 0;
-}
-
static struct crypto_alg serpent_algs[10] = { {
.cra_name = "__ecb-serpent-sse2",
.cra_driver_name = "__driver-ecb-serpent-sse2",
@@ -808,7 +495,7 @@ static struct crypto_alg serpent_algs[10] = { {
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
.cra_blocksize = SERPENT_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct async_serpent_ctx),
+ .cra_ctxsize = sizeof(struct async_helper_ctx),
.cra_alignmask = 0,
.cra_type = &crypto_ablkcipher_type,
.cra_module = THIS_MODULE,
@@ -830,7 +517,7 @@ static struct crypto_alg serpent_algs[10] = { {
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
.cra_blocksize = SERPENT_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct async_serpent_ctx),
+ .cra_ctxsize = sizeof(struct async_helper_ctx),
.cra_alignmask = 0,
.cra_type = &crypto_ablkcipher_type,
.cra_module = THIS_MODULE,
@@ -853,7 +540,7 @@ static struct crypto_alg serpent_algs[10] = { {
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
.cra_blocksize = 1,
- .cra_ctxsize = sizeof(struct async_serpent_ctx),
+ .cra_ctxsize = sizeof(struct async_helper_ctx),
.cra_alignmask = 0,
.cra_type = &crypto_ablkcipher_type,
.cra_module = THIS_MODULE,
@@ -877,7 +564,7 @@ static struct crypto_alg serpent_algs[10] = { {
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
.cra_blocksize = SERPENT_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct async_serpent_ctx),
+ .cra_ctxsize = sizeof(struct async_helper_ctx),
.cra_alignmask = 0,
.cra_type = &crypto_ablkcipher_type,
.cra_module = THIS_MODULE,
@@ -902,7 +589,7 @@ static struct crypto_alg serpent_algs[10] = { {
.cra_priority = 400,
.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
.cra_blocksize = SERPENT_BLOCK_SIZE,
- .cra_ctxsize = sizeof(struct async_serpent_ctx),
+ .cra_ctxsize = sizeof(struct async_helper_ctx),
.cra_alignmask = 0,
.cra_type = &crypto_ablkcipher_type,
.cra_module = THIS_MODULE,
diff --git a/arch/x86/crypto/sha1_ssse3_asm.S b/arch/x86/crypto/sha1_ssse3_asm.S
index b2c2f57d70e8..49d6987a73d9 100644
--- a/arch/x86/crypto/sha1_ssse3_asm.S
+++ b/arch/x86/crypto/sha1_ssse3_asm.S
@@ -468,7 +468,7 @@ W_PRECALC_SSSE3
*/
SHA1_VECTOR_ASM sha1_transform_ssse3
-#ifdef SHA1_ENABLE_AVX_SUPPORT
+#ifdef CONFIG_AS_AVX
.macro W_PRECALC_AVX
diff --git a/arch/x86/crypto/sha1_ssse3_glue.c b/arch/x86/crypto/sha1_ssse3_glue.c
index f916499d0abe..4a11a9d72451 100644
--- a/arch/x86/crypto/sha1_ssse3_glue.c
+++ b/arch/x86/crypto/sha1_ssse3_glue.c
@@ -35,7 +35,7 @@
asmlinkage void sha1_transform_ssse3(u32 *digest, const char *data,
unsigned int rounds);
-#ifdef SHA1_ENABLE_AVX_SUPPORT
+#ifdef CONFIG_AS_AVX
asmlinkage void sha1_transform_avx(u32 *digest, const char *data,
unsigned int rounds);
#endif
@@ -184,7 +184,7 @@ static struct shash_alg alg = {
}
};
-#ifdef SHA1_ENABLE_AVX_SUPPORT
+#ifdef CONFIG_AS_AVX
static bool __init avx_usable(void)
{
u64 xcr0;
@@ -209,7 +209,7 @@ static int __init sha1_ssse3_mod_init(void)
if (cpu_has_ssse3)
sha1_transform_asm = sha1_transform_ssse3;
-#ifdef SHA1_ENABLE_AVX_SUPPORT
+#ifdef CONFIG_AS_AVX
/* allow AVX to override SSSE3, it's a little faster */
if (avx_usable())
sha1_transform_asm = sha1_transform_avx;
diff --git a/arch/x86/crypto/twofish-avx-x86_64-asm_64.S b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
new file mode 100644
index 000000000000..35f45574390d
--- /dev/null
+++ b/arch/x86/crypto/twofish-avx-x86_64-asm_64.S
@@ -0,0 +1,300 @@
+/*
+ * Twofish Cipher 8-way parallel algorithm (AVX/x86_64)
+ *
+ * Copyright (C) 2012 Johannes Goetzfried
+ * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA
+ *
+ */
+
+.file "twofish-avx-x86_64-asm_64.S"
+.text
+
+/* structure of crypto context */
+#define s0 0
+#define s1 1024
+#define s2 2048
+#define s3 3072
+#define w 4096
+#define k 4128
+
+/**********************************************************************
+ 8-way AVX twofish
+ **********************************************************************/
+#define CTX %rdi
+
+#define RA1 %xmm0
+#define RB1 %xmm1
+#define RC1 %xmm2
+#define RD1 %xmm3
+
+#define RA2 %xmm4
+#define RB2 %xmm5
+#define RC2 %xmm6
+#define RD2 %xmm7
+
+#define RX %xmm8
+#define RY %xmm9
+
+#define RK1 %xmm10
+#define RK2 %xmm11
+
+#define RID1 %rax
+#define RID1b %al
+#define RID2 %rbx
+#define RID2b %bl
+
+#define RGI1 %rdx
+#define RGI1bl %dl
+#define RGI1bh %dh
+#define RGI2 %rcx
+#define RGI2bl %cl
+#define RGI2bh %ch
+
+#define RGS1 %r8
+#define RGS1d %r8d
+#define RGS2 %r9
+#define RGS2d %r9d
+#define RGS3 %r10
+#define RGS3d %r10d
+
+
+#define lookup_32bit(t0, t1, t2, t3, src, dst) \
+ movb src ## bl, RID1b; \
+ movb src ## bh, RID2b; \
+ movl t0(CTX, RID1, 4), dst ## d; \
+ xorl t1(CTX, RID2, 4), dst ## d; \
+ shrq $16, src; \
+ movb src ## bl, RID1b; \
+ movb src ## bh, RID2b; \
+ xorl t2(CTX, RID1, 4), dst ## d; \
+ xorl t3(CTX, RID2, 4), dst ## d;
+
+#define G(a, x, t0, t1, t2, t3) \
+ vmovq a, RGI1; \
+ vpsrldq $8, a, x; \
+ vmovq x, RGI2; \
+ \
+ lookup_32bit(t0, t1, t2, t3, RGI1, RGS1); \
+ shrq $16, RGI1; \
+ lookup_32bit(t0, t1, t2, t3, RGI1, RGS2); \
+ shlq $32, RGS2; \
+ orq RGS1, RGS2; \
+ \
+ lookup_32bit(t0, t1, t2, t3, RGI2, RGS1); \
+ shrq $16, RGI2; \
+ lookup_32bit(t0, t1, t2, t3, RGI2, RGS3); \
+ shlq $32, RGS3; \
+ orq RGS1, RGS3; \
+ \
+ vmovq RGS2, x; \
+ vpinsrq $1, RGS3, x, x;
+
+#define encround(a, b, c, d, x, y) \
+ G(a, x, s0, s1, s2, s3); \
+ G(b, y, s1, s2, s3, s0); \
+ vpaddd x, y, x; \
+ vpaddd y, x, y; \
+ vpaddd x, RK1, x; \
+ vpaddd y, RK2, y; \
+ vpxor x, c, c; \
+ vpsrld $1, c, x; \
+ vpslld $(32 - 1), c, c; \
+ vpor c, x, c; \
+ vpslld $1, d, x; \
+ vpsrld $(32 - 1), d, d; \
+ vpor d, x, d; \
+ vpxor d, y, d;
+
+#define decround(a, b, c, d, x, y) \
+ G(a, x, s0, s1, s2, s3); \
+ G(b, y, s1, s2, s3, s0); \
+ vpaddd x, y, x; \
+ vpaddd y, x, y; \
+ vpaddd y, RK2, y; \
+ vpxor d, y, d; \
+ vpsrld $1, d, y; \
+ vpslld $(32 - 1), d, d; \
+ vpor d, y, d; \
+ vpslld $1, c, y; \
+ vpsrld $(32 - 1), c, c; \
+ vpor c, y, c; \
+ vpaddd x, RK1, x; \
+ vpxor x, c, c;
+
+#define encrypt_round(n, a, b, c, d) \
+ vbroadcastss (k+4*(2*(n)))(CTX), RK1; \
+ vbroadcastss (k+4*(2*(n)+1))(CTX), RK2; \
+ encround(a ## 1, b ## 1, c ## 1, d ## 1, RX, RY); \
+ encround(a ## 2, b ## 2, c ## 2, d ## 2, RX, RY);
+
+#define decrypt_round(n, a, b, c, d) \
+ vbroadcastss (k+4*(2*(n)))(CTX), RK1; \
+ vbroadcastss (k+4*(2*(n)+1))(CTX), RK2; \
+ decround(a ## 1, b ## 1, c ## 1, d ## 1, RX, RY); \
+ decround(a ## 2, b ## 2, c ## 2, d ## 2, RX, RY);
+
+#define encrypt_cycle(n) \
+ encrypt_round((2*n), RA, RB, RC, RD); \
+ encrypt_round(((2*n) + 1), RC, RD, RA, RB);
+
+#define decrypt_cycle(n) \
+ decrypt_round(((2*n) + 1), RC, RD, RA, RB); \
+ decrypt_round((2*n), RA, RB, RC, RD);
+
+
+#define transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
+ vpunpckldq x1, x0, t0; \
+ vpunpckhdq x1, x0, t2; \
+ vpunpckldq x3, x2, t1; \
+ vpunpckhdq x3, x2, x3; \
+ \
+ vpunpcklqdq t1, t0, x0; \
+ vpunpckhqdq t1, t0, x1; \
+ vpunpcklqdq x3, t2, x2; \
+ vpunpckhqdq x3, t2, x3;
+
+#define inpack_blocks(in, x0, x1, x2, x3, wkey, t0, t1, t2) \
+ vpxor (0*4*4)(in), wkey, x0; \
+ vpxor (1*4*4)(in), wkey, x1; \
+ vpxor (2*4*4)(in), wkey, x2; \
+ vpxor (3*4*4)(in), wkey, x3; \
+ \
+ transpose_4x4(x0, x1, x2, x3, t0, t1, t2)
+
+#define outunpack_blocks(out, x0, x1, x2, x3, wkey, t0, t1, t2) \
+ transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
+ \
+ vpxor x0, wkey, x0; \
+ vmovdqu x0, (0*4*4)(out); \
+ vpxor x1, wkey, x1; \
+ vmovdqu x1, (1*4*4)(out); \
+ vpxor x2, wkey, x2; \
+ vmovdqu x2, (2*4*4)(out); \
+ vpxor x3, wkey, x3; \
+ vmovdqu x3, (3*4*4)(out);
+
+#define outunpack_xor_blocks(out, x0, x1, x2, x3, wkey, t0, t1, t2) \
+ transpose_4x4(x0, x1, x2, x3, t0, t1, t2) \
+ \
+ vpxor x0, wkey, x0; \
+ vpxor (0*4*4)(out), x0, x0; \
+ vmovdqu x0, (0*4*4)(out); \
+ vpxor x1, wkey, x1; \
+ vpxor (1*4*4)(out), x1, x1; \
+ vmovdqu x1, (1*4*4)(out); \
+ vpxor x2, wkey, x2; \
+ vpxor (2*4*4)(out), x2, x2; \
+ vmovdqu x2, (2*4*4)(out); \
+ vpxor x3, wkey, x3; \
+ vpxor (3*4*4)(out), x3, x3; \
+ vmovdqu x3, (3*4*4)(out);
+
+.align 8
+.global __twofish_enc_blk_8way
+.type __twofish_enc_blk_8way,@function;
+
+__twofish_enc_blk_8way:
+ /* input:
+ * %rdi: ctx, CTX
+ * %rsi: dst
+ * %rdx: src
+ * %rcx: bool, if true: xor output
+ */
+
+ pushq %rbx;
+ pushq %rcx;
+
+ vmovdqu w(CTX), RK1;
+
+ leaq (4*4*4)(%rdx), %rax;
+ inpack_blocks(%rdx, RA1, RB1, RC1, RD1, RK1, RX, RY, RK2);
+ inpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX, RY, RK2);
+
+ xorq RID1, RID1;
+ xorq RID2, RID2;
+
+ encrypt_cycle(0);
+ encrypt_cycle(1);
+ encrypt_cycle(2);
+ encrypt_cycle(3);
+ encrypt_cycle(4);
+ encrypt_cycle(5);
+ encrypt_cycle(6);
+ encrypt_cycle(7);
+
+ vmovdqu (w+4*4)(CTX), RK1;
+
+ popq %rcx;
+ popq %rbx;
+
+ leaq (4*4*4)(%rsi), %rax;
+
+ testb %cl, %cl;
+ jnz __enc_xor8;
+
+ outunpack_blocks(%rsi, RC1, RD1, RA1, RB1, RK1, RX, RY, RK2);
+ outunpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX, RY, RK2);
+
+ ret;
+
+__enc_xor8:
+ outunpack_xor_blocks(%rsi, RC1, RD1, RA1, RB1, RK1, RX, RY, RK2);
+ outunpack_xor_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX, RY, RK2);
+
+ ret;
+
+.align 8
+.global twofish_dec_blk_8way
+.type twofish_dec_blk_8way,@function;
+
+twofish_dec_blk_8way:
+ /* input:
+ * %rdi: ctx, CTX
+ * %rsi: dst
+ * %rdx: src
+ */
+
+ pushq %rbx;
+
+ vmovdqu (w+4*4)(CTX), RK1;
+
+ leaq (4*4*4)(%rdx), %rax;
+ inpack_blocks(%rdx, RC1, RD1, RA1, RB1, RK1, RX, RY, RK2);
+ inpack_blocks(%rax, RC2, RD2, RA2, RB2, RK1, RX, RY, RK2);
+
+ xorq RID1, RID1;
+ xorq RID2, RID2;
+
+ decrypt_cycle(7);
+ decrypt_cycle(6);
+ decrypt_cycle(5);
+ decrypt_cycle(4);
+ decrypt_cycle(3);
+ decrypt_cycle(2);
+ decrypt_cycle(1);
+ decrypt_cycle(0);
+
+ vmovdqu (w)(CTX), RK1;
+
+ popq %rbx;
+
+ leaq (4*4*4)(%rsi), %rax;
+ outunpack_blocks(%rsi, RA1, RB1, RC1, RD1, RK1, RX, RY, RK2);
+ outunpack_blocks(%rax, RA2, RB2, RC2, RD2, RK1, RX, RY, RK2);
+
+ ret;
diff --git a/arch/x86/crypto/twofish_avx_glue.c b/arch/x86/crypto/twofish_avx_glue.c
new file mode 100644
index 000000000000..782b67ddaf6a
--- /dev/null
+++ b/arch/x86/crypto/twofish_avx_glue.c
@@ -0,0 +1,624 @@
+/*
+ * Glue Code for AVX assembler version of Twofish Cipher
+ *
+ * Copyright (C) 2012 Johannes Goetzfried
+ * <Johannes.Goetzfried@informatik.stud.uni-erlangen.de>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/hardirq.h>
+#include <linux/types.h>
+#include <linux/crypto.h>
+#include <linux/err.h>
+#include <crypto/algapi.h>
+#include <crypto/twofish.h>
+#include <crypto/cryptd.h>
+#include <crypto/b128ops.h>
+#include <crypto/ctr.h>
+#include <crypto/lrw.h>
+#include <crypto/xts.h>
+#include <asm/i387.h>
+#include <asm/xcr.h>
+#include <asm/xsave.h>
+#include <asm/crypto/twofish.h>
+#include <asm/crypto/ablk_helper.h>
+#include <asm/crypto/glue_helper.h>
+#include <crypto/scatterwalk.h>
+#include <linux/workqueue.h>
+#include <linux/spinlock.h>
+
+#define TWOFISH_PARALLEL_BLOCKS 8
+
+static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
+ const u8 *src)
+{
+ __twofish_enc_blk_3way(ctx, dst, src, false);
+}
+
+/* 8-way parallel cipher functions */
+asmlinkage void __twofish_enc_blk_8way(struct twofish_ctx *ctx, u8 *dst,
+ const u8 *src, bool xor);
+asmlinkage void twofish_dec_blk_8way(struct twofish_ctx *ctx, u8 *dst,
+ const u8 *src);
+
+static inline void twofish_enc_blk_xway(struct twofish_ctx *ctx, u8 *dst,
+ const u8 *src)
+{
+ __twofish_enc_blk_8way(ctx, dst, src, false);
+}
+
+static inline void twofish_enc_blk_xway_xor(struct twofish_ctx *ctx, u8 *dst,
+ const u8 *src)
+{
+ __twofish_enc_blk_8way(ctx, dst, src, true);
+}
+
+static inline void twofish_dec_blk_xway(struct twofish_ctx *ctx, u8 *dst,
+ const u8 *src)
+{
+ twofish_dec_blk_8way(ctx, dst, src);
+}
+
+static void twofish_dec_blk_cbc_xway(void *ctx, u128 *dst, const u128 *src)
+{
+ u128 ivs[TWOFISH_PARALLEL_BLOCKS - 1];
+ unsigned int j;
+
+ for (j = 0; j < TWOFISH_PARALLEL_BLOCKS - 1; j++)
+ ivs[j] = src[j];
+
+ twofish_dec_blk_xway(ctx, (u8 *)dst, (u8 *)src);
+
+ for (j = 0; j < TWOFISH_PARALLEL_BLOCKS - 1; j++)
+ u128_xor(dst + (j + 1), dst + (j + 1), ivs + j);
+}
+
+static void twofish_enc_blk_ctr_xway(void *ctx, u128 *dst, const u128 *src,
+ u128 *iv)
+{
+ be128 ctrblks[TWOFISH_PARALLEL_BLOCKS];
+ unsigned int i;
+
+ for (i = 0; i < TWOFISH_PARALLEL_BLOCKS; i++) {
+ if (dst != src)
+ dst[i] = src[i];
+
+ u128_to_be128(&ctrblks[i], iv);
+ u128_inc(iv);
+ }
+
+ twofish_enc_blk_xway_xor(ctx, (u8 *)dst, (u8 *)ctrblks);
+}
+
+static const struct common_glue_ctx twofish_enc = {
+ .num_funcs = 3,
+ .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
+
+ .funcs = { {
+ .num_blocks = TWOFISH_PARALLEL_BLOCKS,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_xway) }
+ }, {
+ .num_blocks = 3,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
+ }, {
+ .num_blocks = 1,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
+ } }
+};
+
+static const struct common_glue_ctx twofish_ctr = {
+ .num_funcs = 3,
+ .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
+
+ .funcs = { {
+ .num_blocks = TWOFISH_PARALLEL_BLOCKS,
+ .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_xway) }
+ }, {
+ .num_blocks = 3,
+ .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr_3way) }
+ }, {
+ .num_blocks = 1,
+ .fn_u = { .ctr = GLUE_CTR_FUNC_CAST(twofish_enc_blk_ctr) }
+ } }
+};
+
+static const struct common_glue_ctx twofish_dec = {
+ .num_funcs = 3,
+ .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
+
+ .funcs = { {
+ .num_blocks = TWOFISH_PARALLEL_BLOCKS,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_xway) }
+ }, {
+ .num_blocks = 3,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
+ }, {
+ .num_blocks = 1,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
+ } }
+};
+
+static const struct common_glue_ctx twofish_dec_cbc = {
+ .num_funcs = 3,
+ .fpu_blocks_limit = TWOFISH_PARALLEL_BLOCKS,
+
+ .funcs = { {
+ .num_blocks = TWOFISH_PARALLEL_BLOCKS,
+ .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_xway) }
+ }, {
+ .num_blocks = 3,
+ .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
+ }, {
+ .num_blocks = 1,
+ .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
+ } }
+};
+
+static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes);
+}
+
+static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes);
+}
+
+static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc,
+ dst, src, nbytes);
+}
+
+static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src,
+ nbytes);
+}
+
+static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes);
+}
+
+static inline bool twofish_fpu_begin(bool fpu_enabled, unsigned int nbytes)
+{
+ return glue_fpu_begin(TF_BLOCK_SIZE, TWOFISH_PARALLEL_BLOCKS, NULL,
+ fpu_enabled, nbytes);
+}
+
+static inline void twofish_fpu_end(bool fpu_enabled)
+{
+ glue_fpu_end(fpu_enabled);
+}
+
+struct crypt_priv {
+ struct twofish_ctx *ctx;
+ bool fpu_enabled;
+};
+
+static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
+{
+ const unsigned int bsize = TF_BLOCK_SIZE;
+ struct crypt_priv *ctx = priv;
+ int i;
+
+ ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
+
+ if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
+ twofish_enc_blk_xway(ctx->ctx, srcdst, srcdst);
+ return;
+ }
+
+ for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
+ twofish_enc_blk_3way(ctx->ctx, srcdst, srcdst);
+
+ nbytes %= bsize * 3;
+
+ for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
+ twofish_enc_blk(ctx->ctx, srcdst, srcdst);
+}
+
+static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
+{
+ const unsigned int bsize = TF_BLOCK_SIZE;
+ struct crypt_priv *ctx = priv;
+ int i;
+
+ ctx->fpu_enabled = twofish_fpu_begin(ctx->fpu_enabled, nbytes);
+
+ if (nbytes == bsize * TWOFISH_PARALLEL_BLOCKS) {
+ twofish_dec_blk_xway(ctx->ctx, srcdst, srcdst);
+ return;
+ }
+
+ for (i = 0; i < nbytes / (bsize * 3); i++, srcdst += bsize * 3)
+ twofish_dec_blk_3way(ctx->ctx, srcdst, srcdst);
+
+ nbytes %= bsize * 3;
+
+ for (i = 0; i < nbytes / bsize; i++, srcdst += bsize)
+ twofish_dec_blk(ctx->ctx, srcdst, srcdst);
+}
+
+static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+ be128 buf[TWOFISH_PARALLEL_BLOCKS];
+ struct crypt_priv crypt_ctx = {
+ .ctx = &ctx->twofish_ctx,
+ .fpu_enabled = false,
+ };
+ struct lrw_crypt_req req = {
+ .tbuf = buf,
+ .tbuflen = sizeof(buf),
+
+ .table_ctx = &ctx->lrw_table,
+ .crypt_ctx = &crypt_ctx,
+ .crypt_fn = encrypt_callback,
+ };
+ int ret;
+
+ desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+ ret = lrw_crypt(desc, dst, src, nbytes, &req);
+ twofish_fpu_end(crypt_ctx.fpu_enabled);
+
+ return ret;
+}
+
+static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ struct twofish_lrw_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+ be128 buf[TWOFISH_PARALLEL_BLOCKS];
+ struct crypt_priv crypt_ctx = {
+ .ctx = &ctx->twofish_ctx,
+ .fpu_enabled = false,
+ };
+ struct lrw_crypt_req req = {
+ .tbuf = buf,
+ .tbuflen = sizeof(buf),
+
+ .table_ctx = &ctx->lrw_table,
+ .crypt_ctx = &crypt_ctx,
+ .crypt_fn = decrypt_callback,
+ };
+ int ret;
+
+ desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+ ret = lrw_crypt(desc, dst, src, nbytes, &req);
+ twofish_fpu_end(crypt_ctx.fpu_enabled);
+
+ return ret;
+}
+
+static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+ be128 buf[TWOFISH_PARALLEL_BLOCKS];
+ struct crypt_priv crypt_ctx = {
+ .ctx = &ctx->crypt_ctx,
+ .fpu_enabled = false,
+ };
+ struct xts_crypt_req req = {
+ .tbuf = buf,
+ .tbuflen = sizeof(buf),
+
+ .tweak_ctx = &ctx->tweak_ctx,
+ .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk),
+ .crypt_ctx = &crypt_ctx,
+ .crypt_fn = encrypt_callback,
+ };
+ int ret;
+
+ desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+ ret = xts_crypt(desc, dst, src, nbytes, &req);
+ twofish_fpu_end(crypt_ctx.fpu_enabled);
+
+ return ret;
+}
+
+static int xts_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
+{
+ struct twofish_xts_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
+ be128 buf[TWOFISH_PARALLEL_BLOCKS];
+ struct crypt_priv crypt_ctx = {
+ .ctx = &ctx->crypt_ctx,
+ .fpu_enabled = false,
+ };
+ struct xts_crypt_req req = {
+ .tbuf = buf,
+ .tbuflen = sizeof(buf),
+
+ .tweak_ctx = &ctx->tweak_ctx,
+ .tweak_fn = XTS_TWEAK_CAST(twofish_enc_blk),
+ .crypt_ctx = &crypt_ctx,
+ .crypt_fn = decrypt_callback,
+ };
+ int ret;
+
+ desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+ ret = xts_crypt(desc, dst, src, nbytes, &req);
+ twofish_fpu_end(crypt_ctx.fpu_enabled);
+
+ return ret;
+}
+
+static struct crypto_alg twofish_algs[10] = { {
+ .cra_name = "__ecb-twofish-avx",
+ .cra_driver_name = "__driver-ecb-twofish-avx",
+ .cra_priority = 0,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
+ .cra_blocksize = TF_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct twofish_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_blkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(twofish_algs[0].cra_list),
+ .cra_u = {
+ .blkcipher = {
+ .min_keysize = TF_MIN_KEY_SIZE,
+ .max_keysize = TF_MAX_KEY_SIZE,
+ .setkey = twofish_setkey,
+ .encrypt = ecb_encrypt,
+ .decrypt = ecb_decrypt,
+ },
+ },
+}, {
+ .cra_name = "__cbc-twofish-avx",
+ .cra_driver_name = "__driver-cbc-twofish-avx",
+ .cra_priority = 0,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
+ .cra_blocksize = TF_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct twofish_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_blkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(twofish_algs[1].cra_list),
+ .cra_u = {
+ .blkcipher = {
+ .min_keysize = TF_MIN_KEY_SIZE,
+ .max_keysize = TF_MAX_KEY_SIZE,
+ .setkey = twofish_setkey,
+ .encrypt = cbc_encrypt,
+ .decrypt = cbc_decrypt,
+ },
+ },
+}, {
+ .cra_name = "__ctr-twofish-avx",
+ .cra_driver_name = "__driver-ctr-twofish-avx",
+ .cra_priority = 0,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct twofish_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_blkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(twofish_algs[2].cra_list),
+ .cra_u = {
+ .blkcipher = {
+ .min_keysize = TF_MIN_KEY_SIZE,
+ .max_keysize = TF_MAX_KEY_SIZE,
+ .ivsize = TF_BLOCK_SIZE,
+ .setkey = twofish_setkey,
+ .encrypt = ctr_crypt,
+ .decrypt = ctr_crypt,
+ },
+ },
+}, {
+ .cra_name = "__lrw-twofish-avx",
+ .cra_driver_name = "__driver-lrw-twofish-avx",
+ .cra_priority = 0,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
+ .cra_blocksize = TF_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct twofish_lrw_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_blkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(twofish_algs[3].cra_list),
+ .cra_exit = lrw_twofish_exit_tfm,
+ .cra_u = {
+ .blkcipher = {
+ .min_keysize = TF_MIN_KEY_SIZE +
+ TF_BLOCK_SIZE,
+ .max_keysize = TF_MAX_KEY_SIZE +
+ TF_BLOCK_SIZE,
+ .ivsize = TF_BLOCK_SIZE,
+ .setkey = lrw_twofish_setkey,
+ .encrypt = lrw_encrypt,
+ .decrypt = lrw_decrypt,
+ },
+ },
+}, {
+ .cra_name = "__xts-twofish-avx",
+ .cra_driver_name = "__driver-xts-twofish-avx",
+ .cra_priority = 0,
+ .cra_flags = CRYPTO_ALG_TYPE_BLKCIPHER,
+ .cra_blocksize = TF_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct twofish_xts_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_blkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(twofish_algs[4].cra_list),
+ .cra_u = {
+ .blkcipher = {
+ .min_keysize = TF_MIN_KEY_SIZE * 2,
+ .max_keysize = TF_MAX_KEY_SIZE * 2,
+ .ivsize = TF_BLOCK_SIZE,
+ .setkey = xts_twofish_setkey,
+ .encrypt = xts_encrypt,
+ .decrypt = xts_decrypt,
+ },
+ },
+}, {
+ .cra_name = "ecb(twofish)",
+ .cra_driver_name = "ecb-twofish-avx",
+ .cra_priority = 400,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+ .cra_blocksize = TF_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct async_helper_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(twofish_algs[5].cra_list),
+ .cra_init = ablk_init,
+ .cra_exit = ablk_exit,
+ .cra_u = {
+ .ablkcipher = {
+ .min_keysize = TF_MIN_KEY_SIZE,
+ .max_keysize = TF_MAX_KEY_SIZE,
+ .setkey = ablk_set_key,
+ .encrypt = ablk_encrypt,
+ .decrypt = ablk_decrypt,
+ },
+ },
+}, {
+ .cra_name = "cbc(twofish)",
+ .cra_driver_name = "cbc-twofish-avx",
+ .cra_priority = 400,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+ .cra_blocksize = TF_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct async_helper_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(twofish_algs[6].cra_list),
+ .cra_init = ablk_init,
+ .cra_exit = ablk_exit,
+ .cra_u = {
+ .ablkcipher = {
+ .min_keysize = TF_MIN_KEY_SIZE,
+ .max_keysize = TF_MAX_KEY_SIZE,
+ .ivsize = TF_BLOCK_SIZE,
+ .setkey = ablk_set_key,
+ .encrypt = __ablk_encrypt,
+ .decrypt = ablk_decrypt,
+ },
+ },
+}, {
+ .cra_name = "ctr(twofish)",
+ .cra_driver_name = "ctr-twofish-avx",
+ .cra_priority = 400,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+ .cra_blocksize = 1,
+ .cra_ctxsize = sizeof(struct async_helper_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(twofish_algs[7].cra_list),
+ .cra_init = ablk_init,
+ .cra_exit = ablk_exit,
+ .cra_u = {
+ .ablkcipher = {
+ .min_keysize = TF_MIN_KEY_SIZE,
+ .max_keysize = TF_MAX_KEY_SIZE,
+ .ivsize = TF_BLOCK_SIZE,
+ .setkey = ablk_set_key,
+ .encrypt = ablk_encrypt,
+ .decrypt = ablk_encrypt,
+ .geniv = "chainiv",
+ },
+ },
+}, {
+ .cra_name = "lrw(twofish)",
+ .cra_driver_name = "lrw-twofish-avx",
+ .cra_priority = 400,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+ .cra_blocksize = TF_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct async_helper_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(twofish_algs[8].cra_list),
+ .cra_init = ablk_init,
+ .cra_exit = ablk_exit,
+ .cra_u = {
+ .ablkcipher = {
+ .min_keysize = TF_MIN_KEY_SIZE +
+ TF_BLOCK_SIZE,
+ .max_keysize = TF_MAX_KEY_SIZE +
+ TF_BLOCK_SIZE,
+ .ivsize = TF_BLOCK_SIZE,
+ .setkey = ablk_set_key,
+ .encrypt = ablk_encrypt,
+ .decrypt = ablk_decrypt,
+ },
+ },
+}, {
+ .cra_name = "xts(twofish)",
+ .cra_driver_name = "xts-twofish-avx",
+ .cra_priority = 400,
+ .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC,
+ .cra_blocksize = TF_BLOCK_SIZE,
+ .cra_ctxsize = sizeof(struct async_helper_ctx),
+ .cra_alignmask = 0,
+ .cra_type = &crypto_ablkcipher_type,
+ .cra_module = THIS_MODULE,
+ .cra_list = LIST_HEAD_INIT(twofish_algs[9].cra_list),
+ .cra_init = ablk_init,
+ .cra_exit = ablk_exit,
+ .cra_u = {
+ .ablkcipher = {
+ .min_keysize = TF_MIN_KEY_SIZE * 2,
+ .max_keysize = TF_MAX_KEY_SIZE * 2,
+ .ivsize = TF_BLOCK_SIZE,
+ .setkey = ablk_set_key,
+ .encrypt = ablk_encrypt,
+ .decrypt = ablk_decrypt,
+ },
+ },
+} };
+
+static int __init twofish_init(void)
+{
+ u64 xcr0;
+
+ if (!cpu_has_avx || !cpu_has_osxsave) {
+ printk(KERN_INFO "AVX instructions are not detected.\n");
+ return -ENODEV;
+ }
+
+ xcr0 = xgetbv(XCR_XFEATURE_ENABLED_MASK);
+ if ((xcr0 & (XSTATE_SSE | XSTATE_YMM)) != (XSTATE_SSE | XSTATE_YMM)) {
+ printk(KERN_INFO "AVX detected but unusable.\n");
+ return -ENODEV;
+ }
+
+ return crypto_register_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
+}
+
+static void __exit twofish_exit(void)
+{
+ crypto_unregister_algs(twofish_algs, ARRAY_SIZE(twofish_algs));
+}
+
+module_init(twofish_init);
+module_exit(twofish_exit);
+
+MODULE_DESCRIPTION("Twofish Cipher Algorithm, AVX optimized");
+MODULE_LICENSE("GPL");
+MODULE_ALIAS("twofish");
diff --git a/arch/x86/crypto/twofish_glue_3way.c b/arch/x86/crypto/twofish_glue_3way.c
index 922ab24cce31..15f9347316c8 100644
--- a/arch/x86/crypto/twofish_glue_3way.c
+++ b/arch/x86/crypto/twofish_glue_3way.c
@@ -3,11 +3,6 @@
*
* Copyright (c) 2011 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
*
- * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
- * Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
- * CTR part based on code (crypto/ctr.c) by:
- * (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
- *
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
@@ -33,20 +28,13 @@
#include <crypto/algapi.h>
#include <crypto/twofish.h>
#include <crypto/b128ops.h>
+#include <asm/crypto/twofish.h>
+#include <asm/crypto/glue_helper.h>
#include <crypto/lrw.h>
#include <crypto/xts.h>
-/* regular block cipher functions from twofish_x86_64 module */
-asmlinkage void twofish_enc_blk(struct twofish_ctx *ctx, u8 *dst,
- const u8 *src);
-asmlinkage void twofish_dec_blk(struct twofish_ctx *ctx, u8 *dst,
- const u8 *src);
-
-/* 3-way parallel cipher functions */
-asmlinkage void __twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
- const u8 *src, bool xor);
-asmlinkage void twofish_dec_blk_3way(struct twofish_ctx *ctx, u8 *dst,
- const u8 *src);
+EXPORT_SYMBOL_GPL(__twofish_enc_blk_3way);
+EXPORT_SYMBOL_GPL(twofish_dec_blk_3way);
static inline void twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
const u8 *src)
@@ -60,311 +48,139 @@ static inline void twofish_enc_blk_xor_3way(struct twofish_ctx *ctx, u8 *dst,
__twofish_enc_blk_3way(ctx, dst, src, true);
}
-static int ecb_crypt(struct blkcipher_desc *desc, struct blkcipher_walk *walk,
- void (*fn)(struct twofish_ctx *, u8 *, const u8 *),
- void (*fn_3way)(struct twofish_ctx *, u8 *, const u8 *))
-{
- struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- unsigned int bsize = TF_BLOCK_SIZE;
- unsigned int nbytes;
- int err;
-
- err = blkcipher_walk_virt(desc, walk);
-
- while ((nbytes = walk->nbytes)) {
- u8 *wsrc = walk->src.virt.addr;
- u8 *wdst = walk->dst.virt.addr;
-
- /* Process three block batch */
- if (nbytes >= bsize * 3) {
- do {
- fn_3way(ctx, wdst, wsrc);
-
- wsrc += bsize * 3;
- wdst += bsize * 3;
- nbytes -= bsize * 3;
- } while (nbytes >= bsize * 3);
-
- if (nbytes < bsize)
- goto done;
- }
-
- /* Handle leftovers */
- do {
- fn(ctx, wdst, wsrc);
-
- wsrc += bsize;
- wdst += bsize;
- nbytes -= bsize;
- } while (nbytes >= bsize);
-
-done:
- err = blkcipher_walk_done(desc, walk, nbytes);
- }
-
- return err;
-}
-
-static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
- struct scatterlist *src, unsigned int nbytes)
+void twofish_dec_blk_cbc_3way(void *ctx, u128 *dst, const u128 *src)
{
- struct blkcipher_walk walk;
+ u128 ivs[2];
- blkcipher_walk_init(&walk, dst, src, nbytes);
- return ecb_crypt(desc, &walk, twofish_enc_blk, twofish_enc_blk_3way);
-}
+ ivs[0] = src[0];
+ ivs[1] = src[1];
-static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
- struct scatterlist *src, unsigned int nbytes)
-{
- struct blkcipher_walk walk;
+ twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src);
- blkcipher_walk_init(&walk, dst, src, nbytes);
- return ecb_crypt(desc, &walk, twofish_dec_blk, twofish_dec_blk_3way);
+ u128_xor(&dst[1], &dst[1], &ivs[0]);
+ u128_xor(&dst[2], &dst[2], &ivs[1]);
}
+EXPORT_SYMBOL_GPL(twofish_dec_blk_cbc_3way);
-static unsigned int __cbc_encrypt(struct blkcipher_desc *desc,
- struct blkcipher_walk *walk)
-{
- struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- unsigned int bsize = TF_BLOCK_SIZE;
- unsigned int nbytes = walk->nbytes;
- u128 *src = (u128 *)walk->src.virt.addr;
- u128 *dst = (u128 *)walk->dst.virt.addr;
- u128 *iv = (u128 *)walk->iv;
-
- do {
- u128_xor(dst, src, iv);
- twofish_enc_blk(ctx, (u8 *)dst, (u8 *)dst);
- iv = dst;
-
- src += 1;
- dst += 1;
- nbytes -= bsize;
- } while (nbytes >= bsize);
-
- u128_xor((u128 *)walk->iv, (u128 *)walk->iv, iv);
- return nbytes;
-}
-
-static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
- struct scatterlist *src, unsigned int nbytes)
+void twofish_enc_blk_ctr(void *ctx, u128 *dst, const u128 *src, u128 *iv)
{
- struct blkcipher_walk walk;
- int err;
+ be128 ctrblk;
- blkcipher_walk_init(&walk, dst, src, nbytes);
- err = blkcipher_walk_virt(desc, &walk);
+ if (dst != src)
+ *dst = *src;
- while ((nbytes = walk.nbytes)) {
- nbytes = __cbc_encrypt(desc, &walk);
- err = blkcipher_walk_done(desc, &walk, nbytes);
- }
+ u128_to_be128(&ctrblk, iv);
+ u128_inc(iv);
- return err;
+ twofish_enc_blk(ctx, (u8 *)&ctrblk, (u8 *)&ctrblk);
+ u128_xor(dst, dst, (u128 *)&ctrblk);
}
+EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr);
-static unsigned int __cbc_decrypt(struct blkcipher_desc *desc,
- struct blkcipher_walk *walk)
+void twofish_enc_blk_ctr_3way(void *ctx, u128 *dst, const u128 *src,
+ u128 *iv)
{
- struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- unsigned int bsize = TF_BLOCK_SIZE;
- unsigned int nbytes = walk->nbytes;
- u128 *src = (u128 *)walk->src.virt.addr;
- u128 *dst = (u128 *)walk->dst.virt.addr;
- u128 ivs[3 - 1];
- u128 last_iv;
-
- /* Start of the last block. */
- src += nbytes / bsize - 1;
- dst += nbytes / bsize - 1;
-
- last_iv = *src;
-
- /* Process three block batch */
- if (nbytes >= bsize * 3) {
- do {
- nbytes -= bsize * (3 - 1);
- src -= 3 - 1;
- dst -= 3 - 1;
-
- ivs[0] = src[0];
- ivs[1] = src[1];
-
- twofish_dec_blk_3way(ctx, (u8 *)dst, (u8 *)src);
-
- u128_xor(dst + 1, dst + 1, ivs + 0);
- u128_xor(dst + 2, dst + 2, ivs + 1);
-
- nbytes -= bsize;
- if (nbytes < bsize)
- goto done;
-
- u128_xor(dst, dst, src - 1);
- src -= 1;
- dst -= 1;
- } while (nbytes >= bsize * 3);
-
- if (nbytes < bsize)
- goto done;
- }
-
- /* Handle leftovers */
- for (;;) {
- twofish_dec_blk(ctx, (u8 *)dst, (u8 *)src);
-
- nbytes -= bsize;
- if (nbytes < bsize)
- break;
+ be128 ctrblks[3];
- u128_xor(dst, dst, src - 1);
- src -= 1;
- dst -= 1;
+ if (dst != src) {
+ dst[0] = src[0];
+ dst[1] = src[1];
+ dst[2] = src[2];
}
-done:
- u128_xor(dst, dst, (u128 *)walk->iv);
- *(u128 *)walk->iv = last_iv;
+ u128_to_be128(&ctrblks[0], iv);
+ u128_inc(iv);
+ u128_to_be128(&ctrblks[1], iv);
+ u128_inc(iv);
+ u128_to_be128(&ctrblks[2], iv);
+ u128_inc(iv);
- return nbytes;
+ twofish_enc_blk_xor_3way(ctx, (u8 *)dst, (u8 *)ctrblks);
}
+EXPORT_SYMBOL_GPL(twofish_enc_blk_ctr_3way);
+
+static const struct common_glue_ctx twofish_enc = {
+ .num_funcs = 2,
+ .fpu_blocks_limit = -1,
+
+ .funcs = { {
+ .num_blocks = 3,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_3way) }
+ }, {
+ .num_blocks = 1,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk) }
+ } }
+};
-static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
- struct scatterlist *src, unsigned int nbytes)
-{
- struct blkcipher_walk walk;
- int err;
-
- blkcipher_walk_init(&walk, dst, src, nbytes);
- err = blkcipher_walk_virt(desc, &walk);
+static const struct common_glue_ctx twofish_ctr = {
+ .num_funcs = 2,
+ .fpu_blocks_limit = -1,
+
+ .funcs = { {
+ .num_blocks = 3,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_ctr_3way) }
+ }, {
+ .num_blocks = 1,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_enc_blk_ctr) }
+ } }
+};
- while ((nbytes = walk.nbytes)) {
- nbytes = __cbc_decrypt(desc, &walk);
- err = blkcipher_walk_done(desc, &walk, nbytes);
- }
+static const struct common_glue_ctx twofish_dec = {
+ .num_funcs = 2,
+ .fpu_blocks_limit = -1,
+
+ .funcs = { {
+ .num_blocks = 3,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk_3way) }
+ }, {
+ .num_blocks = 1,
+ .fn_u = { .ecb = GLUE_FUNC_CAST(twofish_dec_blk) }
+ } }
+};
- return err;
-}
+static const struct common_glue_ctx twofish_dec_cbc = {
+ .num_funcs = 2,
+ .fpu_blocks_limit = -1,
+
+ .funcs = { {
+ .num_blocks = 3,
+ .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk_cbc_3way) }
+ }, {
+ .num_blocks = 1,
+ .fn_u = { .cbc = GLUE_CBC_FUNC_CAST(twofish_dec_blk) }
+ } }
+};
-static inline void u128_to_be128(be128 *dst, const u128 *src)
+static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
{
- dst->a = cpu_to_be64(src->a);
- dst->b = cpu_to_be64(src->b);
+ return glue_ecb_crypt_128bit(&twofish_enc, desc, dst, src, nbytes);
}
-static inline void be128_to_u128(u128 *dst, const be128 *src)
+static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
{
- dst->a = be64_to_cpu(src->a);
- dst->b = be64_to_cpu(src->b);
+ return glue_ecb_crypt_128bit(&twofish_dec, desc, dst, src, nbytes);
}
-static inline void u128_inc(u128 *i)
+static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
{
- i->b++;
- if (!i->b)
- i->a++;
+ return glue_cbc_encrypt_128bit(GLUE_FUNC_CAST(twofish_enc_blk), desc,
+ dst, src, nbytes);
}
-static void ctr_crypt_final(struct blkcipher_desc *desc,
- struct blkcipher_walk *walk)
+static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes)
{
- struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- u8 *ctrblk = walk->iv;
- u8 keystream[TF_BLOCK_SIZE];
- u8 *src = walk->src.virt.addr;
- u8 *dst = walk->dst.virt.addr;
- unsigned int nbytes = walk->nbytes;
-
- twofish_enc_blk(ctx, keystream, ctrblk);
- crypto_xor(keystream, src, nbytes);
- memcpy(dst, keystream, nbytes);
-
- crypto_inc(ctrblk, TF_BLOCK_SIZE);
-}
-
-static unsigned int __ctr_crypt(struct blkcipher_desc *desc,
- struct blkcipher_walk *walk)
-{
- struct twofish_ctx *ctx = crypto_blkcipher_ctx(desc->tfm);
- unsigned int bsize = TF_BLOCK_SIZE;
- unsigned int nbytes = walk->nbytes;
- u128 *src = (u128 *)walk->src.virt.addr;
- u128 *dst = (u128 *)walk->dst.virt.addr;
- u128 ctrblk;
- be128 ctrblocks[3];
-
- be128_to_u128(&ctrblk, (be128 *)walk->iv);
-
- /* Process three block batch */
- if (nbytes >= bsize * 3) {
- do {
- if (dst != src) {
- dst[0] = src[0];
- dst[1] = src[1];
- dst[2] = src[2];
- }
-
- /* create ctrblks for parallel encrypt */
- u128_to_be128(&ctrblocks[0], &ctrblk);
- u128_inc(&ctrblk);
- u128_to_be128(&ctrblocks[1], &ctrblk);
- u128_inc(&ctrblk);
- u128_to_be128(&ctrblocks[2], &ctrblk);
- u128_inc(&ctrblk);
-
- twofish_enc_blk_xor_3way(ctx, (u8 *)dst,
- (u8 *)ctrblocks);
-
- src += 3;
- dst += 3;
- nbytes -= bsize * 3;
- } while (nbytes >= bsize * 3);
-
- if (nbytes < bsize)
- goto done;
- }
-
- /* Handle leftovers */
- do {
- if (dst != src)
- *dst = *src;
-
- u128_to_be128(&ctrblocks[0], &ctrblk);
- u128_inc(&ctrblk);
-
- twofish_enc_blk(ctx, (u8 *)ctrblocks, (u8 *)ctrblocks);
- u128_xor(dst, dst, (u128 *)ctrblocks);
-
- src += 1;
- dst += 1;
- nbytes -= bsize;
- } while (nbytes >= bsize);
-
-done:
- u128_to_be128((be128 *)walk->iv, &ctrblk);
- return nbytes;
+ return glue_cbc_decrypt_128bit(&twofish_dec_cbc, desc, dst, src,
+ nbytes);
}
static int ctr_crypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
{
- struct blkcipher_walk walk;
- int err;
-
- blkcipher_walk_init(&walk, dst, src, nbytes);
- err = blkcipher_walk_virt_block(desc, &walk, TF_BLOCK_SIZE);
-
- while ((nbytes = walk.nbytes) >= TF_BLOCK_SIZE) {
- nbytes = __ctr_crypt(desc, &walk);
- err = blkcipher_walk_done(desc, &walk, nbytes);
- }
-
- if (walk.nbytes) {
- ctr_crypt_final(desc, &walk);
- err = blkcipher_walk_done(desc, &walk, 0);
- }
-
- return err;
+ return glue_ctr_crypt_128bit(&twofish_ctr, desc, dst, src, nbytes);
}
static void encrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
@@ -397,13 +213,8 @@ static void decrypt_callback(void *priv, u8 *srcdst, unsigned int nbytes)
twofish_dec_blk(ctx, srcdst, srcdst);
}
-struct twofish_lrw_ctx {
- struct lrw_table_ctx lrw_table;
- struct twofish_ctx twofish_ctx;
-};
-
-static int lrw_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
- unsigned int keylen)
+int lrw_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
+ unsigned int keylen)
{
struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
int err;
@@ -415,6 +226,7 @@ static int lrw_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
return lrw_init_table(&ctx->lrw_table, key + keylen - TF_BLOCK_SIZE);
}
+EXPORT_SYMBOL_GPL(lrw_twofish_setkey);
static int lrw_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
@@ -450,20 +262,16 @@ static int lrw_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
return lrw_crypt(desc, dst, src, nbytes, &req);
}
-static void lrw_exit_tfm(struct crypto_tfm *tfm)
+void lrw_twofish_exit_tfm(struct crypto_tfm *tfm)
{
struct twofish_lrw_ctx *ctx = crypto_tfm_ctx(tfm);
lrw_free_table(&ctx->lrw_table);
}
+EXPORT_SYMBOL_GPL(lrw_twofish_exit_tfm);
-struct twofish_xts_ctx {
- struct twofish_ctx tweak_ctx;
- struct twofish_ctx crypt_ctx;
-};
-
-static int xts_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
- unsigned int keylen)
+int xts_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
+ unsigned int keylen)
{
struct twofish_xts_ctx *ctx = crypto_tfm_ctx(tfm);
u32 *flags = &tfm->crt_flags;
@@ -486,6 +294,7 @@ static int xts_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
return __twofish_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2,
flags);
}
+EXPORT_SYMBOL_GPL(xts_twofish_setkey);
static int xts_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst,
struct scatterlist *src, unsigned int nbytes)
@@ -596,7 +405,7 @@ static struct crypto_alg tf_algs[5] = { {
.cra_type = &crypto_blkcipher_type,
.cra_module = THIS_MODULE,
.cra_list = LIST_HEAD_INIT(tf_algs[3].cra_list),
- .cra_exit = lrw_exit_tfm,
+ .cra_exit = lrw_twofish_exit_tfm,
.cra_u = {
.blkcipher = {
.min_keysize = TF_MIN_KEY_SIZE + TF_BLOCK_SIZE,
diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h
index 3ea51a84a0e4..f34261296ffb 100644
--- a/arch/x86/include/asm/apic.h
+++ b/arch/x86/include/asm/apic.h
@@ -546,7 +546,7 @@ static inline const struct cpumask *online_target_cpus(void)
return cpu_online_mask;
}
-DECLARE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid);
+DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid);
static inline unsigned int read_apic_id(void)
diff --git a/arch/x86/include/asm/bootparam.h b/arch/x86/include/asm/bootparam.h
index eb45aa6b1f27..2ad874cb661c 100644
--- a/arch/x86/include/asm/bootparam.h
+++ b/arch/x86/include/asm/bootparam.h
@@ -66,6 +66,7 @@ struct setup_header {
__u64 setup_data;
__u64 pref_address;
__u32 init_size;
+ __u32 handover_offset;
} __attribute__((packed));
struct sys_desc_table {
diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h
index f91e80f4f180..6b7ee5ff6820 100644
--- a/arch/x86/include/asm/cpufeature.h
+++ b/arch/x86/include/asm/cpufeature.h
@@ -207,6 +207,8 @@
#define X86_FEATURE_ERMS (9*32+ 9) /* Enhanced REP MOVSB/STOSB */
#define X86_FEATURE_INVPCID (9*32+10) /* Invalidate Processor Context ID */
#define X86_FEATURE_RTM (9*32+11) /* Restricted Transactional Memory */
+#define X86_FEATURE_RDSEED (9*32+18) /* The RDSEED instruction */
+#define X86_FEATURE_ADX (9*32+19) /* The ADCX and ADOX instructions */
#if defined(__KERNEL__) && !defined(__ASSEMBLY__)
diff --git a/arch/x86/include/asm/crypto/ablk_helper.h b/arch/x86/include/asm/crypto/ablk_helper.h
new file mode 100644
index 000000000000..4f93df50c23e
--- /dev/null
+++ b/arch/x86/include/asm/crypto/ablk_helper.h
@@ -0,0 +1,31 @@
+/*
+ * Shared async block cipher helpers
+ */
+
+#ifndef _CRYPTO_ABLK_HELPER_H
+#define _CRYPTO_ABLK_HELPER_H
+
+#include <linux/crypto.h>
+#include <linux/kernel.h>
+#include <crypto/cryptd.h>
+
+struct async_helper_ctx {
+ struct cryptd_ablkcipher *cryptd_tfm;
+};
+
+extern int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key,
+ unsigned int key_len);
+
+extern int __ablk_encrypt(struct ablkcipher_request *req);
+
+extern int ablk_encrypt(struct ablkcipher_request *req);
+
+extern int ablk_decrypt(struct ablkcipher_request *req);
+
+extern void ablk_exit(struct crypto_tfm *tfm);
+
+extern int ablk_init_common(struct crypto_tfm *tfm, const char *drv_name);
+
+extern int ablk_init(struct crypto_tfm *tfm);
+
+#endif /* _CRYPTO_ABLK_HELPER_H */
diff --git a/arch/x86/include/asm/aes.h b/arch/x86/include/asm/crypto/aes.h
index 80545a1cbe39..80545a1cbe39 100644
--- a/arch/x86/include/asm/aes.h
+++ b/arch/x86/include/asm/crypto/aes.h
diff --git a/arch/x86/include/asm/crypto/glue_helper.h b/arch/x86/include/asm/crypto/glue_helper.h
new file mode 100644
index 000000000000..3e408bddc96f
--- /dev/null
+++ b/arch/x86/include/asm/crypto/glue_helper.h
@@ -0,0 +1,115 @@
+/*
+ * Shared glue code for 128bit block ciphers
+ */
+
+#ifndef _CRYPTO_GLUE_HELPER_H
+#define _CRYPTO_GLUE_HELPER_H
+
+#include <linux/kernel.h>
+#include <linux/crypto.h>
+#include <asm/i387.h>
+#include <crypto/b128ops.h>
+
+typedef void (*common_glue_func_t)(void *ctx, u8 *dst, const u8 *src);
+typedef void (*common_glue_cbc_func_t)(void *ctx, u128 *dst, const u128 *src);
+typedef void (*common_glue_ctr_func_t)(void *ctx, u128 *dst, const u128 *src,
+ u128 *iv);
+
+#define GLUE_FUNC_CAST(fn) ((common_glue_func_t)(fn))
+#define GLUE_CBC_FUNC_CAST(fn) ((common_glue_cbc_func_t)(fn))
+#define GLUE_CTR_FUNC_CAST(fn) ((common_glue_ctr_func_t)(fn))
+
+struct common_glue_func_entry {
+ unsigned int num_blocks; /* number of blocks that @fn will process */
+ union {
+ common_glue_func_t ecb;
+ common_glue_cbc_func_t cbc;
+ common_glue_ctr_func_t ctr;
+ } fn_u;
+};
+
+struct common_glue_ctx {
+ unsigned int num_funcs;
+ int fpu_blocks_limit; /* -1 means fpu not needed at all */
+
+ /*
+ * First funcs entry must have largest num_blocks and last funcs entry
+ * must have num_blocks == 1!
+ */
+ struct common_glue_func_entry funcs[];
+};
+
+static inline bool glue_fpu_begin(unsigned int bsize, int fpu_blocks_limit,
+ struct blkcipher_desc *desc,
+ bool fpu_enabled, unsigned int nbytes)
+{
+ if (likely(fpu_blocks_limit < 0))
+ return false;
+
+ if (fpu_enabled)
+ return true;
+
+ /*
+ * Vector-registers are only used when chunk to be processed is large
+ * enough, so do not enable FPU until it is necessary.
+ */
+ if (nbytes < bsize * (unsigned int)fpu_blocks_limit)
+ return false;
+
+ if (desc) {
+ /* prevent sleeping if FPU is in use */
+ desc->flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP;
+ }
+
+ kernel_fpu_begin();
+ return true;
+}
+
+static inline void glue_fpu_end(bool fpu_enabled)
+{
+ if (fpu_enabled)
+ kernel_fpu_end();
+}
+
+static inline void u128_to_be128(be128 *dst, const u128 *src)
+{
+ dst->a = cpu_to_be64(src->a);
+ dst->b = cpu_to_be64(src->b);
+}
+
+static inline void be128_to_u128(u128 *dst, const be128 *src)
+{
+ dst->a = be64_to_cpu(src->a);
+ dst->b = be64_to_cpu(src->b);
+}
+
+static inline void u128_inc(u128 *i)
+{
+ i->b++;
+ if (!i->b)
+ i->a++;
+}
+
+extern int glue_ecb_crypt_128bit(const struct common_glue_ctx *gctx,
+ struct blkcipher_desc *desc,
+ struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes);
+
+extern int glue_cbc_encrypt_128bit(const common_glue_func_t fn,
+ struct blkcipher_desc *desc,
+ struct scatterlist *dst,
+ struct scatterlist *src,
+ unsigned int nbytes);
+
+extern int glue_cbc_decrypt_128bit(const struct common_glue_ctx *gctx,
+ struct blkcipher_desc *desc,
+ struct scatterlist *dst,
+ struct scatterlist *src,
+ unsigned int nbytes);
+
+extern int glue_ctr_crypt_128bit(const struct common_glue_ctx *gctx,
+ struct blkcipher_desc *desc,
+ struct scatterlist *dst,
+ struct scatterlist *src, unsigned int nbytes);
+
+#endif /* _CRYPTO_GLUE_HELPER_H */
diff --git a/arch/x86/include/asm/crypto/serpent-avx.h b/arch/x86/include/asm/crypto/serpent-avx.h
new file mode 100644
index 000000000000..432deedd2945
--- /dev/null
+++ b/arch/x86/include/asm/crypto/serpent-avx.h
@@ -0,0 +1,32 @@
+#ifndef ASM_X86_SERPENT_AVX_H
+#define ASM_X86_SERPENT_AVX_H
+
+#include <linux/crypto.h>
+#include <crypto/serpent.h>
+
+#define SERPENT_PARALLEL_BLOCKS 8
+
+asmlinkage void __serpent_enc_blk_8way_avx(struct serpent_ctx *ctx, u8 *dst,
+ const u8 *src, bool xor);
+asmlinkage void serpent_dec_blk_8way_avx(struct serpent_ctx *ctx, u8 *dst,
+ const u8 *src);
+
+static inline void serpent_enc_blk_xway(struct serpent_ctx *ctx, u8 *dst,
+ const u8 *src)
+{
+ __serpent_enc_blk_8way_avx(ctx, dst, src, false);
+}
+
+static inline void serpent_enc_blk_xway_xor(struct serpent_ctx *ctx, u8 *dst,
+ const u8 *src)
+{
+ __serpent_enc_blk_8way_avx(ctx, dst, src, true);
+}
+
+static inline void serpent_dec_blk_xway(struct serpent_ctx *ctx, u8 *dst,
+ const u8 *src)
+{
+ serpent_dec_blk_8way_avx(ctx, dst, src);
+}
+
+#endif
diff --git a/arch/x86/include/asm/serpent.h b/arch/x86/include/asm/crypto/serpent-sse2.h
index d3ef63fe0c81..e6e77dffbdab 100644
--- a/arch/x86/include/asm/serpent.h
+++ b/arch/x86/include/asm/crypto/serpent-sse2.h
@@ -1,5 +1,5 @@
-#ifndef ASM_X86_SERPENT_H
-#define ASM_X86_SERPENT_H
+#ifndef ASM_X86_SERPENT_SSE2_H
+#define ASM_X86_SERPENT_SSE2_H
#include <linux/crypto.h>
#include <crypto/serpent.h>
diff --git a/arch/x86/include/asm/crypto/twofish.h b/arch/x86/include/asm/crypto/twofish.h
new file mode 100644
index 000000000000..9d2c514bd5f9
--- /dev/null
+++ b/arch/x86/include/asm/crypto/twofish.h
@@ -0,0 +1,46 @@
+#ifndef ASM_X86_TWOFISH_H
+#define ASM_X86_TWOFISH_H
+
+#include <linux/crypto.h>
+#include <crypto/twofish.h>
+#include <crypto/lrw.h>
+#include <crypto/b128ops.h>
+
+struct twofish_lrw_ctx {
+ struct lrw_table_ctx lrw_table;
+ struct twofish_ctx twofish_ctx;
+};
+
+struct twofish_xts_ctx {
+ struct twofish_ctx tweak_ctx;
+ struct twofish_ctx crypt_ctx;
+};
+
+/* regular block cipher functions from twofish_x86_64 module */
+asmlinkage void twofish_enc_blk(struct twofish_ctx *ctx, u8 *dst,
+ const u8 *src);
+asmlinkage void twofish_dec_blk(struct twofish_ctx *ctx, u8 *dst,
+ const u8 *src);
+
+/* 3-way parallel cipher functions */
+asmlinkage void __twofish_enc_blk_3way(struct twofish_ctx *ctx, u8 *dst,
+ const u8 *src, bool xor);
+asmlinkage void twofish_dec_blk_3way(struct twofish_ctx *ctx, u8 *dst,
+ const u8 *src);
+
+/* helpers from twofish_x86_64-3way module */
+extern void twofish_dec_blk_cbc_3way(void *ctx, u128 *dst, const u128 *src);
+extern void twofish_enc_blk_ctr(void *ctx, u128 *dst, const u128 *src,
+ u128 *iv);
+extern void twofish_enc_blk_ctr_3way(void *ctx, u128 *dst, const u128 *src,
+ u128 *iv);
+
+extern int lrw_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
+ unsigned int keylen);
+
+extern void lrw_twofish_exit_tfm(struct crypto_tfm *tfm);
+
+extern int xts_twofish_setkey(struct crypto_tfm *tfm, const u8 *key,
+ unsigned int keylen);
+
+#endif /* ASM_X86_TWOFISH_H */
diff --git a/arch/x86/include/asm/entry_arch.h b/arch/x86/include/asm/entry_arch.h
index 0baa628e330c..40afa0005c69 100644
--- a/arch/x86/include/asm/entry_arch.h
+++ b/arch/x86/include/asm/entry_arch.h
@@ -15,15 +15,6 @@ BUILD_INTERRUPT(call_function_interrupt,CALL_FUNCTION_VECTOR)
BUILD_INTERRUPT(call_function_single_interrupt,CALL_FUNCTION_SINGLE_VECTOR)
BUILD_INTERRUPT(irq_move_cleanup_interrupt,IRQ_MOVE_CLEANUP_VECTOR)
BUILD_INTERRUPT(reboot_interrupt,REBOOT_VECTOR)
-
-.irp idx,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, \
- 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
-.if NUM_INVALIDATE_TLB_VECTORS > \idx
-BUILD_INTERRUPT3(invalidate_interrupt\idx,
- (INVALIDATE_TLB_VECTOR_START)+\idx,
- smp_invalidate_interrupt)
-.endif
-.endr
#endif
BUILD_INTERRUPT(x86_platform_ipi, X86_PLATFORM_IPI_VECTOR)
diff --git a/arch/x86/include/asm/irq_vectors.h b/arch/x86/include/asm/irq_vectors.h
index 4b4448761e88..1508e518c7e3 100644
--- a/arch/x86/include/asm/irq_vectors.h
+++ b/arch/x86/include/asm/irq_vectors.h
@@ -119,17 +119,6 @@
*/
#define LOCAL_TIMER_VECTOR 0xef
-/* up to 32 vectors used for spreading out TLB flushes: */
-#if NR_CPUS <= 32
-# define NUM_INVALIDATE_TLB_VECTORS (NR_CPUS)
-#else
-# define NUM_INVALIDATE_TLB_VECTORS (32)
-#endif
-
-#define INVALIDATE_TLB_VECTOR_END (0xee)
-#define INVALIDATE_TLB_VECTOR_START \
- (INVALIDATE_TLB_VECTOR_END-NUM_INVALIDATE_TLB_VECTORS+1)
-
#define NR_VECTORS 256
#define FPU_IRQ 13
diff --git a/arch/x86/include/asm/paravirt.h b/arch/x86/include/asm/paravirt.h
index 0b47ddb6f00b..a0facf3908d7 100644
--- a/arch/x86/include/asm/paravirt.h
+++ b/arch/x86/include/asm/paravirt.h
@@ -360,9 +360,10 @@ static inline void __flush_tlb_single(unsigned long addr)
static inline void flush_tlb_others(const struct cpumask *cpumask,
struct mm_struct *mm,
- unsigned long va)
+ unsigned long start,
+ unsigned long end)
{
- PVOP_VCALL3(pv_mmu_ops.flush_tlb_others, cpumask, mm, va);
+ PVOP_VCALL4(pv_mmu_ops.flush_tlb_others, cpumask, mm, start, end);
}
static inline int paravirt_pgd_alloc(struct mm_struct *mm)
diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
index 8613cbb7ba41..142236ed83af 100644
--- a/arch/x86/include/asm/paravirt_types.h
+++ b/arch/x86/include/asm/paravirt_types.h
@@ -248,7 +248,8 @@ struct pv_mmu_ops {
void (*flush_tlb_single)(unsigned long addr);
void (*flush_tlb_others)(const struct cpumask *cpus,
struct mm_struct *mm,
- unsigned long va);
+ unsigned long start,
+ unsigned long end);
/* Hooks for allocating and freeing a pagetable top-level */
int (*pgd_alloc)(struct mm_struct *mm);
diff --git a/arch/x86/include/asm/percpu.h b/arch/x86/include/asm/percpu.h
index d9b8e3f7f42a..1104afaba52b 100644
--- a/arch/x86/include/asm/percpu.h
+++ b/arch/x86/include/asm/percpu.h
@@ -551,6 +551,12 @@ DECLARE_PER_CPU(unsigned long, this_cpu_off);
{ [0 ... NR_CPUS-1] = _initvalue }; \
__typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map
+#define DEFINE_EARLY_PER_CPU_READ_MOSTLY(_type, _name, _initvalue) \
+ DEFINE_PER_CPU_READ_MOSTLY(_type, _name) = _initvalue; \
+ __typeof__(_type) _name##_early_map[NR_CPUS] __initdata = \
+ { [0 ... NR_CPUS-1] = _initvalue }; \
+ __typeof__(_type) *_name##_early_ptr __refdata = _name##_early_map
+
#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \
EXPORT_PER_CPU_SYMBOL(_name)
@@ -559,6 +565,11 @@ DECLARE_PER_CPU(unsigned long, this_cpu_off);
extern __typeof__(_type) *_name##_early_ptr; \
extern __typeof__(_type) _name##_early_map[]
+#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name) \
+ DECLARE_PER_CPU_READ_MOSTLY(_type, _name); \
+ extern __typeof__(_type) *_name##_early_ptr; \
+ extern __typeof__(_type) _name##_early_map[]
+
#define early_per_cpu_ptr(_name) (_name##_early_ptr)
#define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx])
#define early_per_cpu(_name, _cpu) \
@@ -570,12 +581,18 @@ DECLARE_PER_CPU(unsigned long, this_cpu_off);
#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \
DEFINE_PER_CPU(_type, _name) = _initvalue
+#define DEFINE_EARLY_PER_CPU_READ_MOSTLY(_type, _name, _initvalue) \
+ DEFINE_PER_CPU_READ_MOSTLY(_type, _name) = _initvalue
+
#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \
EXPORT_PER_CPU_SYMBOL(_name)
#define DECLARE_EARLY_PER_CPU(_type, _name) \
DECLARE_PER_CPU(_type, _name)
+#define DECLARE_EARLY_PER_CPU_READ_MOSTLY(_type, _name) \
+ DECLARE_PER_CPU_READ_MOSTLY(_type, _name)
+
#define early_per_cpu(_name, _cpu) per_cpu(_name, _cpu)
#define early_per_cpu_ptr(_name) NULL
/* no early_per_cpu_map() */
diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 39bc5777211a..d048cad9bcad 100644
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -61,6 +61,19 @@ static inline void *current_text_addr(void)
# define ARCH_MIN_MMSTRUCT_ALIGN 0
#endif
+enum tlb_infos {
+ ENTRIES,
+ NR_INFO
+};
+
+extern u16 __read_mostly tlb_lli_4k[NR_INFO];
+extern u16 __read_mostly tlb_lli_2m[NR_INFO];
+extern u16 __read_mostly tlb_lli_4m[NR_INFO];
+extern u16 __read_mostly tlb_lld_4k[NR_INFO];
+extern u16 __read_mostly tlb_lld_2m[NR_INFO];
+extern u16 __read_mostly tlb_lld_4m[NR_INFO];
+extern s8 __read_mostly tlb_flushall_shift;
+
/*
* CPU type and hardware bug flags. Kept separately for each CPU.
* Members of this structure are referenced in head.S, so think twice
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index 2ffa95dc2333..4f19a1526037 100644
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -31,12 +31,12 @@ static inline bool cpu_has_ht_siblings(void)
return has_siblings;
}
-DECLARE_PER_CPU(cpumask_var_t, cpu_sibling_map);
-DECLARE_PER_CPU(cpumask_var_t, cpu_core_map);
+DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
+DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map);
/* cpus sharing the last level cache: */
-DECLARE_PER_CPU(cpumask_var_t, cpu_llc_shared_map);
-DECLARE_PER_CPU(u16, cpu_llc_id);
-DECLARE_PER_CPU(int, cpu_number);
+DECLARE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map);
+DECLARE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id);
+DECLARE_PER_CPU_READ_MOSTLY(int, cpu_number);
static inline struct cpumask *cpu_sibling_mask(int cpu)
{
@@ -53,10 +53,10 @@ static inline struct cpumask *cpu_llc_shared_mask(int cpu)
return per_cpu(cpu_llc_shared_map, cpu);
}
-DECLARE_EARLY_PER_CPU(u16, x86_cpu_to_apicid);
-DECLARE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid);
+DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid);
+DECLARE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid);
#if defined(CONFIG_X86_LOCAL_APIC) && defined(CONFIG_X86_32)
-DECLARE_EARLY_PER_CPU(int, x86_cpu_to_logical_apicid);
+DECLARE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid);
#endif
/* Static state in head.S used to set up a CPU */
diff --git a/arch/x86/include/asm/tlb.h b/arch/x86/include/asm/tlb.h
index 829215fef9ee..4fef20773b8f 100644
--- a/arch/x86/include/asm/tlb.h
+++ b/arch/x86/include/asm/tlb.h
@@ -4,7 +4,14 @@
#define tlb_start_vma(tlb, vma) do { } while (0)
#define tlb_end_vma(tlb, vma) do { } while (0)
#define __tlb_remove_tlb_entry(tlb, ptep, address) do { } while (0)
-#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
+
+#define tlb_flush(tlb) \
+{ \
+ if (tlb->fullmm == 0) \
+ flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end, 0UL); \
+ else \
+ flush_tlb_mm_range(tlb->mm, 0UL, TLB_FLUSH_ALL, 0UL); \
+}
#include <asm-generic/tlb.h>
diff --git a/arch/x86/include/asm/tlbflush.h b/arch/x86/include/asm/tlbflush.h
index 36a1a2ab87d2..74a44333545a 100644
--- a/arch/x86/include/asm/tlbflush.h
+++ b/arch/x86/include/asm/tlbflush.h
@@ -73,14 +73,10 @@ static inline void __flush_tlb_one(unsigned long addr)
* - flush_tlb_page(vma, vmaddr) flushes one page
* - flush_tlb_range(vma, start, end) flushes a range of pages
* - flush_tlb_kernel_range(start, end) flushes a range of kernel pages
- * - flush_tlb_others(cpumask, mm, va) flushes TLBs on other cpus
+ * - flush_tlb_others(cpumask, mm, start, end) flushes TLBs on other cpus
*
* ..but the i386 has somewhat limited tlb flushing capabilities,
* and page-granular flushes are available only on i486 and up.
- *
- * x86-64 can only flush individual pages or full VMs. For a range flush
- * we always do the full VM. Might be worth trying if for a small
- * range a few INVLPGs in a row are a win.
*/
#ifndef CONFIG_SMP
@@ -109,9 +105,17 @@ static inline void flush_tlb_range(struct vm_area_struct *vma,
__flush_tlb();
}
+static inline void flush_tlb_mm_range(struct mm_struct *mm,
+ unsigned long start, unsigned long end, unsigned long vmflag)
+{
+ if (mm == current->active_mm)
+ __flush_tlb();
+}
+
static inline void native_flush_tlb_others(const struct cpumask *cpumask,
struct mm_struct *mm,
- unsigned long va)
+ unsigned long start,
+ unsigned long end)
{
}
@@ -119,27 +123,35 @@ static inline void reset_lazy_tlbstate(void)
{
}
+static inline void flush_tlb_kernel_range(unsigned long start,
+ unsigned long end)
+{
+ flush_tlb_all();
+}
+
#else /* SMP */
#include <asm/smp.h>
#define local_flush_tlb() __flush_tlb()
+#define flush_tlb_mm(mm) flush_tlb_mm_range(mm, 0UL, TLB_FLUSH_ALL, 0UL)
+
+#define flush_tlb_range(vma, start, end) \
+ flush_tlb_mm_range(vma->vm_mm, start, end, vma->vm_flags)
+
extern void flush_tlb_all(void);
extern void flush_tlb_current_task(void);
-extern void flush_tlb_mm(struct mm_struct *);
extern void flush_tlb_page(struct vm_area_struct *, unsigned long);
+extern void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
+ unsigned long end, unsigned long vmflag);
+extern void flush_tlb_kernel_range(unsigned long start, unsigned long end);
#define flush_tlb() flush_tlb_current_task()
-static inline void flush_tlb_range(struct vm_area_struct *vma,
- unsigned long start, unsigned long end)
-{
- flush_tlb_mm(vma->vm_mm);
-}
-
void native_flush_tlb_others(const struct cpumask *cpumask,
- struct mm_struct *mm, unsigned long va);
+ struct mm_struct *mm,
+ unsigned long start, unsigned long end);
#define TLBSTATE_OK 1
#define TLBSTATE_LAZY 2
@@ -159,13 +171,8 @@ static inline void reset_lazy_tlbstate(void)
#endif /* SMP */
#ifndef CONFIG_PARAVIRT
-#define flush_tlb_others(mask, mm, va) native_flush_tlb_others(mask, mm, va)
+#define flush_tlb_others(mask, mm, start, end) \
+ native_flush_tlb_others(mask, mm, start, end)
#endif
-static inline void flush_tlb_kernel_range(unsigned long start,
- unsigned long end)
-{
- flush_tlb_all();
-}
-
#endif /* _ASM_X86_TLBFLUSH_H */
diff --git a/arch/x86/include/asm/uv/uv.h b/arch/x86/include/asm/uv/uv.h
index 3bb9491b7659..b47c2a82ff15 100644
--- a/arch/x86/include/asm/uv/uv.h
+++ b/arch/x86/include/asm/uv/uv.h
@@ -15,7 +15,8 @@ extern void uv_nmi_init(void);
extern void uv_system_init(void);
extern const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
struct mm_struct *mm,
- unsigned long va,
+ unsigned long start,
+ unsigned end,
unsigned int cpu);
#else /* X86_UV */
@@ -26,7 +27,7 @@ static inline void uv_cpu_init(void) { }
static inline void uv_system_init(void) { }
static inline const struct cpumask *
uv_flush_tlb_others(const struct cpumask *cpumask, struct mm_struct *mm,
- unsigned long va, unsigned int cpu)
+ unsigned long start, unsigned long end, unsigned int cpu)
{ return cpumask; }
#endif /* X86_UV */
diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c
index 98e24131ff3a..24deb3082328 100644
--- a/arch/x86/kernel/apic/apic.c
+++ b/arch/x86/kernel/apic/apic.c
@@ -75,8 +75,8 @@ physid_mask_t phys_cpu_present_map;
/*
* Map cpu index to physical APIC ID
*/
-DEFINE_EARLY_PER_CPU(u16, x86_cpu_to_apicid, BAD_APICID);
-DEFINE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid, BAD_APICID);
+DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_cpu_to_apicid, BAD_APICID);
+DEFINE_EARLY_PER_CPU_READ_MOSTLY(u16, x86_bios_cpu_apicid, BAD_APICID);
EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid);
EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
@@ -88,7 +88,7 @@ EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid);
* used for the mapping. This is where the behaviors of x86_64 and 32
* actually diverge. Let's keep it ugly for now.
*/
-DEFINE_EARLY_PER_CPU(int, x86_cpu_to_logical_apicid, BAD_APICID);
+DEFINE_EARLY_PER_CPU_READ_MOSTLY(int, x86_cpu_to_logical_apicid, BAD_APICID);
/*
* Knob to control our willingness to enable the local APIC.
diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile
index bac4c3804cc7..d30a6a9a0121 100644
--- a/arch/x86/kernel/cpu/Makefile
+++ b/arch/x86/kernel/cpu/Makefile
@@ -14,7 +14,7 @@ CFLAGS_common.o := $(nostackp)
obj-y := intel_cacheinfo.o scattered.o topology.o
obj-y += proc.o capflags.o powerflags.o common.o
-obj-y += vmware.o hypervisor.o sched.o mshyperv.o
+obj-y += vmware.o hypervisor.o mshyperv.o
obj-y += rdrand.o
obj-y += match.o
diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 5bbc082c47ad..46d8786d655e 100644
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -452,6 +452,35 @@ void __cpuinit cpu_detect_cache_sizes(struct cpuinfo_x86 *c)
c->x86_cache_size = l2size;
}
+u16 __read_mostly tlb_lli_4k[NR_INFO];
+u16 __read_mostly tlb_lli_2m[NR_INFO];
+u16 __read_mostly tlb_lli_4m[NR_INFO];
+u16 __read_mostly tlb_lld_4k[NR_INFO];
+u16 __read_mostly tlb_lld_2m[NR_INFO];
+u16 __read_mostly tlb_lld_4m[NR_INFO];
+
+/*
+ * tlb_flushall_shift shows the balance point in replacing cr3 write
+ * with multiple 'invlpg'. It will do this replacement when
+ * flush_tlb_lines <= active_lines/2^tlb_flushall_shift.
+ * If tlb_flushall_shift is -1, means the replacement will be disabled.
+ */
+s8 __read_mostly tlb_flushall_shift = -1;
+
+void __cpuinit cpu_detect_tlb(struct cpuinfo_x86 *c)
+{
+ if (this_cpu->c_detect_tlb)
+ this_cpu->c_detect_tlb(c);
+
+ printk(KERN_INFO "Last level iTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \
+ "Last level dTLB entries: 4KB %d, 2MB %d, 4MB %d\n" \
+ "tlb_flushall_shift is 0x%x\n",
+ tlb_lli_4k[ENTRIES], tlb_lli_2m[ENTRIES],
+ tlb_lli_4m[ENTRIES], tlb_lld_4k[ENTRIES],
+ tlb_lld_2m[ENTRIES], tlb_lld_4m[ENTRIES],
+ tlb_flushall_shift);
+}
+
void __cpuinit detect_ht(struct cpuinfo_x86 *c)
{
#ifdef CONFIG_X86_HT
@@ -911,6 +940,8 @@ void __init identify_boot_cpu(void)
#else
vgetcpu_set_mode();
#endif
+ if (boot_cpu_data.cpuid_level >= 2)
+ cpu_detect_tlb(&boot_cpu_data);
}
void __cpuinit identify_secondary_cpu(struct cpuinfo_x86 *c)
diff --git a/arch/x86/kernel/cpu/cpu.h b/arch/x86/kernel/cpu/cpu.h
index 8bacc7826fb3..4041c24ae7db 100644
--- a/arch/x86/kernel/cpu/cpu.h
+++ b/arch/x86/kernel/cpu/cpu.h
@@ -20,10 +20,19 @@ struct cpu_dev {
void (*c_bsp_init)(struct cpuinfo_x86 *);
void (*c_init)(struct cpuinfo_x86 *);
void (*c_identify)(struct cpuinfo_x86 *);
+ void (*c_detect_tlb)(struct cpuinfo_x86 *);
unsigned int (*c_size_cache)(struct cpuinfo_x86 *, unsigned int);
int c_x86_vendor;
};
+struct _tlb_table {
+ unsigned char descriptor;
+ char tlb_type;
+ unsigned int entries;
+ /* unsigned int ways; */
+ char info[128];
+};
+
#define cpu_dev_register(cpu_devX) \
static const struct cpu_dev *const __cpu_dev_##cpu_devX __used \
__attribute__((__section__(".x86_cpu_dev.init"))) = \
diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c
index 3e6ff6cbf42a..0a4ce2980a5a 100644
--- a/arch/x86/kernel/cpu/intel.c
+++ b/arch/x86/kernel/cpu/intel.c
@@ -491,6 +491,181 @@ static unsigned int __cpuinit intel_size_cache(struct cpuinfo_x86 *c, unsigned i
}
#endif
+#define TLB_INST_4K 0x01
+#define TLB_INST_4M 0x02
+#define TLB_INST_2M_4M 0x03
+
+#define TLB_INST_ALL 0x05
+#define TLB_INST_1G 0x06
+
+#define TLB_DATA_4K 0x11
+#define TLB_DATA_4M 0x12
+#define TLB_DATA_2M_4M 0x13
+#define TLB_DATA_4K_4M 0x14
+
+#define TLB_DATA_1G 0x16
+
+#define TLB_DATA0_4K 0x21
+#define TLB_DATA0_4M 0x22
+#define TLB_DATA0_2M_4M 0x23
+
+#define STLB_4K 0x41
+
+static const struct _tlb_table intel_tlb_table[] __cpuinitconst = {
+ { 0x01, TLB_INST_4K, 32, " TLB_INST 4 KByte pages, 4-way set associative" },
+ { 0x02, TLB_INST_4M, 2, " TLB_INST 4 MByte pages, full associative" },
+ { 0x03, TLB_DATA_4K, 64, " TLB_DATA 4 KByte pages, 4-way set associative" },
+ { 0x04, TLB_DATA_4M, 8, " TLB_DATA 4 MByte pages, 4-way set associative" },
+ { 0x05, TLB_DATA_4M, 32, " TLB_DATA 4 MByte pages, 4-way set associative" },
+ { 0x0b, TLB_INST_4M, 4, " TLB_INST 4 MByte pages, 4-way set associative" },
+ { 0x4f, TLB_INST_4K, 32, " TLB_INST 4 KByte pages */" },
+ { 0x50, TLB_INST_ALL, 64, " TLB_INST 4 KByte and 2-MByte or 4-MByte pages" },
+ { 0x51, TLB_INST_ALL, 128, " TLB_INST 4 KByte and 2-MByte or 4-MByte pages" },
+ { 0x52, TLB_INST_ALL, 256, " TLB_INST 4 KByte and 2-MByte or 4-MByte pages" },
+ { 0x55, TLB_INST_2M_4M, 7, " TLB_INST 2-MByte or 4-MByte pages, fully associative" },
+ { 0x56, TLB_DATA0_4M, 16, " TLB_DATA0 4 MByte pages, 4-way set associative" },
+ { 0x57, TLB_DATA0_4K, 16, " TLB_DATA0 4 KByte pages, 4-way associative" },
+ { 0x59, TLB_DATA0_4K, 16, " TLB_DATA0 4 KByte pages, fully associative" },
+ { 0x5a, TLB_DATA0_2M_4M, 32, " TLB_DATA0 2-MByte or 4 MByte pages, 4-way set associative" },
+ { 0x5b, TLB_DATA_4K_4M, 64, " TLB_DATA 4 KByte and 4 MByte pages" },
+ { 0x5c, TLB_DATA_4K_4M, 128, " TLB_DATA 4 KByte and 4 MByte pages" },
+ { 0x5d, TLB_DATA_4K_4M, 256, " TLB_DATA 4 KByte and 4 MByte pages" },
+ { 0xb0, TLB_INST_4K, 128, " TLB_INST 4 KByte pages, 4-way set associative" },
+ { 0xb1, TLB_INST_2M_4M, 4, " TLB_INST 2M pages, 4-way, 8 entries or 4M pages, 4-way entries" },
+ { 0xb2, TLB_INST_4K, 64, " TLB_INST 4KByte pages, 4-way set associative" },
+ { 0xb3, TLB_DATA_4K, 128, " TLB_DATA 4 KByte pages, 4-way set associative" },
+ { 0xb4, TLB_DATA_4K, 256, " TLB_DATA 4 KByte pages, 4-way associative" },
+ { 0xba, TLB_DATA_4K, 64, " TLB_DATA 4 KByte pages, 4-way associative" },
+ { 0xc0, TLB_DATA_4K_4M, 8, " TLB_DATA 4 KByte and 4 MByte pages, 4-way associative" },
+ { 0xca, STLB_4K, 512, " STLB 4 KByte pages, 4-way associative" },
+ { 0x00, 0, 0 }
+};
+
+static void __cpuinit intel_tlb_lookup(const unsigned char desc)
+{
+ unsigned char k;
+ if (desc == 0)
+ return;
+
+ /* look up this descriptor in the table */
+ for (k = 0; intel_tlb_table[k].descriptor != desc && \
+ intel_tlb_table[k].descriptor != 0; k++)
+ ;
+
+ if (intel_tlb_table[k].tlb_type == 0)
+ return;
+
+ switch (intel_tlb_table[k].tlb_type) {
+ case STLB_4K:
+ if (tlb_lli_4k[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lli_4k[ENTRIES] = intel_tlb_table[k].entries;
+ if (tlb_lld_4k[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lld_4k[ENTRIES] = intel_tlb_table[k].entries;
+ break;
+ case TLB_INST_ALL:
+ if (tlb_lli_4k[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lli_4k[ENTRIES] = intel_tlb_table[k].entries;
+ if (tlb_lli_2m[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lli_2m[ENTRIES] = intel_tlb_table[k].entries;
+ if (tlb_lli_4m[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lli_4m[ENTRIES] = intel_tlb_table[k].entries;
+ break;
+ case TLB_INST_4K:
+ if (tlb_lli_4k[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lli_4k[ENTRIES] = intel_tlb_table[k].entries;
+ break;
+ case TLB_INST_4M:
+ if (tlb_lli_4m[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lli_4m[ENTRIES] = intel_tlb_table[k].entries;
+ break;
+ case TLB_INST_2M_4M:
+ if (tlb_lli_2m[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lli_2m[ENTRIES] = intel_tlb_table[k].entries;
+ if (tlb_lli_4m[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lli_4m[ENTRIES] = intel_tlb_table[k].entries;
+ break;
+ case TLB_DATA_4K:
+ case TLB_DATA0_4K:
+ if (tlb_lld_4k[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lld_4k[ENTRIES] = intel_tlb_table[k].entries;
+ break;
+ case TLB_DATA_4M:
+ case TLB_DATA0_4M:
+ if (tlb_lld_4m[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lld_4m[ENTRIES] = intel_tlb_table[k].entries;
+ break;
+ case TLB_DATA_2M_4M:
+ case TLB_DATA0_2M_4M:
+ if (tlb_lld_2m[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lld_2m[ENTRIES] = intel_tlb_table[k].entries;
+ if (tlb_lld_4m[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lld_4m[ENTRIES] = intel_tlb_table[k].entries;
+ break;
+ case TLB_DATA_4K_4M:
+ if (tlb_lld_4k[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lld_4k[ENTRIES] = intel_tlb_table[k].entries;
+ if (tlb_lld_4m[ENTRIES] < intel_tlb_table[k].entries)
+ tlb_lld_4m[ENTRIES] = intel_tlb_table[k].entries;
+ break;
+ }
+}
+
+static void __cpuinit intel_tlb_flushall_shift_set(struct cpuinfo_x86 *c)
+{
+ if (!cpu_has_invlpg) {
+ tlb_flushall_shift = -1;
+ return;
+ }
+ switch ((c->x86 << 8) + c->x86_model) {
+ case 0x60f: /* original 65 nm celeron/pentium/core2/xeon, "Merom"/"Conroe" */
+ case 0x616: /* single-core 65 nm celeron/core2solo "Merom-L"/"Conroe-L" */
+ case 0x617: /* current 45 nm celeron/core2/xeon "Penryn"/"Wolfdale" */
+ case 0x61d: /* six-core 45 nm xeon "Dunnington" */
+ tlb_flushall_shift = -1;
+ break;
+ case 0x61a: /* 45 nm nehalem, "Bloomfield" */
+ case 0x61e: /* 45 nm nehalem, "Lynnfield" */
+ case 0x625: /* 32 nm nehalem, "Clarkdale" */
+ case 0x62c: /* 32 nm nehalem, "Gulftown" */
+ case 0x62e: /* 45 nm nehalem-ex, "Beckton" */
+ case 0x62f: /* 32 nm Xeon E7 */
+ tlb_flushall_shift = 6;
+ break;
+ case 0x62a: /* SandyBridge */
+ case 0x62d: /* SandyBridge, "Romely-EP" */
+ tlb_flushall_shift = 5;
+ break;
+ case 0x63a: /* Ivybridge */
+ tlb_flushall_shift = 1;
+ break;
+ default:
+ tlb_flushall_shift = 6;
+ }
+}
+
+static void __cpuinit intel_detect_tlb(struct cpuinfo_x86 *c)
+{
+ int i, j, n;
+ unsigned int regs[4];
+ unsigned char *desc = (unsigned char *)regs;
+ /* Number of times to iterate */
+ n = cpuid_eax(2) & 0xFF;
+
+ for (i = 0 ; i < n ; i++) {
+ cpuid(2, &regs[0], &regs[1], &regs[2], &regs[3]);
+
+ /* If bit 31 is set, this is an unknown format */
+ for (j = 0 ; j < 3 ; j++)
+ if (regs[j] & (1 << 31))
+ regs[j] = 0;
+
+ /* Byte 0 is level count, not a descriptor */
+ for (j = 1 ; j < 16 ; j++)
+ intel_tlb_lookup(desc[j]);
+ }
+ intel_tlb_flushall_shift_set(c);
+}
+
static const struct cpu_dev __cpuinitconst intel_cpu_dev = {
.c_vendor = "Intel",
.c_ident = { "GenuineIntel" },
@@ -546,6 +721,7 @@ static const struct cpu_dev __cpuinitconst intel_cpu_dev = {
},
.c_size_cache = intel_size_cache,
#endif
+ .c_detect_tlb = intel_detect_tlb,
.c_early_init = early_init_intel,
.c_init = init_intel,
.c_x86_vendor = X86_VENDOR_INTEL,
diff --git a/arch/x86/kernel/cpu/sched.c b/arch/x86/kernel/cpu/sched.c
deleted file mode 100644
index a640ae5ad201..000000000000
--- a/arch/x86/kernel/cpu/sched.c
+++ /dev/null
@@ -1,55 +0,0 @@
-#include <linux/sched.h>
-#include <linux/math64.h>
-#include <linux/percpu.h>
-#include <linux/irqflags.h>
-
-#include <asm/cpufeature.h>
-#include <asm/processor.h>
-
-#ifdef CONFIG_SMP
-
-static DEFINE_PER_CPU(struct aperfmperf, old_perf_sched);
-
-static unsigned long scale_aperfmperf(void)
-{
- struct aperfmperf val, *old = &__get_cpu_var(old_perf_sched);
- unsigned long ratio, flags;
-
- local_irq_save(flags);
- get_aperfmperf(&val);
- local_irq_restore(flags);
-
- ratio = calc_aperfmperf_ratio(old, &val);
- *old = val;
-
- return ratio;
-}
-
-unsigned long arch_scale_freq_power(struct sched_domain *sd, int cpu)
-{
- /*
- * do aperf/mperf on the cpu level because it includes things
- * like turbo mode, which are relevant to full cores.
- */
- if (boot_cpu_has(X86_FEATURE_APERFMPERF))
- return scale_aperfmperf();
-
- /*
- * maybe have something cpufreq here
- */
-
- return default_scale_freq_power(sd, cpu);
-}
-
-unsigned long arch_scale_smt_power(struct sched_domain *sd, int cpu)
-{
- /*
- * aperf/mperf already includes the smt gain
- */
- if (boot_cpu_has(X86_FEATURE_APERFMPERF))
- return SCHED_LOAD_SCALE;
-
- return default_scale_smt_power(sd, cpu);
-}
-
-#endif
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S
index 111f6bbd8b38..69babd8c834f 100644
--- a/arch/x86/kernel/entry_64.S
+++ b/arch/x86/kernel/entry_64.S
@@ -1048,24 +1048,6 @@ apicinterrupt LOCAL_TIMER_VECTOR \
apicinterrupt X86_PLATFORM_IPI_VECTOR \
x86_platform_ipi smp_x86_platform_ipi
-#ifdef CONFIG_SMP
- ALIGN
- INTR_FRAME
-.irp idx,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, \
- 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
-.if NUM_INVALIDATE_TLB_VECTORS > \idx
-ENTRY(invalidate_interrupt\idx)
- pushq_cfi $~(INVALIDATE_TLB_VECTOR_START+\idx)
- jmp .Lcommon_invalidate_interrupt0
- CFI_ADJUST_CFA_OFFSET -8
-END(invalidate_interrupt\idx)
-.endif
-.endr
- CFI_ENDPROC
-apicinterrupt INVALIDATE_TLB_VECTOR_START, \
- invalidate_interrupt0, smp_invalidate_interrupt
-#endif
-
apicinterrupt THRESHOLD_APIC_VECTOR \
threshold_interrupt smp_threshold_interrupt
apicinterrupt THERMAL_APIC_VECTOR \
diff --git a/arch/x86/kernel/irqinit.c b/arch/x86/kernel/irqinit.c
index 252981afd6c4..6e03b0d69138 100644
--- a/arch/x86/kernel/irqinit.c
+++ b/arch/x86/kernel/irqinit.c
@@ -171,79 +171,6 @@ static void __init smp_intr_init(void)
*/
alloc_intr_gate(RESCHEDULE_VECTOR, reschedule_interrupt);
- /* IPIs for invalidation */
-#define ALLOC_INVTLB_VEC(NR) \
- alloc_intr_gate(INVALIDATE_TLB_VECTOR_START+NR, \
- invalidate_interrupt##NR)
-
- switch (NUM_INVALIDATE_TLB_VECTORS) {
- default:
- ALLOC_INVTLB_VEC(31);
- case 31:
- ALLOC_INVTLB_VEC(30);
- case 30:
- ALLOC_INVTLB_VEC(29);
- case 29:
- ALLOC_INVTLB_VEC(28);
- case 28:
- ALLOC_INVTLB_VEC(27);
- case 27:
- ALLOC_INVTLB_VEC(26);
- case 26:
- ALLOC_INVTLB_VEC(25);
- case 25:
- ALLOC_INVTLB_VEC(24);
- case 24:
- ALLOC_INVTLB_VEC(23);
- case 23:
- ALLOC_INVTLB_VEC(22);
- case 22:
- ALLOC_INVTLB_VEC(21);
- case 21:
- ALLOC_INVTLB_VEC(20);
- case 20:
- ALLOC_INVTLB_VEC(19);
- case 19:
- ALLOC_INVTLB_VEC(18);
- case 18:
- ALLOC_INVTLB_VEC(17);
- case 17:
- ALLOC_INVTLB_VEC(16);
- case 16:
- ALLOC_INVTLB_VEC(15);
- case 15:
- ALLOC_INVTLB_VEC(14);
- case 14:
- ALLOC_INVTLB_VEC(13);
- case 13:
- ALLOC_INVTLB_VEC(12);
- case 12:
- ALLOC_INVTLB_VEC(11);
- case 11:
- ALLOC_INVTLB_VEC(10);
- case 10:
- ALLOC_INVTLB_VEC(9);
- case 9:
- ALLOC_INVTLB_VEC(8);
- case 8:
- ALLOC_INVTLB_VEC(7);
- case 7:
- ALLOC_INVTLB_VEC(6);
- case 6:
- ALLOC_INVTLB_VEC(5);
- case 5:
- ALLOC_INVTLB_VEC(4);
- case 4:
- ALLOC_INVTLB_VEC(3);
- case 3:
- ALLOC_INVTLB_VEC(2);
- case 2:
- ALLOC_INVTLB_VEC(1);
- case 1:
- ALLOC_INVTLB_VEC(0);
- break;
- }
-
/* IPI for generic function call */
alloc_intr_gate(CALL_FUNCTION_VECTOR, call_function_interrupt);
diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c
index 5a98aa272184..5cdff0357746 100644
--- a/arch/x86/kernel/setup_percpu.c
+++ b/arch/x86/kernel/setup_percpu.c
@@ -21,7 +21,7 @@
#include <asm/cpu.h>
#include <asm/stackprotector.h>
-DEFINE_PER_CPU(int, cpu_number);
+DEFINE_PER_CPU_READ_MOSTLY(int, cpu_number);
EXPORT_PER_CPU_SYMBOL(cpu_number);
#ifdef CONFIG_X86_64
diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index c1a310fb8309..7c5a8c314c02 100644
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -106,17 +106,17 @@ int smp_num_siblings = 1;
EXPORT_SYMBOL(smp_num_siblings);
/* Last level cache ID of each logical CPU */
-DEFINE_PER_CPU(u16, cpu_llc_id) = BAD_APICID;
+DEFINE_PER_CPU_READ_MOSTLY(u16, cpu_llc_id) = BAD_APICID;
/* representing HT siblings of each logical CPU */
-DEFINE_PER_CPU(cpumask_var_t, cpu_sibling_map);
+DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_sibling_map);
EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
/* representing HT and core siblings of each logical CPU */
-DEFINE_PER_CPU(cpumask_var_t, cpu_core_map);
+DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_core_map);
EXPORT_PER_CPU_SYMBOL(cpu_core_map);
-DEFINE_PER_CPU(cpumask_var_t, cpu_llc_shared_map);
+DEFINE_PER_CPU_READ_MOSTLY(cpumask_var_t, cpu_llc_shared_map);
/* Per CPU bogomips and other parameters */
DEFINE_PER_CPU_SHARED_ALIGNED(struct cpuinfo_x86, cpu_info);
diff --git a/arch/x86/mm/pageattr.c b/arch/x86/mm/pageattr.c
index a718e0d23503..931930a96160 100644
--- a/arch/x86/mm/pageattr.c
+++ b/arch/x86/mm/pageattr.c
@@ -919,11 +919,13 @@ static int change_page_attr_set_clr(unsigned long *addr, int numpages,
/*
* On success we use clflush, when the CPU supports it to
- * avoid the wbindv. If the CPU does not support it and in the
- * error case we fall back to cpa_flush_all (which uses
- * wbindv):
+ * avoid the wbindv. If the CPU does not support it, in the
+ * error case, and during early boot (for EFI) we fall back
+ * to cpa_flush_all (which uses wbinvd):
*/
- if (!ret && cpu_has_clflush) {
+ if (early_boot_irqs_disabled)
+ __cpa_flush_all((void *)(long)cache);
+ else if (!ret && cpu_has_clflush) {
if (cpa.flags & (CPA_PAGES_ARRAY | CPA_ARRAY)) {
cpa_flush_array(addr, numpages, cache,
cpa.flags, pages);
diff --git a/arch/x86/mm/tlb.c b/arch/x86/mm/tlb.c
index 5e57e113b72c..613cd83e8c0c 100644
--- a/arch/x86/mm/tlb.c
+++ b/arch/x86/mm/tlb.c
@@ -12,6 +12,7 @@
#include <asm/cache.h>
#include <asm/apic.h>
#include <asm/uv/uv.h>
+#include <linux/debugfs.h>
DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate)
= { &init_mm, 0, };
@@ -27,33 +28,14 @@ DEFINE_PER_CPU_SHARED_ALIGNED(struct tlb_state, cpu_tlbstate)
*
* More scalable flush, from Andi Kleen
*
- * To avoid global state use 8 different call vectors.
- * Each CPU uses a specific vector to trigger flushes on other
- * CPUs. Depending on the received vector the target CPUs look into
- * the right array slot for the flush data.
- *
- * With more than 8 CPUs they are hashed to the 8 available
- * vectors. The limited global vector space forces us to this right now.
- * In future when interrupts are split into per CPU domains this could be
- * fixed, at the cost of triggering multiple IPIs in some cases.
+ * Implement flush IPI by CALL_FUNCTION_VECTOR, Alex Shi
*/
-union smp_flush_state {
- struct {
- struct mm_struct *flush_mm;
- unsigned long flush_va;
- raw_spinlock_t tlbstate_lock;
- DECLARE_BITMAP(flush_cpumask, NR_CPUS);
- };
- char pad[INTERNODE_CACHE_BYTES];
-} ____cacheline_internodealigned_in_smp;
-
-/* State is put into the per CPU data section, but padded
- to a full cache line because other CPUs can access it and we don't
- want false sharing in the per cpu data segment. */
-static union smp_flush_state flush_state[NUM_INVALIDATE_TLB_VECTORS];
-
-static DEFINE_PER_CPU_READ_MOSTLY(int, tlb_vector_offset);
+struct flush_tlb_info {
+ struct mm_struct *flush_mm;
+ unsigned long flush_start;
+ unsigned long flush_end;
+};
/*
* We cannot call mmdrop() because we are in interrupt context,
@@ -72,28 +54,25 @@ void leave_mm(int cpu)
EXPORT_SYMBOL_GPL(leave_mm);
/*
- *
* The flush IPI assumes that a thread switch happens in this order:
* [cpu0: the cpu that switches]
* 1) switch_mm() either 1a) or 1b)
* 1a) thread switch to a different mm
- * 1a1) cpu_clear(cpu, old_mm->cpu_vm_mask);
- * Stop ipi delivery for the old mm. This is not synchronized with
- * the other cpus, but smp_invalidate_interrupt ignore flush ipis
- * for the wrong mm, and in the worst case we perform a superfluous
- * tlb flush.
- * 1a2) set cpu mmu_state to TLBSTATE_OK
- * Now the smp_invalidate_interrupt won't call leave_mm if cpu0
- * was in lazy tlb mode.
- * 1a3) update cpu active_mm
+ * 1a1) set cpu_tlbstate to TLBSTATE_OK
+ * Now the tlb flush NMI handler flush_tlb_func won't call leave_mm
+ * if cpu0 was in lazy tlb mode.
+ * 1a2) update cpu active_mm
* Now cpu0 accepts tlb flushes for the new mm.
- * 1a4) cpu_set(cpu, new_mm->cpu_vm_mask);
+ * 1a3) cpu_set(cpu, new_mm->cpu_vm_mask);
* Now the other cpus will send tlb flush ipis.
* 1a4) change cr3.
+ * 1a5) cpu_clear(cpu, old_mm->cpu_vm_mask);
+ * Stop ipi delivery for the old mm. This is not synchronized with
+ * the other cpus, but flush_tlb_func ignore flush ipis for the wrong
+ * mm, and in the worst case we perform a superfluous tlb flush.
* 1b) thread switch without mm change
- * cpu active_mm is correct, cpu0 already handles
- * flush ipis.
- * 1b1) set cpu mmu_state to TLBSTATE_OK
+ * cpu active_mm is correct, cpu0 already handles flush ipis.
+ * 1b1) set cpu_tlbstate to TLBSTATE_OK
* 1b2) test_and_set the cpu bit in cpu_vm_mask.
* Atomically set the bit [other cpus will start sending flush ipis],
* and test the bit.
@@ -106,174 +85,62 @@ EXPORT_SYMBOL_GPL(leave_mm);
* runs in kernel space, the cpu could load tlb entries for user space
* pages.
*
- * The good news is that cpu mmu_state is local to each cpu, no
+ * The good news is that cpu_tlbstate is local to each cpu, no
* write/read ordering problems.
*/
/*
- * TLB flush IPI:
- *
+ * TLB flush funcation:
* 1) Flush the tlb entries if the cpu uses the mm that's being flushed.
* 2) Leave the mm if we are in the lazy tlb mode.
- *
- * Interrupts are disabled.
- */
-
-/*
- * FIXME: use of asmlinkage is not consistent. On x86_64 it's noop
- * but still used for documentation purpose but the usage is slightly
- * inconsistent. On x86_32, asmlinkage is regparm(0) but interrupt
- * entry calls in with the first parameter in %eax. Maybe define
- * intrlinkage?
*/
-#ifdef CONFIG_X86_64
-asmlinkage
-#endif
-void smp_invalidate_interrupt(struct pt_regs *regs)
+static void flush_tlb_func(void *info)
{
- unsigned int cpu;
- unsigned int sender;
- union smp_flush_state *f;
-
- cpu = smp_processor_id();
- /*
- * orig_rax contains the negated interrupt vector.
- * Use that to determine where the sender put the data.
- */
- sender = ~regs->orig_ax - INVALIDATE_TLB_VECTOR_START;
- f = &flush_state[sender];
-
- if (!cpumask_test_cpu(cpu, to_cpumask(f->flush_cpumask)))
- goto out;
- /*
- * This was a BUG() but until someone can quote me the
- * line from the intel manual that guarantees an IPI to
- * multiple CPUs is retried _only_ on the erroring CPUs
- * its staying as a return
- *
- * BUG();
- */
-
- if (f->flush_mm == this_cpu_read(cpu_tlbstate.active_mm)) {
- if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
- if (f->flush_va == TLB_FLUSH_ALL)
- local_flush_tlb();
- else
- __flush_tlb_one(f->flush_va);
- } else
- leave_mm(cpu);
- }
-out:
- ack_APIC_irq();
- smp_mb__before_clear_bit();
- cpumask_clear_cpu(cpu, to_cpumask(f->flush_cpumask));
- smp_mb__after_clear_bit();
- inc_irq_stat(irq_tlb_count);
-}
+ struct flush_tlb_info *f = info;
-static void flush_tlb_others_ipi(const struct cpumask *cpumask,
- struct mm_struct *mm, unsigned long va)
-{
- unsigned int sender;
- union smp_flush_state *f;
-
- /* Caller has disabled preemption */
- sender = this_cpu_read(tlb_vector_offset);
- f = &flush_state[sender];
-
- if (nr_cpu_ids > NUM_INVALIDATE_TLB_VECTORS)
- raw_spin_lock(&f->tlbstate_lock);
-
- f->flush_mm = mm;
- f->flush_va = va;
- if (cpumask_andnot(to_cpumask(f->flush_cpumask), cpumask, cpumask_of(smp_processor_id()))) {
- /*
- * We have to send the IPI only to
- * CPUs affected.
- */
- apic->send_IPI_mask(to_cpumask(f->flush_cpumask),
- INVALIDATE_TLB_VECTOR_START + sender);
-
- while (!cpumask_empty(to_cpumask(f->flush_cpumask)))
- cpu_relax();
- }
+ if (f->flush_mm != this_cpu_read(cpu_tlbstate.active_mm))
+ return;
+
+ if (this_cpu_read(cpu_tlbstate.state) == TLBSTATE_OK) {
+ if (f->flush_end == TLB_FLUSH_ALL || !cpu_has_invlpg)
+ local_flush_tlb();
+ else if (!f->flush_end)
+ __flush_tlb_single(f->flush_start);
+ else {
+ unsigned long addr;
+ addr = f->flush_start;
+ while (addr < f->flush_end) {
+ __flush_tlb_single(addr);
+ addr += PAGE_SIZE;
+ }
+ }
+ } else
+ leave_mm(smp_processor_id());
- f->flush_mm = NULL;
- f->flush_va = 0;
- if (nr_cpu_ids > NUM_INVALIDATE_TLB_VECTORS)
- raw_spin_unlock(&f->tlbstate_lock);
}
void native_flush_tlb_others(const struct cpumask *cpumask,
- struct mm_struct *mm, unsigned long va)
+ struct mm_struct *mm, unsigned long start,
+ unsigned long end)
{
+ struct flush_tlb_info info;
+ info.flush_mm = mm;
+ info.flush_start = start;
+ info.flush_end = end;
+
if (is_uv_system()) {
unsigned int cpu;
cpu = smp_processor_id();
- cpumask = uv_flush_tlb_others(cpumask, mm, va, cpu);
+ cpumask = uv_flush_tlb_others(cpumask, mm, start, end, cpu);
if (cpumask)
- flush_tlb_others_ipi(cpumask, mm, va);
+ smp_call_function_many(cpumask, flush_tlb_func,
+ &info, 1);
return;
}
- flush_tlb_others_ipi(cpumask, mm, va);
+ smp_call_function_many(cpumask, flush_tlb_func, &info, 1);
}
-static void __cpuinit calculate_tlb_offset(void)
-{
- int cpu, node, nr_node_vecs, idx = 0;
- /*
- * we are changing tlb_vector_offset for each CPU in runtime, but this
- * will not cause inconsistency, as the write is atomic under X86. we
- * might see more lock contentions in a short time, but after all CPU's
- * tlb_vector_offset are changed, everything should go normal
- *
- * Note: if NUM_INVALIDATE_TLB_VECTORS % nr_online_nodes !=0, we might
- * waste some vectors.
- **/
- if (nr_online_nodes > NUM_INVALIDATE_TLB_VECTORS)
- nr_node_vecs = 1;
- else
- nr_node_vecs = NUM_INVALIDATE_TLB_VECTORS/nr_online_nodes;
-
- for_each_online_node(node) {
- int node_offset = (idx % NUM_INVALIDATE_TLB_VECTORS) *
- nr_node_vecs;
- int cpu_offset = 0;
- for_each_cpu(cpu, cpumask_of_node(node)) {
- per_cpu(tlb_vector_offset, cpu) = node_offset +
- cpu_offset;
- cpu_offset++;
- cpu_offset = cpu_offset % nr_node_vecs;
- }
- idx++;
- }
-}
-
-static int __cpuinit tlb_cpuhp_notify(struct notifier_block *n,
- unsigned long action, void *hcpu)
-{
- switch (action & 0xf) {
- case CPU_ONLINE:
- case CPU_DEAD:
- calculate_tlb_offset();
- }
- return NOTIFY_OK;
-}
-
-static int __cpuinit init_smp_flush(void)
-{
- int i;
-
- for (i = 0; i < ARRAY_SIZE(flush_state); i++)
- raw_spin_lock_init(&flush_state[i].tlbstate_lock);
-
- calculate_tlb_offset();
- hotcpu_notifier(tlb_cpuhp_notify, 0);
- return 0;
-}
-core_initcall(init_smp_flush);
-
void flush_tlb_current_task(void)
{
struct mm_struct *mm = current->mm;
@@ -282,27 +149,91 @@ void flush_tlb_current_task(void)
local_flush_tlb();
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
- flush_tlb_others(mm_cpumask(mm), mm, TLB_FLUSH_ALL);
+ flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
preempt_enable();
}
-void flush_tlb_mm(struct mm_struct *mm)
+/*
+ * It can find out the THP large page, or
+ * HUGETLB page in tlb_flush when THP disabled
+ */
+static inline unsigned long has_large_page(struct mm_struct *mm,
+ unsigned long start, unsigned long end)
+{
+ pgd_t *pgd;
+ pud_t *pud;
+ pmd_t *pmd;
+ unsigned long addr = ALIGN(start, HPAGE_SIZE);
+ for (; addr < end; addr += HPAGE_SIZE) {
+ pgd = pgd_offset(mm, addr);
+ if (likely(!pgd_none(*pgd))) {
+ pud = pud_offset(pgd, addr);
+ if (likely(!pud_none(*pud))) {
+ pmd = pmd_offset(pud, addr);
+ if (likely(!pmd_none(*pmd)))
+ if (pmd_large(*pmd))
+ return addr;
+ }
+ }
+ }
+ return 0;
+}
+
+void flush_tlb_mm_range(struct mm_struct *mm, unsigned long start,
+ unsigned long end, unsigned long vmflag)
{
+ unsigned long addr;
+ unsigned act_entries, tlb_entries = 0;
+
preempt_disable();
+ if (current->active_mm != mm)
+ goto flush_all;
- if (current->active_mm == mm) {
- if (current->mm)
+ if (!current->mm) {
+ leave_mm(smp_processor_id());
+ goto flush_all;
+ }
+
+ if (end == TLB_FLUSH_ALL || tlb_flushall_shift == -1
+ || vmflag == VM_HUGETLB) {
+ local_flush_tlb();
+ goto flush_all;
+ }
+
+ /* In modern CPU, last level tlb used for both data/ins */
+ if (vmflag & VM_EXEC)
+ tlb_entries = tlb_lli_4k[ENTRIES];
+ else
+ tlb_entries = tlb_lld_4k[ENTRIES];
+ /* Assume all of TLB entries was occupied by this task */
+ act_entries = mm->total_vm > tlb_entries ? tlb_entries : mm->total_vm;
+
+ /* tlb_flushall_shift is on balance point, details in commit log */
+ if ((end - start) >> PAGE_SHIFT > act_entries >> tlb_flushall_shift)
+ local_flush_tlb();
+ else {
+ if (has_large_page(mm, start, end)) {
local_flush_tlb();
- else
- leave_mm(smp_processor_id());
+ goto flush_all;
+ }
+ /* flush range by one by one 'invlpg' */
+ for (addr = start; addr < end; addr += PAGE_SIZE)
+ __flush_tlb_single(addr);
+
+ if (cpumask_any_but(mm_cpumask(mm),
+ smp_processor_id()) < nr_cpu_ids)
+ flush_tlb_others(mm_cpumask(mm), mm, start, end);
+ preempt_enable();
+ return;
}
- if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
- flush_tlb_others(mm_cpumask(mm), mm, TLB_FLUSH_ALL);
+flush_all:
+ if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
+ flush_tlb_others(mm_cpumask(mm), mm, 0UL, TLB_FLUSH_ALL);
preempt_enable();
}
-void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
+void flush_tlb_page(struct vm_area_struct *vma, unsigned long start)
{
struct mm_struct *mm = vma->vm_mm;
@@ -310,13 +241,13 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long va)
if (current->active_mm == mm) {
if (current->mm)
- __flush_tlb_one(va);
+ __flush_tlb_one(start);
else
leave_mm(smp_processor_id());
}
if (cpumask_any_but(mm_cpumask(mm), smp_processor_id()) < nr_cpu_ids)
- flush_tlb_others(mm_cpumask(mm), mm, va);
+ flush_tlb_others(mm_cpumask(mm), mm, start, 0UL);
preempt_enable();
}
@@ -332,3 +263,83 @@ void flush_tlb_all(void)
{
on_each_cpu(do_flush_tlb_all, NULL, 1);
}
+
+static void do_kernel_range_flush(void *info)
+{
+ struct flush_tlb_info *f = info;
+ unsigned long addr;
+
+ /* flush range by one by one 'invlpg' */
+ for (addr = f->flush_start; addr < f->flush_end; addr += PAGE_SIZE)
+ __flush_tlb_single(addr);
+}
+
+void flush_tlb_kernel_range(unsigned long start, unsigned long end)
+{
+ unsigned act_entries;
+ struct flush_tlb_info info;
+
+ /* In modern CPU, last level tlb used for both data/ins */
+ act_entries = tlb_lld_4k[ENTRIES];
+
+ /* Balance as user space task's flush, a bit conservative */
+ if (end == TLB_FLUSH_ALL || tlb_flushall_shift == -1 ||
+ (end - start) >> PAGE_SHIFT > act_entries >> tlb_flushall_shift)
+
+ on_each_cpu(do_flush_tlb_all, NULL, 1);
+ else {
+ info.flush_start = start;
+ info.flush_end = end;
+ on_each_cpu(do_kernel_range_flush, &info, 1);
+ }
+}
+
+#ifdef CONFIG_DEBUG_TLBFLUSH
+static ssize_t tlbflush_read_file(struct file *file, char __user *user_buf,
+ size_t count, loff_t *ppos)
+{
+ char buf[32];
+ unsigned int len;
+
+ len = sprintf(buf, "%hd\n", tlb_flushall_shift);
+ return simple_read_from_buffer(user_buf, count, ppos, buf, len);
+}
+
+static ssize_t tlbflush_write_file(struct file *file,
+ const char __user *user_buf, size_t count, loff_t *ppos)
+{
+ char buf[32];
+ ssize_t len;
+ s8 shift;
+
+ len = min(count, sizeof(buf) - 1);
+ if (copy_from_user(buf, user_buf, len))
+ return -EFAULT;
+
+ buf[len] = '\0';
+ if (kstrtos8(buf, 0, &shift))
+ return -EINVAL;
+
+ if (shift > 64)
+ return -EINVAL;
+
+ tlb_flushall_shift = shift;
+ return count;
+}
+
+static const struct file_operations fops_tlbflush = {
+ .read = tlbflush_read_file,
+ .write = tlbflush_write_file,
+ .llseek = default_llseek,
+};
+
+static int __cpuinit create_tlb_flushall_shift(void)
+{
+ if (cpu_has_invlpg) {
+ debugfs_create_file("tlb_flushall_shift", S_IRUSR | S_IWUSR,
+ arch_debugfs_dir, NULL, &fops_tlbflush);
+ }
+ return 0;
+}
+late_initcall(create_tlb_flushall_shift);
+#endif
diff --git a/arch/x86/platform/efi/efi.c b/arch/x86/platform/efi/efi.c
index 92660edaa1e7..2dc29f51e75a 100644
--- a/arch/x86/platform/efi/efi.c
+++ b/arch/x86/platform/efi/efi.c
@@ -234,22 +234,7 @@ static efi_status_t __init phys_efi_set_virtual_address_map(
return status;
}
-static efi_status_t __init phys_efi_get_time(efi_time_t *tm,
- efi_time_cap_t *tc)
-{
- unsigned long flags;
- efi_status_t status;
-
- spin_lock_irqsave(&rtc_lock, flags);
- efi_call_phys_prelog();
- status = efi_call_phys2(efi_phys.get_time, virt_to_phys(tm),
- virt_to_phys(tc));
- efi_call_phys_epilog();
- spin_unlock_irqrestore(&rtc_lock, flags);
- return status;
-}
-
-int efi_set_rtc_mmss(unsigned long nowtime)
+static int efi_set_rtc_mmss(unsigned long nowtime)
{
int real_seconds, real_minutes;
efi_status_t status;
@@ -278,7 +263,7 @@ int efi_set_rtc_mmss(unsigned long nowtime)
return 0;
}
-unsigned long efi_get_time(void)
+static unsigned long efi_get_time(void)
{
efi_status_t status;
efi_time_t eft;
@@ -621,18 +606,13 @@ static int __init efi_runtime_init(void)
}
/*
* We will only need *early* access to the following
- * two EFI runtime services before set_virtual_address_map
+ * EFI runtime service before set_virtual_address_map
* is invoked.
*/
- efi_phys.get_time = (efi_get_time_t *)runtime->get_time;
efi_phys.set_virtual_address_map =
(efi_set_virtual_address_map_t *)
runtime->set_virtual_address_map;
- /*
- * Make efi_get_time can be called before entering
- * virtual mode.
- */
- efi.get_time = phys_efi_get_time;
+
early_iounmap(runtime, sizeof(efi_runtime_services_t));
return 0;
@@ -720,12 +700,10 @@ void __init efi_init(void)
efi_enabled = 0;
return;
}
-#ifdef CONFIG_X86_32
if (efi_native) {
x86_platform.get_wallclock = efi_get_time;
x86_platform.set_wallclock = efi_set_rtc_mmss;
}
-#endif
#if EFI_DEBUG
print_efi_memmap();
diff --git a/arch/x86/platform/uv/tlb_uv.c b/arch/x86/platform/uv/tlb_uv.c
index 71b5d5a07d7b..b8b3a37c80cd 100644
--- a/arch/x86/platform/uv/tlb_uv.c
+++ b/arch/x86/platform/uv/tlb_uv.c
@@ -1055,8 +1055,8 @@ static int set_distrib_bits(struct cpumask *flush_mask, struct bau_control *bcp,
* done. The returned pointer is valid till preemption is re-enabled.
*/
const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
- struct mm_struct *mm, unsigned long va,
- unsigned int cpu)
+ struct mm_struct *mm, unsigned long start,
+ unsigned end, unsigned int cpu)
{
int locals = 0;
int remotes = 0;
@@ -1113,7 +1113,7 @@ const struct cpumask *uv_flush_tlb_others(const struct cpumask *cpumask,
record_send_statistics(stat, locals, hubs, remotes, bau_desc);
- bau_desc->payload.address = va;
+ bau_desc->payload.address = start;
bau_desc->payload.sending_cpu = cpu;
/*
* uv_flush_send_and_wait returns 0 if all cpu's were messaged,
diff --git a/arch/x86/xen/mmu.c b/arch/x86/xen/mmu.c
index 27336dfcda8e..b65a76133f4f 100644
--- a/arch/x86/xen/mmu.c
+++ b/arch/x86/xen/mmu.c
@@ -1256,7 +1256,8 @@ static void xen_flush_tlb_single(unsigned long addr)
}
static void xen_flush_tlb_others(const struct cpumask *cpus,
- struct mm_struct *mm, unsigned long va)
+ struct mm_struct *mm, unsigned long start,
+ unsigned long end)
{
struct {
struct mmuext_op op;
@@ -1268,7 +1269,7 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
} *args;
struct multicall_space mcs;
- trace_xen_mmu_flush_tlb_others(cpus, mm, va);
+ trace_xen_mmu_flush_tlb_others(cpus, mm, start, end);
if (cpumask_empty(cpus))
return; /* nothing to do */
@@ -1281,11 +1282,10 @@ static void xen_flush_tlb_others(const struct cpumask *cpus,
cpumask_and(to_cpumask(args->mask), cpus, cpu_online_mask);
cpumask_clear_cpu(smp_processor_id(), to_cpumask(args->mask));
- if (va == TLB_FLUSH_ALL) {
- args->op.cmd = MMUEXT_TLB_FLUSH_MULTI;
- } else {
+ args->op.cmd = MMUEXT_TLB_FLUSH_MULTI;
+ if (start != TLB_FLUSH_ALL && (end - start) <= PAGE_SIZE) {
args->op.cmd = MMUEXT_INVLPG_MULTI;
- args->op.arg1.linear_addr = va;
+ args->op.arg1.linear_addr = start;
}
MULTI_mmuext_op(mcs.mc, &args->op, 1, NULL, DOMID_SELF);