/*
 * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com>
 * Copyright 2002 Andi Kleen, SuSE Labs.
 * Subject to the GNU Public License v2.
 *
 * Functions to copy from and to user space.
 */

#include <linux/linkage.h>
#include <asm/dwarf2.h>

#define FIX_ALIGNMENT 1

#include <asm/current.h>
#include <asm/asm-offsets.h>
#include <asm/thread_info.h>

	.macro ALIGN_DESTINATION
#ifdef FIX_ALIGNMENT
	/* check for bad alignment of destination */
	movl %edi,%ecx
	andl $7,%ecx
	jz 102f				/* already aligned */
	subl $8,%ecx
	negl %ecx
	subl %ecx,%edx
100:	movb (%rsi),%al
101:	movb %al,(%rdi)
	incq %rsi
	incq %rdi
	decl %ecx
	jnz 100b
102:
	.section .fixup,"ax"
103:	addl %ecx,%edx			/* ecx is zerorest also */
	jmp copy_user_handle_tail
	.previous

	.section __ex_table,"a"
	.align 8
	.quad 100b,103b
	.quad 101b,103b
	.previous
#endif
	.endm

/*
 * copy_user_nocache - Uncached memory copy with exception handling
 * This will force destination/source out of cache for more performance.
 */
ENTRY(__copy_user_nocache)
	CFI_STARTPROC
	cmpl $8,%edx
	jb 20f		/* less then 8 bytes, go to byte copy loop */
	ALIGN_DESTINATION
	movl %edx,%ecx
	andl $63,%edx
	shrl $6,%ecx
	jz 17f
1:	movq (%rsi),%r8
2:	movq 1*8(%rsi),%r9
3:	movq 2*8(%rsi),%r10
4:	movq 3*8(%rsi),%r11
5:	movnti %r8,(%rdi)
6:	movnti %r9,1*8(%rdi)
7:	movnti %r10,2*8(%rdi)
8:	movnti %r11,3*8(%rdi)
9:	movq 4*8(%rsi),%r8
10:	movq 5*8(%rsi),%r9
11:	movq 6*8(%rsi),%r10
12:	movq 7*8(%rsi),%r11
13:	movnti %r8,4*8(%rdi)
14:	movnti %r9,5*8(%rdi)
15:	movnti %r10,6*8(%rdi)
16:	movnti %r11,7*8(%rdi)
	leaq 64(%rsi),%rsi
	leaq 64(%rdi),%rdi
	decl %ecx
	jnz 1b
17:	movl %edx,%ecx
	andl $7,%edx
	shrl $3,%ecx
	jz 20f
18:	movq (%rsi),%r8
19:	movnti %r8,(%rdi)
	leaq 8(%rsi),%rsi
	leaq 8(%rdi),%rdi
	decl %ecx
	jnz 18b
20:	andl %edx,%edx
	jz 23f
	movl %edx,%ecx
21:	movb (%rsi),%al
22:	movb %al,(%rdi)
	incq %rsi
	incq %rdi
	decl %ecx
	jnz 21b
23:	xorl %eax,%eax
	sfence
	ret

	.section .fixup,"ax"
30:	shll $6,%ecx
	addl %ecx,%edx
	jmp 60f
40:	lea (%rdx,%rcx,8),%rdx
	jmp 60f
50:	movl %ecx,%edx
60:	sfence
	jmp copy_user_handle_tail
	.previous

	.section __ex_table,"a"
	.quad 1b,30b
	.quad 2b,30b
	.quad 3b,30b
	.quad 4b,30b
	.quad 5b,30b
	.quad 6b,30b
	.quad 7b,30b
	.quad 8b,30b
	.quad 9b,30b
	.quad 10b,30b
	.quad 11b,30b
	.quad 12b,30b
	.quad 13b,30b
	.quad 14b,30b
	.quad 15b,30b
	.quad 16b,30b
	.quad 18b,40b
	.quad 19b,40b
	.quad 21b,50b
	.quad 22b,50b
	.previous
	CFI_ENDPROC
ENDPROC(__copy_user_nocache)
og/security?id=9deb343189b3cf45e84dd08480f330575ffe2004'>security</a>/<a href='/drm-tip/log/security/security.c?id=9deb343189b3cf45e84dd08480f330575ffe2004'>security.c</a></div><div class='content'><table class='list nowrap'><tr class='nohover'><th class='left'>Age</th><th class='left'>Commit message (<a href='/drm-tip/log/security/security.c?id=9deb343189b3cf45e84dd08480f330575ffe2004&amp;showmsg=1'>Expand</a>)</th><th class='left'>Author</th><th class='left'>Files</th><th class='left'>Lines</th></tr>
<tr><td><span title='2010-03-01 09:36:31 +1100'>2010-03-01</span></td><td><a href='/drm-tip/commit/security/security.c?id=b4ccebdd37ff70d349321a198f416ba737a5e833'>Merge branch 'next' into for-linus</a></td><td>James Morris</td><td>1</td><td><span class='deletions'>-21</span>/<span class='insertions'>+28</span></td></tr>
<tr><td><span title='2010-02-24 08:11:02 +1100'>2010-02-24</span></td><td><a href='/drm-tip/commit/security/security.c?id=189b3b1c89761054fee3438f063d7f257306e2d8'>Security: add static to security_ops and default_security_ops variable</a></td><td>wzt.wzt@gmail.com</td><td>1</td><td><span class='deletions'>-2</span>/<span class='insertions'>+9</span></td></tr>
<tr><td><span title='2010-02-07 03:07:29 -0500'>2010-02-07</span></td><td><a href='/drm-tip/commit/security/security.c?id=89068c576bf324ef6fbd50dfc745148f7def202c'>Take ima_file_free() to proper place.</a></td><td>Al Viro</td><td>1</td><td><span class='deletions'>-2</span>/<span class='insertions'>+0</span></td></tr>
<tr><td><span title='2010-02-04 14:20:12 +1100'>2010-02-04</span></td><td><a href='/drm-tip/commit/security/security.c?id=002345925e6c45861f60db6f4fc6236713fd8847'>syslog: distinguish between /proc/kmsg and syscalls</a></td><td>Kees Cook</td><td>1</td><td><span class='deletions'>-2</span>/<span class='insertions'>+2</span></td></tr>
<tr><td><span title='2010-01-15 08:23:57 +1100'>2010-01-15</span></td><td><a href='/drm-tip/commit/security/security.c?id=8d9525048c74786205b99f3fcd05a839721edfb7'>security: correct error returns for get/set security with private inodes</a></td><td>James Morris</td><td>1</td><td><span class='deletions'>-2</span>/<span class='insertions'>+2</span></td></tr>
<tr><td><span title='2009-12-08 14:58:00 +1100'>2009-12-08</span></td><td><a href='/drm-tip/commit/security/security.c?id=5d0901a3a0c39c97ca504f73d24030f63cfc9fa2'>LSM: Rename security_path_ functions argument names.</a></td><td>Tetsuo Handa</td><td>1</td><td><span class='deletions'>-15</span>/<span class='insertions'>+15</span></td></tr>
<tr><td><span title='2009-11-10 09:33:46 +1100'>2009-11-10</span></td><td><a href='/drm-tip/commit/security/security.c?id=dd8dbf2e6880e30c00b18600c962d0cb5a03c555'>security: report the module name to security_module_request</a></td><td>Eric Paris</td><td>1</td><td><span class='deletions'>-2</span>/<span class='insertions'>+2</span></td></tr>
<tr><td><span title='2009-11-09 08:40:07 +1100'>2009-11-09</span></td><td><a href='/drm-tip/commit/security/security.c?id=6e65f92ff0d6f18580737321718d09035085a3fb'>Config option to set a default LSM</a></td><td>John Johansen</td><td>1</td><td><span class='deletions'>-3</span>/<span class='insertions'>+6</span></td></tr>
<tr><td><span title='2009-10-25 12:22:48 +0800'>2009-10-25</span></td><td><a href='/drm-tip/commit/security/security.c?id=6c21a7fb492bf7e2c4985937082ce58ddeca84bd'>LSM: imbed ima calls in the security hooks</a></td><td>Mimi Zohar</td><td>1</td><td><span class='deletions'>-3</span>/<span class='insertions'>+25</span></td></tr>
<tr><td><span title='2009-10-12 10:56:02 +1100'>2009-10-12</span></td><td><a href='/drm-tip/commit/security/security.c?id=8b8efb44033c7e86b3dc76f825c693ec92ae30e9'>LSM: Add security_path_chroot().</a></td><td>Tetsuo Handa</td><td>1</td><td><span class='deletions'>-0</span>/<span class='insertions'>+5</span></td></tr>
<tr><td><span title='2009-10-12 10:56:00 +1100'>2009-10-12</span></td><td><a href='/drm-tip/commit/security/security.c?id=89eda06837094ce9f34fae269b8773fcfd70f046'>LSM: Add security_path_chmod() and security_path_chown().</a></td><td>Tetsuo Handa</td><td>1</td><td><span class='deletions'>-0</span>/<span class='insertions'>+15</span></td></tr>
<tr><td><span title='2009-09-10 10:11:24 +1000'>2009-09-10</span></td><td><a href='/drm-tip/commit/security/security.c?id=1ee65e37e904b959c24404139f5752edc66319d5'>LSM/SELinux: inode_{get,set,notify}secctx hooks to access LSM security contex...</a></td><td>David P. Quigley</td><td>1</td><td><span class='deletions'>-0</span>/<span class='insertions'>+18</span></td></tr>
<tr><td><span title='2009-09-02 21:29:22 +1000'>2009-09-02</span></td><td><a href='/drm-tip/commit/security/security.c?id=ee18d64c1f632043a02e6f5ba5e045bb26a5465f'>KEYS: Add a keyctl to install a process's session keyring on its parent [try #6]</a></td><td>David Howells</td><td>1</td><td><span class='deletions'>-0</span>/<span class='insertions'>+17</span></td></tr>
<tr><td><span title='2009-09-01 08:29:48 +1000'>2009-09-01</span></td><td><a href='/drm-tip/commit/security/security.c?id=2b980dbd77d229eb60588802162c9659726b11f4'>lsm: Add hooks to the TUN driver</a></td><td>Paul Moore</td><td>1</td><td><span class='deletions'>-0</span>/<span class='insertions'>+18</span></td></tr>
<tr><td><span title='2009-08-14 11:18:37 +1000'>2009-08-14</span></td><td><a href='/drm-tip/commit/security/security.c?id=9188499cdb117d86a1ea6b04374095b098d56936'>security: introducing security_request_module</a></td><td>Eric Paris</td><td>1</td><td><span class='deletions'>-0</span>/<span class='insertions'>+5</span></td></tr>
<tr><td><span title='2009-06-25 00:18:05 +1000'>2009-06-25</span></td><td><a href='/drm-tip/commit/security/security.c?id=9e48858f7d36a6a3849f1d1b40c3bf5624b4ee7c'>security: rename ptrace_may_access =&gt; ptrace_access_check</a></td><td>Ingo Molnar</td><td>1</td><td><span class='deletions'>-2</span>/<span class='insertions'>+2</span></td></tr>
<tr><td><span title='2009-06-04 12:07:48 +1000'>2009-06-04</span></td><td><a href='/drm-tip/commit/security/security.c?id=e0a94c2a63f2644826069044649669b5e7ca75d3'>security: use mmap_min_addr indepedently of security models</a></td><td>Christoph Lameter</td><td>1</td><td><span class='deletions'>-3</span>/<span class='insertions'>+0</span></td></tr>
<tr><td><span title='2009-04-03 16:42:40 +0100'>2009-04-03</span></td><td><a href='/drm-tip/commit/security/security.c?id=800a964787faef3509d194fa33268628c3d1daa9'>CacheFiles: Export things for CacheFiles</a></td><td>David Howells</td><td>1</td><td><span class='deletions'>-0</span>/<span class='insertions'>+2</span></td></tr>
<tr><td><span title='2009-03-28 15:01:37 +1100'>2009-03-28</span></td><td><a href='/drm-tip/commit/security/security.c?id=8651d5c0b1f874c5b8307ae2b858bc40f9f02482'>lsm: Remove the socket_post_accept() hook</a></td><td>Paul Moore</td><td>1</td><td><span class='deletions'>-5</span>/<span class='insertions'>+0</span></td></tr>
<tr><td><span title='2009-01-07 09:58:22 +1100'>2009-01-07</span></td><td><a href='/drm-tip/commit/security/security.c?id=ac8cc0fa5395fe2278e305a4cbed48e90d88d878'>Merge branch 'next' into for-linus</a></td><td>James Morris</td><td>1</td><td><span class='deletions'>-4</span>/<span class='insertions'>+22</span></td></tr>
<tr><td><span title='2009-01-07 09:38:48 +1100'>2009-01-07</span></td><td><a href='/drm-tip/commit/security/security.c?id=3699c53c485bf0168e6500d0ed18bf931584dd7c'>CRED: Fix regression in cap_capable() as shown up by sys_faccessat() [ver #3]</a></td><td>David Howells</td><td>1</td><td><span class='deletions'>-4</span>/<span class='insertions'>+22</span></td></tr>
<tr><td><span title='2009-01-07 09:21:54 +1100'>2009-01-07</span></td><td><a href='/drm-tip/commit/security/security.c?id=29881c4502ba05f46bc12ae8053d4e08d7e2615c'>Revert "CRED: Fix regression in cap_capable() as shown up by sys_faccessat() ...</a></td><td>James Morris</td><td>1</td><td><span class='deletions'>-21</span>/<span class='insertions'>+4</span></td></tr>
<tr><td><span title='2009-01-05 11:17:04 +1100'>2009-01-05</span></td><td><a href='/drm-tip/commit/security/security.c?id=14eaddc967b16017d4a1a24d2be6c28ecbe06ed8'>CRED: Fix regression in cap_capable() as shown up by sys_faccessat() [ver #2]</a></td><td>David Howells</td><td>1</td><td><span class='deletions'>-4</span>/<span class='insertions'>+21</span></td></tr>
<tr><td><span title='2008-12-31 18:07:37 -0500'>2008-12-31</span></td><td><a href='/drm-tip/commit/security/security.c?id=be6d3e56a6b9b3a4ee44a0685e39e595073c6f0d'>introduce new LSM hooks where vfsmount is available.</a></td><td>Kentaro Takeda</td><td>1</td><td><span class='deletions'>-0</span>/<span class='insertions'>+66</span></td></tr>
<tr><td><span title='2008-12-20 09:02:39 +1100'>2008-12-20</span></td><td><a href='/drm-tip/commit/security/security.c?id=12204e24b1330428c3062faee10a0d80b8a5cb61'>security: pass mount flags to security_sb_kern_mount()</a></td><td>James Morris</td><td>1</td><td><span class='deletions'>-2</span>/<span class='insertions'>+2</span></td></tr>
<tr><td><span title='2008-11-25 06:33:17 +0530'>2008-11-25</span></td><td><a href='/drm-tip/commit/security/security.c?id=200036ca9b3f0b2250912142552ce56682190f95'>CRED: fix sparse warnings</a></td><td>Hannes Eder</td><td>1</td><td><span class='deletions'>-3</span>/<span class='insertions'>+3</span></td></tr>
<tr><td><span title='2008-11-14 10:39:28 +1100'>2008-11-14</span></td><td><a href='/drm-tip/commit/security/security.c?id=3a3b7ce9336952ea7b9564d976d068a238976c9d'>CRED: Allow kernel services to override LSM settings for task actions</a></td><td>David Howells</td><td>1</td><td><span class='deletions'>-0</span>/<span class='insertions'>+10</span></td></tr>
<tr><td><span title='2008-11-14 10:39:24 +1100'>2008-11-14</span></td><td><a href='/drm-tip/commit/security/security.c?id=a6f76f23d297f70e2a6b3ec607f7aeeea9e37e8d'>CRED: Make execve() take advantage of copy-on-write credentials</a></td><td>David Howells</td><td>1</td><td><span class='deletions'>-18</span>/<span class='insertions'>+8</span></td></tr>
<tr><td><span title='2008-11-14 10:39:23 +1100'>2008-11-14</span></td><td><a href='/drm-tip/commit/security/security.c?id=d84f4f992cbd76e8f39c488cf0c5d123843923b1'>CRED: Inaugurate COW credentials</a></td><td>David Howells</td><td>1</td><td><span class='deletions'>-31</span>/<span class='insertions'>+27</span></td></tr>
<tr><td><span title='2008-11-14 10:39:22 +1100'>2008-11-14</span></td><td><a href='/drm-tip/commit/security/security.c?id=745ca2475a6ac596e3d8d37c2759c0fbe2586227'>CRED: Pass credentials through dentry_open()</a></td><td>David Howells</td><td>1</td><td><span class='deletions'>-2</span>/<span class='insertions'>+2</span></td></tr>
<tr><td><span title='2008-11-14 10:39:17 +1100'>2008-11-14</span></td><td><a href='/drm-tip/commit/security/security.c?id=f1752eec6145c97163dbce62d17cf5d928e28a27'>CRED: Detach the credentials from task_struct</a></td><td>David Howells</td><td>1</td><td><span class='deletions'>-4</span>/<span class='insertions'>+4</span></td></tr>
<tr><td><span title='2008-11-14 10:39:15 +1100'>2008-11-14</span></td><td><a href='/drm-tip/commit/security/security.c?id=15a2460ed0af7538ca8e6c610fe607a2cd9da142'>CRED: Constify the kernel_cap_t arguments to the capset LSM hooks</a></td><td>David Howells</td><td>1</td><td><span class='deletions'>-6</span>/<span class='insertions'>+6</span></td></tr>
<tr><td><span title='2008-11-14 10:39:14 +1100'>2008-11-14</span></td><td><a href='/drm-tip/commit/security/security.c?id=1cdcbec1a3372c0c49c59d292e708fd07b509f18'>CRED: Neuter sys_capset()</a></td><td>David Howells</td><td>1</td><td><span class='deletions'>-10</span>/<span class='insertions'>+8</span></td></tr>
<tr><td><span title='2008-11-11 22:02:50 +1100'>2008-11-11</span></td><td><a href='/drm-tip/commit/security/security.c?id=06112163f5fd9e491a7f810443d81efa9d88e247'>Add a new capable interface that will be used by systems that use audit to</a></td><td>Eric Paris</td><td>1</td><td><span class='deletions'>-1</span>/<span class='insertions'>+6</span></td></tr>
<tr><td><span title='2008-10-30 11:38:47 -0700'>2008-10-30</span></td><td><a href='/drm-tip/commit/security/security.c?id=731572d39fcd3498702eda4600db4c43d51e0b26'>nfsd: fix vm overcommit crash</a></td><td>Alan Cox</td><td>1</td><td><span class='deletions'>-0</span>/<span class='insertions'>+9</span></td></tr>
<tr><td><span title='2008-08-28 10:47:34 +1000'>2008-08-28</span></td><td><a href='/drm-tip/commit/security/security.c?id=86d688984deefa3ae5a802880c11f2b408b5d6cf'>Merge branch 'master' into next</a></td><td>James Morris</td><td>1</td><td><span class='deletions'>-3</span>/<span class='insertions'>+7</span></td></tr>
<tr><td><span title='2008-08-20 20:16:32 +1000'>2008-08-20</span></td><td><a href='/drm-tip/commit/security/security.c?id=3f23d815c5049c9d7022226cec2242e384dd0b43'>security: add/fix security kernel-doc</a></td><td>Randy Dunlap</td><td>1</td><td><span class='deletions'>-4</span>/<span class='insertions'>+4</span></td></tr>
<tr><td><span title='2008-08-14 22:59:43 +1000'>2008-08-14</span></td><td><a href='/drm-tip/commit/security/security.c?id=5cd9c58fbe9ec92b45b27e131719af4f2bd9eb40'>security: Fix setting of PF_SUPERPRIV by __capable()</a></td><td>David Howells</td><td>1</td><td><span class='deletions'>-3</span>/<span class='insertions'>+7</span></td></tr>
<tr><td><span title='2008-07-26 20:53:27 -0400'>2008-07-26</span></td><td><a href='/drm-tip/commit/security/security.c?id=b1da47e29e467f1ec36dc78d009bfb109fd533c7'>[patch 3/4] fat: dont call notify_change</a></td><td>Miklos Szeredi</td><td>1</td><td><span class='deletions'>-0</span>/<span class='insertions'>+1</span></td></tr>
<tr><td><span title='2008-07-26 20:53:22 -0400'>2008-07-26</span></td><td><a href='/drm-tip/commit/security/security.c?id=b77b0646ef4efe31a7449bb3d9360fd00f95433d'>[PATCH] pass MAY_OPEN to vfs_permission() explicitly</a></td><td>Al Viro</td><td>1</td><td><span class='deletions'>-2</span>/<span class='insertions'>+2</span></td></tr>