[net-next-2.6.git] / arch / x86 / lib / memset_64.S

/* Copyright 2002 Andi Kleen, SuSE Labs */

#include <linux/linkage.h>
#include <asm/dwarf2.h>

/*
 * ISO C memset - set a memory block to a byte value.
 *	
 * rdi   destination
 * rsi   value (char) 
 * rdx   count (bytes) 
 * 
 * rax   original destination
 */	
	.section .altinstr_replacement, "ax", @progbits
.Lmemset_c:
	movq %rdi,%r9
	movl %edx,%r8d
	andl $7,%r8d
	movl %edx,%ecx
	shrl $3,%ecx
	/* expand byte value  */
	movzbl %sil,%esi
	movabs $0x0101010101010101,%rax
	mulq %rsi		/* with rax, clobbers rdx */
	rep stosq
	movl %r8d,%ecx
	rep stosb
	movq %r9,%rax
	ret
.Lmemset_e:
	.previous

ENTRY(memset)
ENTRY(__memset)
	CFI_STARTPROC
	movq %rdi,%r10
	movq %rdx,%r11

	/* expand byte value  */
	movzbl %sil,%ecx
	movabs $0x0101010101010101,%rax
	mul    %rcx		/* with rax, clobbers rdx */

	/* align dst */
	movl  %edi,%r9d
	andl  $7,%r9d
	jnz  .Lbad_alignment
	CFI_REMEMBER_STATE
.Lafter_bad_alignment:

	movl %r11d,%ecx
	shrl $6,%ecx
	jz	 .Lhandle_tail

	.p2align 4
.Lloop_64:
	decl   %ecx
	movq  %rax,(%rdi)
	movq  %rax,8(%rdi)
	movq  %rax,16(%rdi)
	movq  %rax,24(%rdi)
	movq  %rax,32(%rdi)
	movq  %rax,40(%rdi)
	movq  %rax,48(%rdi)
	movq  %rax,56(%rdi)
	leaq  64(%rdi),%rdi
	jnz    .Lloop_64

	/* Handle tail in loops. The loops should be faster than hard
	   to predict jump tables. */
	.p2align 4
.Lhandle_tail:
	movl	%r11d,%ecx
	andl    $63&(~7),%ecx
	jz 		.Lhandle_7
	shrl	$3,%ecx
	.p2align 4
.Lloop_8:
	decl   %ecx
	movq  %rax,(%rdi)
	leaq  8(%rdi),%rdi
	jnz    .Lloop_8

.Lhandle_7:
	movl	%r11d,%ecx
	andl	$7,%ecx
	jz      .Lende
	.p2align 4
.Lloop_1:
	decl    %ecx
	movb 	%al,(%rdi)
	leaq	1(%rdi),%rdi
	jnz     .Lloop_1

.Lende:
	movq	%r10,%rax
	ret

	CFI_RESTORE_STATE
.Lbad_alignment:
	cmpq $7,%r11
	jbe	.Lhandle_7
	movq %rax,(%rdi)	/* unaligned store */
	movq $8,%r8
	subq %r9,%r8
	addq %r8,%rdi
	subq %r8,%r11
	jmp .Lafter_bad_alignment
.Lfinal:
	CFI_ENDPROC
ENDPROC(memset)
ENDPROC(__memset)

	/* Some CPUs run faster using the string instructions.
	   It is also a lot simpler. Use this when possible */

#include <asm/cpufeature.h>

	.section .altinstructions,"a"
	.align 8
	.quad memset
	.quad .Lmemset_c
	.word X86_FEATURE_REP_GOOD
	.byte .Lfinal - memset
	.byte .Lmemset_e - .Lmemset_c
	.previous
Commit	Line	Data
1da177e4	1	/* Copyright 2002 Andi Kleen, SuSE Labs */
8d379dad	2
8d379dad JB	3	#include <linux/linkage.h>
	4	#include <asm/dwarf2.h>
	5
1da177e4 LT	6	/*
	7	* ISO C memset - set a memory block to a byte value.
	8	*
	9	* rdi destination
	10	* rsi value (char)
	11	* rdx count (bytes)
	12	*
	13	* rax original destination
	14	*/
7269e881 JB	15	.section .altinstr_replacement, "ax", @progbits
7269e881 JB	16	.Lmemset_c:
8d379dad JB	17	movq %rdi,%r9
	18	movl %edx,%r8d
	19	andl $7,%r8d
	20	movl %edx,%ecx
	21	shrl $3,%ecx
	22	/* expand byte value */
	23	movzbl %sil,%esi
	24	movabs $0x0101010101010101,%rax
	25	mulq %rsi /* with rax, clobbers rdx */
	26	rep stosq
	27	movl %r8d,%ecx
	28	rep stosb
	29	movq %r9,%rax
	30	ret
7269e881 JB	31	.Lmemset_e:
7269e881 JB	32	.previous
8d379dad JB	33
	34	ENTRY(memset)
	35	ENTRY(__memset)
	36	CFI_STARTPROC
7bcd3f34 AK	37	movq %rdi,%r10
	38	movq %rdx,%r11
	39
	40	/* expand byte value */
	41	movzbl %sil,%ecx
	42	movabs $0x0101010101010101,%rax
	43	mul %rcx /* with rax, clobbers rdx */
	44
	45	/* align dst */
	46	movl %edi,%r9d
	47	andl $7,%r9d
	48	jnz .Lbad_alignment
8d379dad	49	CFI_REMEMBER_STATE
7bcd3f34 AK	50	.Lafter_bad_alignment:
	51
	52	movl %r11d,%ecx
	53	shrl $6,%ecx
	54	jz .Lhandle_tail
	55
	56	.p2align 4
	57	.Lloop_64:
	58	decl %ecx
	59	movq %rax,(%rdi)
	60	movq %rax,8(%rdi)
	61	movq %rax,16(%rdi)
	62	movq %rax,24(%rdi)
	63	movq %rax,32(%rdi)
	64	movq %rax,40(%rdi)
	65	movq %rax,48(%rdi)
	66	movq %rax,56(%rdi)
	67	leaq 64(%rdi),%rdi
	68	jnz .Lloop_64
	69
	70	/* Handle tail in loops. The loops should be faster than hard
	71	to predict jump tables. */
	72	.p2align 4
	73	.Lhandle_tail:
	74	movl %r11d,%ecx
	75	andl $63&(~7),%ecx
	76	jz .Lhandle_7
	77	shrl $3,%ecx
	78	.p2align 4
	79	.Lloop_8:
	80	decl %ecx
	81	movq %rax,(%rdi)
	82	leaq 8(%rdi),%rdi
	83	jnz .Lloop_8
	84
	85	.Lhandle_7:
	86	movl %r11d,%ecx
	87	andl $7,%ecx
	88	jz .Lende
	89	.p2align 4
	90	.Lloop_1:
	91	decl %ecx
	92	movb %al,(%rdi)
	93	leaq 1(%rdi),%rdi
	94	jnz .Lloop_1
	95
	96	.Lende:
	97	movq %r10,%rax
	98	ret
	99
8d379dad	100	CFI_RESTORE_STATE
7bcd3f34 AK	101	.Lbad_alignment:
	102	cmpq $7,%r11
	103	jbe .Lhandle_7
	104	movq %rax,(%rdi) /* unaligned store */
	105	movq $8,%r8
	106	subq %r9,%r8
	107	addq %r8,%rdi
	108	subq %r8,%r11
	109	jmp .Lafter_bad_alignment
8d379dad JB	110	.Lfinal:
	111	CFI_ENDPROC
	112	ENDPROC(memset)
	113	ENDPROC(__memset)
7bcd3f34 AK	114
	115	/* Some CPUs run faster using the string instructions.
	116	It is also a lot simpler. Use this when possible */
	117
	118	#include <asm/cpufeature.h>
	119
	120	.section .altinstructions,"a"
	121	.align 8
8d379dad	122	.quad memset
7269e881	123	.quad .Lmemset_c
83a7a2ad	124	.word X86_FEATURE_REP_GOOD
8d379dad	125	.byte .Lfinal - memset
7269e881	126	.byte .Lmemset_e - .Lmemset_c
7bcd3f34	127	.previous