2 * Linux/PA-RISC Project (http://www.parisc-linux.org/)
4 * System call entry code Copyright (c) Matthew Wilcox 1999 <willy@bofh.ai>
5 * Licensed under the GNU GPL.
6 * thanks to Philipp Rumpf, Mike Shaver and various others
7 * sorry about the wall, puffin..
10 #include <asm/asm-offsets.h>
11 #include <asm/unistd.h>
12 #include <asm/errno.h>
15 #include <asm/thread_info.h>
16 #include <asm/assembly.h>
17 #include <asm/processor.h>
19 #include <linux/linkage.h>
21 /* We fill the empty parts of the gateway page with
22 * something that will kill the kernel or a
23 * userspace application.
25 #define KILL_INSN break 0,0
31 .import syscall_exit,code
32 .import syscall_exit_rfi,code
34 /* Linux gateway page is aliased to virtual page 0 in the kernel
35 * address space. Since it is a gateway page it cannot be
36 * dereferenced, so null pointers will still fault. We start
37 * the actual entry point at 0x100. We put break instructions
38 * at the beginning of the page to trap null indirect function
43 ENTRY(linux_gateway_page)
45 /* ADDRESS 0x00 to 0xb0 = 176 bytes / 4 bytes per insn = 44 insns */
50 /* ADDRESS 0xb0 to 0xb4, lws uses 1 insns for entry */
51 /* Light-weight-syscall entry must always be located at 0xb0 */
52 /* WARNING: Keep this number updated with table size changes */
53 #define __NR_lws_entries (2)
56 /* Unconditional branch to lws_start, located on the
60 /* Fill from 0xb4 to 0xe0 */
65 /* This function MUST be located at 0xe0 for glibc's threading
66 mechanism to work. DO NOT MOVE THIS CODE EVER! */
68 gate .+8, %r0 /* increase privilege */
69 depi 3, 31, 2, %r31 /* Ensure we return into user mode. */
70 be 0(%sr7,%r31) /* return to user space */
71 mtctl %r26, %cr27 /* move arg0 to the control register */
73 /* Increase the chance of trapping if random jumps occur to this
74 address, fill from 0xf0 to 0x100 */
79 /* This address must remain fixed at 0x100 for glibc's syscalls to work */
82 gate .+8, %r0 /* become privileged */
83 mtsp %r0,%sr4 /* get kernel space into sr4 */
84 mtsp %r0,%sr5 /* get kernel space into sr5 */
85 mtsp %r0,%sr6 /* get kernel space into sr6 */
86 mfsp %sr7,%r1 /* save user sr7 */
87 mtsp %r1,%sr3 /* and store it in sr3 */
90 /* for now we can *always* set the W bit on entry to the syscall
91 * since we don't support wide userland processes. We could
92 * also save the current SM other than in r0 and restore it on
93 * exit from the syscall, and also use that value to know
94 * whether to do narrow or wide syscalls. -PB
97 extrd,u %r1,PSW_W_BIT,1,%r1
98 /* sp must be aligned on 4, so deposit the W bit setting into
99 * the bottom of sp temporarily */
102 /* The top halves of argument registers must be cleared on syscall
103 * entry from narrow executable.
105 depdi 0, 31, 32, %r26
106 depdi 0, 31, 32, %r25
107 depdi 0, 31, 32, %r24
108 depdi 0, 31, 32, %r23
109 depdi 0, 31, 32, %r22
110 depdi 0, 31, 32, %r21
114 xor %r1,%r30,%r30 /* ye olde xor trick */
118 ldo THREAD_SZ_ALGN+FRAME_SIZE(%r30),%r30 /* set up kernel stack */
120 /* N.B.: It is critical that we don't set sr7 to 0 until r30
121 * contains a valid kernel stack pointer. It is also
122 * critical that we don't start using the kernel stack
123 * until after sr7 has been set to 0.
126 mtsp %r0,%sr7 /* get kernel space into sr7 */
127 STREGM %r1,FRAME_SIZE(%r30) /* save r1 (usp) here for now */
128 mfctl %cr30,%r1 /* get task ptr in %r1 */
129 LDREG TI_TASK(%r1),%r1
131 /* Save some registers for sigcontext and potential task
132 switch (see entry.S for the details of which ones are
133 saved/restored). TASK_PT_PSW is zeroed so we can see whether
134 a process is on a syscall or not. For an interrupt the real
135 PSW value is stored. This is needed for gdb and sys_ptrace. */
136 STREG %r0, TASK_PT_PSW(%r1)
137 STREG %r2, TASK_PT_GR2(%r1) /* preserve rp */
138 STREG %r19, TASK_PT_GR19(%r1)
140 LDREGM -FRAME_SIZE(%r30), %r2 /* get users sp back */
142 extrd,u %r2,63,1,%r19 /* W hidden in bottom bit */
144 xor %r19,%r2,%r2 /* clear bottom bit */
146 std %r19,TASK_PT_PSW(%r1)
149 STREG %r2, TASK_PT_GR30(%r1) /* ... and save it */
151 STREG %r20, TASK_PT_GR20(%r1) /* Syscall number */
152 STREG %r21, TASK_PT_GR21(%r1)
153 STREG %r22, TASK_PT_GR22(%r1)
154 STREG %r23, TASK_PT_GR23(%r1) /* 4th argument */
155 STREG %r24, TASK_PT_GR24(%r1) /* 3rd argument */
156 STREG %r25, TASK_PT_GR25(%r1) /* 2nd argument */
157 STREG %r26, TASK_PT_GR26(%r1) /* 1st argument */
158 STREG %r27, TASK_PT_GR27(%r1) /* user dp */
159 STREG %r28, TASK_PT_GR28(%r1) /* return value 0 */
160 STREG %r28, TASK_PT_ORIG_R28(%r1) /* return value 0 (saved for signals) */
161 STREG %r29, TASK_PT_GR29(%r1) /* return value 1 */
162 STREG %r31, TASK_PT_GR31(%r1) /* preserve syscall return ptr */
164 ldo TASK_PT_FR0(%r1), %r27 /* save fpregs from the kernel */
165 save_fp %r27 /* or potential task switch */
167 mfctl %cr11, %r27 /* i.e. SAR */
168 STREG %r27, TASK_PT_SAR(%r1)
173 ldo -16(%r30),%r29 /* Reference param save area */
174 copy %r19,%r2 /* W bit back to r2 */
176 /* no need to save these on stack in wide mode because the first 8
177 * args are passed in registers */
178 stw %r22, -52(%r30) /* 5th argument */
179 stw %r21, -56(%r30) /* 6th argument */
182 /* Are we being ptraced? */
184 LDREG TI_TASK(%r1),%r1
185 ldw TASK_PTRACE(%r1), %r1
186 bb,<,n %r1,31,.Ltracesys
188 /* Note! We cannot use the syscall table that is mapped
189 nearby since the gateway page is mapped execute-only. */
192 ldil L%sys_call_table, %r1
194 addil L%(sys_call_table64-sys_call_table), %r1
195 ldo R%sys_call_table(%r1), %r19
197 ldo R%sys_call_table64(%r1), %r19
199 ldil L%sys_call_table, %r1
200 ldo R%sys_call_table(%r1), %r19
202 comiclr,>> __NR_Linux_syscalls, %r20, %r0
205 LDREGX %r20(%r19), %r19
207 /* If this is a sys_rt_sigreturn call, and the signal was received
208 * when not in_syscall, then we want to return via syscall_exit_rfi,
209 * not syscall_exit. Signal no. in r20, in_syscall in r25 (see
210 * trampoline code in signal.c).
212 ldi __NR_rt_sigreturn,%r2
213 comb,= %r2,%r20,.Lrt_sigreturn
215 ldil L%syscall_exit,%r2
217 ldo R%syscall_exit(%r2),%r2
219 comib,<> 0,%r25,.Lin_syscall
220 ldil L%syscall_exit_rfi,%r2
222 ldo R%syscall_exit_rfi(%r2),%r2
224 /* Note! Because we are not running where we were linked, any
225 calls to functions external to this file must be indirect. To
226 be safe, we apply the opposite rule to functions within this
227 file, with local labels given to them to ensure correctness. */
231 ldil L%syscall_exit,%r1
232 be R%syscall_exit(%sr7,%r1)
233 ldo -ENOSYS(%r0),%r28 /* set errno */
236 /* Warning! This trace code is a virtual duplicate of the code above so be
237 * sure to maintain both! */
240 /* Need to save more registers so the debugger can see where we
241 * are. This saves only the lower 8 bits of PSW, so that the C
242 * bit is still clear on syscalls, and the D bit is set if this
243 * full register save path has been executed. We check the D
244 * bit on syscall_return_rfi to determine which registers to
245 * restore. An interrupt results in a full PSW saved with the
246 * C bit set, a non-straced syscall entry results in C and D clear
249 ldo -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 /* get task ptr */
250 LDREG TI_TASK(%r1), %r1
252 STREG %r2,TASK_PT_PSW(%r1) /* Lower 8 bits only!! */
254 STREG %r2,TASK_PT_SR0(%r1)
256 STREG %r2,TASK_PT_SR1(%r1)
258 STREG %r2,TASK_PT_SR2(%r1)
260 STREG %r2,TASK_PT_SR3(%r1)
261 STREG %r2,TASK_PT_SR4(%r1)
262 STREG %r2,TASK_PT_SR5(%r1)
263 STREG %r2,TASK_PT_SR6(%r1)
264 STREG %r2,TASK_PT_SR7(%r1)
265 STREG %r2,TASK_PT_IASQ0(%r1)
266 STREG %r2,TASK_PT_IASQ1(%r1)
267 LDREG TASK_PT_GR31(%r1),%r2
268 STREG %r2,TASK_PT_IAOQ0(%r1)
270 STREG %r2,TASK_PT_IAOQ1(%r1)
271 ldo TASK_REGS(%r1),%r2
273 STREG %r3,PT_GR3(%r2)
274 STREG %r4,PT_GR4(%r2)
275 STREG %r5,PT_GR5(%r2)
276 STREG %r6,PT_GR6(%r2)
277 STREG %r7,PT_GR7(%r2)
278 STREG %r8,PT_GR8(%r2)
279 STREG %r9,PT_GR9(%r2)
280 STREG %r10,PT_GR10(%r2)
281 STREG %r11,PT_GR11(%r2)
282 STREG %r12,PT_GR12(%r2)
283 STREG %r13,PT_GR13(%r2)
284 STREG %r14,PT_GR14(%r2)
285 STREG %r15,PT_GR15(%r2)
286 STREG %r16,PT_GR16(%r2)
287 STREG %r17,PT_GR17(%r2)
288 STREG %r18,PT_GR18(%r2)
289 /* Finished saving things for the debugger */
291 ldi 0,%r26 /* syscall entry */
292 ldil L%syscall_trace,%r1
293 ldil L%tracesys_next,%r2
294 be R%syscall_trace(%sr7,%r1)
295 ldo R%tracesys_next(%r2),%r2
298 ldil L%sys_call_table,%r1
299 ldo R%sys_call_table(%r1), %r19
301 ldo -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 /* get task ptr */
302 LDREG TI_TASK(%r1), %r1
303 LDREG TASK_PT_GR20(%r1), %r20
304 LDREG TASK_PT_GR26(%r1), %r26 /* Restore the users args */
305 LDREG TASK_PT_GR25(%r1), %r25
306 LDREG TASK_PT_GR24(%r1), %r24
307 LDREG TASK_PT_GR23(%r1), %r23
309 LDREG TASK_PT_GR22(%r1), %r22
310 LDREG TASK_PT_GR21(%r1), %r21
311 ldo -16(%r30),%r29 /* Reference param save area */
314 comiclr,>>= __NR_Linux_syscalls, %r20, %r0
317 LDREGX %r20(%r19), %r19
319 /* If this is a sys_rt_sigreturn call, and the signal was received
320 * when not in_syscall, then we want to return via syscall_exit_rfi,
321 * not syscall_exit. Signal no. in r20, in_syscall in r25 (see
322 * trampoline code in signal.c).
324 ldi __NR_rt_sigreturn,%r2
325 comb,= %r2,%r20,.Ltrace_rt_sigreturn
327 ldil L%tracesys_exit,%r2
329 ldo R%tracesys_exit(%r2),%r2
331 /* Do *not* call this function on the gateway page, because it
332 makes a direct call to syscall_trace. */
335 ldo -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 /* get task ptr */
336 LDREG TI_TASK(%r1), %r1
338 ldo -16(%r30),%r29 /* Reference param save area */
340 ldi 1,%r26 /* syscall exit */
341 bl syscall_trace, %r2
342 STREG %r28,TASK_PT_GR28(%r1) /* save return value now */
343 ldo -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 /* get task ptr */
344 LDREG TI_TASK(%r1), %r1
345 LDREG TASK_PT_GR28(%r1), %r28 /* Restore return val. */
347 ldil L%syscall_exit,%r1
348 be,n R%syscall_exit(%sr7,%r1)
350 .Ltrace_rt_sigreturn:
351 comib,<> 0,%r25,.Ltrace_in_syscall
352 ldil L%tracesys_sigexit,%r2
354 ldo R%tracesys_sigexit(%r2),%r2
357 ldo -THREAD_SZ_ALGN-FRAME_SIZE(%r30),%r1 /* get task ptr */
358 LDREG TI_TASK(%r1), %r1
360 ldo -16(%r30),%r29 /* Reference param save area */
362 bl syscall_trace, %r2
363 ldi 1,%r26 /* syscall exit */
365 ldil L%syscall_exit_rfi,%r1
366 be,n R%syscall_exit_rfi(%sr7,%r1)
369 /*********************************************************
370 32/64-bit Light-Weight-Syscall ABI
372 * - Indicates a hint for userspace inline asm
375 Syscall number (caller-saves)
379 Argument registers (caller-saves)
380 - %r26, %r25, %r24, %r23, %r22
383 Return registers (caller-saves)
384 - %r28 (return), %r21 (errno)
387 Caller-saves registers
389 - %r2 (return pointer)
390 - %r31 (ble link register)
393 Callee-saves registers
395 - %r30 (stack pointer)
396 * Not in asm clobber.
398 If userspace is 32-bit:
399 Callee-saves registers
400 - %r19 (32-bit PIC register)
402 Differences from 32-bit calling convention:
403 - Syscall number in %r20
404 - Additional argument register %r22 (arg4)
407 If userspace is 64-bit:
408 Callee-saves registers
409 - %r27 (64-bit PIC register)
411 Differences from 64-bit calling convention:
412 - Syscall number in %r20
413 - Additional argument register %r22 (arg4)
416 Error codes returned by entry path:
418 ENOSYS - r20 was an invalid LWS number.
420 *********************************************************/
422 /* Gate and ensure we return to userspace */
424 depi 3, 31, 2, %r31 /* Ensure we return to userspace */
427 /* FIXME: If we are a 64-bit kernel just
428 * turn this on unconditionally.
431 extrd,u %r1,PSW_W_BIT,1,%r1
432 /* sp must be aligned on 4, so deposit the W bit setting into
433 * the bottom of sp temporarily */
436 /* Clip LWS number to a 32-bit value always */
437 depdi 0, 31, 32, %r20
440 /* Is the lws entry number valid? */
441 comiclr,>>= __NR_lws_entries, %r20, %r0
444 /* WARNING: Trashing sr2 and sr3 */
445 mfsp %sr7,%r1 /* get userspace into sr3 */
447 mtsp %r0,%sr2 /* get kernel space into sr2 */
449 /* Load table start */
450 ldil L%lws_table, %r1
451 ldo R%lws_table(%r1), %r28 /* Scratch use of r28 */
452 LDREGX %r20(%sr2,r28), %r21 /* Scratch use of r21 */
454 /* Jump to lws, lws table pointers already relocated */
458 ldo -ENOSYS(%r0),%r21 /* set errno */
459 /* Fall through: Return to userspace */
463 /* decide whether to reset the wide mode bit
465 * For a syscall, the W bit is stored in the lowest bit
466 * of sp. Extract it and reset W if it is zero */
467 extrd,u,*<> %r30,63,1,%r1
469 /* now reset the lowest bit of sp if it was set */
476 /***************************************************
477 Implementing CAS as an atomic operation:
479 %r26 - Address to examine
480 %r25 - Old value to check (old)
481 %r24 - New value to set (new)
482 %r28 - Return prev through this register.
483 %r21 - Kernel error code
485 If debugging is DISabled:
487 %r21 has the following meanings:
489 EAGAIN - CAS is busy, ldcw failed, try again.
490 EFAULT - Read or write failed.
492 If debugging is enabled:
494 EDEADLOCK - CAS called recursively.
495 EAGAIN && r28 == 1 - CAS is busy. Lock contended.
496 EAGAIN && r28 == 2 - CAS is busy. ldcw failed.
497 EFAULT - Read or write failed.
499 Scratch: r20, r28, r1
501 ****************************************************/
503 /* Do not enable LWS debugging */
504 #define ENABLE_LWS_DEBUG 0
506 /* ELF64 Process entry path */
507 lws_compare_and_swap64:
509 b,n lws_compare_and_swap
511 /* If we are not a 64-bit kernel, then we don't
512 * have 64-bit input registers, and calling
513 * the 64-bit LWS CAS returns ENOSYS.
518 /* ELF32 Process entry path */
519 lws_compare_and_swap32:
521 /* Clip all the input registers */
522 depdi 0, 31, 32, %r26
523 depdi 0, 31, 32, %r25
524 depdi 0, 31, 32, %r24
527 lws_compare_and_swap:
529 /* Load start of lock table */
530 ldil L%lws_lock_start, %r20
531 ldo R%lws_lock_start(%r20), %r28
533 /* Extract four bits from r26 and hash lock (Bits 4-7) */
534 extru %r26, 27, 4, %r20
536 /* Find lock to use, the hash is either one of 0 to
537 15, multiplied by 16 (keep it 16-byte aligned)
538 and add to the lock table offset. */
542 # if ENABLE_LWS_DEBUG
544 DEBUG, check for deadlock!
545 If the thread register values are the same
546 then we were the one that locked it last and
547 this is a recurisve call that will deadlock.
548 We *must* giveup this call and fail.
550 ldw 4(%sr2,%r20), %r28 /* Load thread register */
551 /* WARNING: If cr27 cycles to the same value we have problems */
552 mfctl %cr27, %r21 /* Get current thread register */
553 cmpb,<>,n %r21, %r28, cas_lock /* Called recursive? */
554 b lws_exit /* Return error! */
555 ldo -EDEADLOCK(%r0), %r21
557 cmpb,=,n %r0, %r28, cas_nocontend /* Is nobody using it? */
558 ldo 1(%r0), %r28 /* 1st case */
559 b lws_exit /* Contended... */
560 ldo -EAGAIN(%r0), %r21 /* Spin in userspace */
563 /* ENABLE_LWS_DEBUG */
565 LDCW 0(%sr2,%r20), %r28 /* Try to acquire the lock */
566 cmpb,<>,n %r0, %r28, cas_action /* Did we get it? */
568 ldo 2(%r0), %r28 /* 2nd case */
569 b lws_exit /* Contended... */
570 ldo -EAGAIN(%r0), %r21 /* Spin in userspace */
582 This all works becuse intr_do_signal
583 and schedule both check the return iasq
584 and see that we are on the kernel page
585 so this process is never scheduled off
586 or is ever sent any signal of any sort,
587 thus it is wholly atomic from usrspaces
591 #if defined CONFIG_SMP && ENABLE_LWS_DEBUG
594 stw %r1, 4(%sr2,%r20)
596 /* The load and store could fail */
597 1: ldw 0(%sr3,%r26), %r28
598 sub,<> %r28, %r25, %r0
599 2: stw %r24, 0(%sr3,%r26)
602 stw %r20, 0(%sr2,%r20)
603 # if ENABLE_LWS_DEBUG
604 /* Clear thread register indicator */
605 stw %r0, 4(%sr2,%r20)
608 /* Return to userspace, set no error */
613 /* Error occured on load or store */
616 stw %r20, 0(%sr2,%r20)
617 # if ENABLE_LWS_DEBUG
618 stw %r0, 4(%sr2,%r20)
622 ldo -EFAULT(%r0),%r21 /* set errno */
628 /* Two exception table entries, one for the load,
629 the other for the store. Either return -EFAULT.
630 Each of the entries must be relocated. */
631 .section __ex_table,"aw"
632 ASM_ULONG_INSN (1b - linux_gateway_page), (3b - linux_gateway_page)
633 ASM_ULONG_INSN (2b - linux_gateway_page), (3b - linux_gateway_page)
637 /* Make sure nothing else is placed on this page */
639 END(linux_gateway_page)
640 ENTRY(end_linux_gateway_page)
642 /* Relocate symbols assuming linux_gateway_page is mapped
643 to virtual address 0x0 */
645 #define LWS_ENTRY(_name_) ASM_ULONG_INSN (lws_##_name_ - linux_gateway_page)
650 /* Light-weight-syscall table */
651 /* Start of lws table. */
653 LWS_ENTRY(compare_and_swap32) /* 0 - ELF32 Atomic compare and swap */
654 LWS_ENTRY(compare_and_swap64) /* 1 - ELF64 Atomic compare and swap */
656 /* End of lws table */
659 ENTRY(sys_call_table)
660 #include "syscall_table.S"
665 ENTRY(sys_call_table64)
666 #define SYSCALL_TABLE_64BIT
667 #include "syscall_table.S"
668 END(sys_call_table64)
673 All light-weight-syscall atomic operations
674 will use this set of locks
676 NOTE: The lws_lock_start symbol must be
677 at least 16-byte aligned for safe use
682 ENTRY(lws_lock_start)
685 /* Keep locks aligned at 16-bytes */
694 /* CONFIG_SMP for lws_lock_start */