3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
21 #include <linux/errno.h>
22 #include <linux/err.h>
23 #include <linux/magic.h>
24 #include <asm/unistd.h>
25 #include <asm/processor.h>
28 #include <asm/thread_info.h>
29 #include <asm/ppc_asm.h>
30 #include <asm/asm-offsets.h>
31 #include <asm/cputable.h>
32 #include <asm/firmware.h>
34 #include <asm/ptrace.h>
35 #include <asm/irqflags.h>
36 #include <asm/ftrace.h>
37 #include <asm/hw_irq.h>
38 #include <asm/context_tracking.h>
46 .tc sys_call_table[TC],sys_call_table
48 /* This value is used to mark exception frames on the stack. */
50 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
55 .globl system_call_common
57 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
59 extrdi. r10, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
61 END_FTR_SECTION_IFSET(CPU_FTR_TM)
65 addi r1,r1,-INT_FRAME_SIZE
73 beq 2f /* if from kernel mode */
74 ACCOUNT_CPU_USER_ENTRY(r10, r11)
93 * This clears CR0.SO (bit 28), which is the error indication on
94 * return from this system call.
96 rldimi r2,r11,28,(63-28)
103 addi r9,r1,STACK_FRAME_OVERHEAD
104 ld r11,exception_marker@toc(r2)
105 std r11,-16(r9) /* "regshere" marker */
106 #if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(CONFIG_PPC_SPLPAR)
109 /* if from user, see if there are any DTL entries to process */
110 ld r10,PACALPPACAPTR(r13) /* get ptr to VPA */
111 ld r11,PACA_DTL_RIDX(r13) /* get log read index */
112 addi r10,r10,LPPACA_DTLIDX
113 LDX_BE r10,0,r10 /* get log write index */
116 bl accumulate_stolen_time
120 addi r9,r1,STACK_FRAME_OVERHEAD
122 END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
123 #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE && CONFIG_PPC_SPLPAR */
126 * A syscall should always be called with interrupts enabled
127 * so we just unconditionally hard-enable here. When some kind
128 * of irq tracing is used, we additionally check that condition
131 #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG)
132 lbz r10,PACASOFTIRQEN(r13)
135 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
138 #ifdef CONFIG_PPC_BOOK3E
144 #endif /* CONFIG_PPC_BOOK3E */
146 /* We do need to set SOFTE in the stack frame or the return
147 * from interrupt will be painful
152 CURRENT_THREAD_INFO(r11, r1)
154 andi. r11,r10,_TIF_SYSCALL_DOTRACE
155 bne syscall_dotrace /* does not return */
156 cmpldi 0,r0,NR_syscalls
159 system_call: /* label this so stack traces look sane */
161 * Need to vector to 32 Bit or default sys_call_table here,
162 * based on caller's run-mode / personality.
164 ld r11,SYS_CALL_TABLE@toc(2)
165 andi. r10,r10,_TIF_32BIT
167 addi r11,r11,8 /* use 32-bit syscall entries */
176 ldx r12,r11,r0 /* Fetch system call handler [ptr] */
178 bctrl /* Call handler */
182 CURRENT_THREAD_INFO(r12, r1)
185 #ifdef CONFIG_PPC_BOOK3S
186 /* No MSR:RI on BookE */
191 * Disable interrupts so current_thread_info()->flags can't change,
192 * and so that we don't get interrupted after loading SRR0/1.
194 #ifdef CONFIG_PPC_BOOK3E
199 * For performance reasons we clear RI the same time that we
200 * clear EE. We only need to clear RI just before we restore r13
201 * below, but batching it with EE saves us one expensive mtmsrd call.
202 * We have to be careful to restore RI if we branch anywhere from
203 * here (eg syscall_exit_work).
208 #endif /* CONFIG_PPC_BOOK3E */
212 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
213 bne- syscall_exit_work
217 #ifdef CONFIG_ALTIVEC
218 andis. r0,r8,MSR_VEC@h
221 2: addi r3,r1,STACK_FRAME_OVERHEAD
222 #ifdef CONFIG_PPC_BOOK3S
223 mtmsrd r10,1 /* Restore RI */
226 #ifdef CONFIG_PPC_BOOK3S
229 andc r11,r10,r9 /* Re-clear RI */
239 .Lsyscall_error_cont:
242 stdcx. r0,0,r1 /* to clear the reservation */
243 END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
248 ACCOUNT_CPU_USER_EXIT(r11, r12)
252 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
254 ld r13,GPR13(r1) /* only restore r13 if returning to usermode */
262 b . /* prevent speculative execution */
265 oris r5,r5,0x1000 /* Set SO bit in CR */
268 b .Lsyscall_error_cont
270 /* Traced system call support */
273 addi r3,r1,STACK_FRAME_OVERHEAD
274 bl do_syscall_trace_enter
277 * We use the return value of do_syscall_trace_enter() as the syscall
278 * number. If the syscall was rejected for any reason do_syscall_trace_enter()
279 * returns an invalid syscall number and the test below against
280 * NR_syscalls will fail.
284 /* Restore argument registers just clobbered and/or possibly changed. */
292 /* Repopulate r9 and r10 for the system_call path */
293 addi r9,r1,STACK_FRAME_OVERHEAD
294 CURRENT_THREAD_INFO(r10, r1)
297 cmpldi r0,NR_syscalls
300 /* Return code is already in r3 thanks to do_syscall_trace_enter() */
309 #ifdef CONFIG_PPC_BOOK3S
310 mtmsrd r10,1 /* Restore RI */
312 /* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
313 If TIF_NOERROR is set, just save r3 as it is. */
315 andi. r0,r9,_TIF_RESTOREALL
319 0: cmpld r3,r11 /* r11 is -MAX_ERRNO */
321 andi. r0,r9,_TIF_NOERROR
325 oris r5,r5,0x1000 /* Set SO bit in CR */
328 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
331 /* Clear per-syscall TIF flags if any are set. */
333 li r11,_TIF_PERSYSCALL_MASK
334 addi r12,r12,TI_FLAGS
339 subi r12,r12,TI_FLAGS
341 4: /* Anything else left to do? */
343 lis r3,INIT_PPR@highest /* Set thread.ppr = 3 */
344 ld r10,PACACURRENT(r13)
345 sldi r3,r3,32 /* bits 11-13 are used for ppr */
346 std r3,TASKTHREADPPR(r10)
347 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
349 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP)
350 beq ret_from_except_lite
352 /* Re-enable interrupts */
353 #ifdef CONFIG_PPC_BOOK3E
359 #endif /* CONFIG_PPC_BOOK3E */
362 addi r3,r1,STACK_FRAME_OVERHEAD
363 bl do_syscall_trace_leave
366 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
368 /* Firstly we need to enable TM in the kernel */
371 rldimi r10, r13, MSR_TM_LG, 63-MSR_TM_LG
374 /* tabort, this dooms the transaction, nothing else */
375 li r13, (TM_CAUSE_SYSCALL|TM_CAUSE_PERSISTENT)
379 * Return directly to userspace. We have corrupted user register state,
380 * but userspace will never see that register state. Execution will
381 * resume after the tbegin of the aborted transaction with the
382 * checkpointed register state.
391 b . /* prevent speculative execution */
394 /* Save non-volatile GPRs, if not already saved. */
406 * The sigsuspend and rt_sigsuspend system calls can call do_signal
407 * and thus put the process into the stopped state where we might
408 * want to examine its user state with ptrace. Therefore we need
409 * to save all the nonvolatile registers (r14 - r31) before calling
410 * the C code. Similarly, fork, vfork and clone need the full
411 * register state on the stack so that it can be copied to the child.
429 _GLOBAL(ppc32_swapcontext)
431 bl compat_sys_swapcontext
434 _GLOBAL(ppc64_swapcontext)
439 _GLOBAL(ppc_switch_endian)
444 _GLOBAL(ret_from_fork)
450 _GLOBAL(ret_from_kernel_thread)
455 #if defined(_CALL_ELF) && _CALL_ELF == 2
463 * This routine switches between two different tasks. The process
464 * state of one is saved on its kernel stack. Then the state
465 * of the other is restored from its kernel stack. The memory
466 * management hardware is updated to the second process's state.
467 * Finally, we can return to the second process, via ret_from_except.
468 * On entry, r3 points to the THREAD for the current task, r4
469 * points to the THREAD for the new task.
471 * Note: there are two ways to get to the "going out" portion
472 * of this code; either by coming in via the entry (_switch)
473 * or via "fork" which must set up an environment equivalent
474 * to the "_switch" path. If you change this you'll have to change
475 * the fork code also.
477 * The code which creates the new task context is in 'copy_thread'
478 * in arch/powerpc/kernel/process.c
484 stdu r1,-SWITCH_FRAME_SIZE(r1)
485 /* r3-r13 are caller saved -- Cort */
488 std r0,_NIP(r1) /* Return to switch caller */
491 std r1,KSP(r3) /* Set old stack pointer */
494 /* We need a sync somewhere here to make sure that if the
495 * previous task gets rescheduled on another CPU, it sees all
496 * stores it has performed on this one.
499 #endif /* CONFIG_SMP */
502 * If we optimise away the clear of the reservation in system
503 * calls because we know the CPU tracks the address of the
504 * reservation, then we need to clear it here to cover the
505 * case that the kernel context switch path has no larx
510 END_FTR_SECTION_IFSET(CPU_FTR_STCX_CHECKS_ADDRESS)
512 #ifdef CONFIG_PPC_BOOK3S
513 /* Cancel all explict user streams as they will have no use after context
514 * switch and will stop the HW from creating streams itself
516 DCBT_STOP_ALL_STREAM_IDS(r6)
519 addi r6,r4,-THREAD /* Convert THREAD to 'current' */
520 std r6,PACACURRENT(r13) /* Set new 'current' */
522 ld r8,KSP(r4) /* new stack pointer */
523 #ifdef CONFIG_PPC_BOOK3S
525 clrrdi r6,r8,28 /* get its ESID */
526 clrrdi r9,r1,28 /* get current sp ESID */
528 clrrdi r6,r8,40 /* get its 1T ESID */
529 clrrdi r9,r1,40 /* get current sp 1T ESID */
530 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_1T_SEGMENT)
531 clrldi. r0,r6,2 /* is new ESID c00000000? */
532 cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
534 beq 2f /* if yes, don't slbie it */
536 /* Bolt in the new stack SLB entry */
537 ld r7,KSP_VSID(r4) /* Get new stack's VSID */
538 oris r0,r6,(SLB_ESID_V)@h
539 ori r0,r0,(SLB_NUM_BOLTED-1)@l
541 li r9,MMU_SEGSIZE_1T /* insert B field */
542 oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
543 rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
544 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
546 /* Update the last bolted SLB. No write barriers are needed
547 * here, provided we only update the current CPU's SLB shadow
550 ld r9,PACA_SLBSHADOWPTR(r13)
552 std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
553 li r12,SLBSHADOW_STACKVSID
554 STDX_BE r7,r12,r9 /* Save VSID */
555 li r12,SLBSHADOW_STACKESID
556 STDX_BE r0,r12,r9 /* Save ESID */
558 /* No need to check for MMU_FTR_NO_SLBIE_B here, since when
559 * we have 1TB segments, the only CPUs known to have the errata
560 * only support less than 1TB of system memory and we'll never
561 * actually hit this code path.
565 slbie r6 /* Workaround POWER5 < DD2.1 issue */
569 #endif /* !CONFIG_PPC_BOOK3S */
571 CURRENT_THREAD_INFO(r7, r8) /* base of new stack */
572 /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
573 because we don't need to leave the 288-byte ABI gap at the
574 top of the kernel stack. */
575 addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE
577 mr r1,r8 /* start using new stack pointer */
578 std r7,PACAKSAVE(r13)
583 /* r3-r13 are destroyed -- Cort */
587 /* convert old thread to its task_struct for return value */
589 ld r7,_NIP(r1) /* Return to _switch caller in new task */
591 addi r1,r1,SWITCH_FRAME_SIZE
595 _GLOBAL(ret_from_except)
598 bne ret_from_except_lite
601 _GLOBAL(ret_from_except_lite)
603 * Disable interrupts so that current_thread_info()->flags
604 * can't change between when we test it and when we return
605 * from the interrupt.
607 #ifdef CONFIG_PPC_BOOK3E
610 ld r10,PACAKMSR(r13) /* Get kernel MSR without EE */
611 mtmsrd r10,1 /* Update machine state */
612 #endif /* CONFIG_PPC_BOOK3E */
614 CURRENT_THREAD_INFO(r9, r1)
616 #ifdef CONFIG_PPC_BOOK3E
617 ld r10,PACACURRENT(r13)
618 #endif /* CONFIG_PPC_BOOK3E */
622 #ifdef CONFIG_PPC_BOOK3E
623 lwz r3,(THREAD+THREAD_DBCR0)(r10)
624 #endif /* CONFIG_PPC_BOOK3E */
626 /* Check current_thread_info()->flags */
627 andi. r0,r4,_TIF_USER_WORK_MASK
629 #ifdef CONFIG_PPC_BOOK3E
631 * Check to see if the dbcr0 register is set up to debug.
632 * Use the internal debug mode bit to do this.
634 andis. r0,r3,DBCR0_IDM@h
637 rlwinm r0,r0,0,~MSR_DE /* Clear MSR.DE */
644 addi r3,r1,STACK_FRAME_OVERHEAD
648 1: andi. r0,r4,_TIF_NEED_RESCHED
650 bl restore_interrupts
652 b ret_from_except_lite
654 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
655 andi. r0,r4,_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM
656 bne 3f /* only restore TM if nothing else to do */
657 addi r3,r1,STACK_FRAME_OVERHEAD
664 * Use a non volatile GPR to save and restore our thread_info flags
665 * across the call to restore_interrupts.
668 bl restore_interrupts
670 addi r3,r1,STACK_FRAME_OVERHEAD
675 /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
676 andis. r8,r4,_TIF_EMULATE_STACK_STORE@h
679 addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
682 subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
683 mr r4,r1 /* src: current exception frame */
684 mr r1,r3 /* Reroute the trampoline frame to r1 */
686 /* Copy from the original to the trampoline. */
687 li r5,INT_FRAME_SIZE/8 /* size: INT_FRAME_SIZE */
688 li r6,0 /* start offset: 0 */
695 /* Do real store operation to complete stwu */
699 /* Clear _TIF_EMULATE_STACK_STORE flag */
700 lis r11,_TIF_EMULATE_STACK_STORE@h
708 #ifdef CONFIG_PREEMPT
709 /* Check if we need to preempt */
710 andi. r0,r4,_TIF_NEED_RESCHED
712 /* Check that preempt_count() == 0 and interrupts are enabled */
713 lwz r8,TI_PREEMPT(r9)
717 crandc eq,cr1*4+eq,eq
721 * Here we are preempting the current task. We want to make
722 * sure we are soft-disabled first and reconcile irq state.
724 RECONCILE_IRQ_STATE(r3,r4)
725 1: bl preempt_schedule_irq
727 /* Re-test flags and eventually loop */
728 CURRENT_THREAD_INFO(r9, r1)
730 andi. r0,r4,_TIF_NEED_RESCHED
734 * arch_local_irq_restore() from preempt_schedule_irq above may
735 * enable hard interrupt but we really should disable interrupts
736 * when we return from the interrupt, and so that we don't get
737 * interrupted after loading SRR0/1.
739 #ifdef CONFIG_PPC_BOOK3E
742 ld r10,PACAKMSR(r13) /* Get kernel MSR without EE */
743 mtmsrd r10,1 /* Update machine state */
744 #endif /* CONFIG_PPC_BOOK3E */
745 #endif /* CONFIG_PREEMPT */
747 .globl fast_exc_return_irq
751 * This is the main kernel exit path. First we check if we
752 * are about to re-enable interrupts
755 lbz r6,PACASOFTIRQEN(r13)
759 /* We are enabling, were we already enabled ? Yes, just return */
764 * We are about to soft-enable interrupts (we are hard disabled
765 * at this point). We check if there's anything that needs to
768 lbz r0,PACAIRQHAPPENED(r13)
770 bne- restore_check_irq_replay
773 * Get here when nothing happened while soft-disabled, just
774 * soft-enable and move-on. We will hard-enable as a side
780 stb r0,PACASOFTIRQEN(r13);
783 * Final return path. BookE is handled in a different file
786 #ifdef CONFIG_PPC_BOOK3E
787 b exception_return_book3e
790 * Clear the reservation. If we know the CPU tracks the address of
791 * the reservation then we can potentially save some cycles and use
792 * a larx. On POWER6 and POWER7 this is significantly faster.
795 stdcx. r0,0,r1 /* to clear the reservation */
798 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
801 * Some code path such as load_up_fpu or altivec return directly
802 * here. They run entirely hard disabled and do not alter the
803 * interrupt state. They also don't use lwarx/stwcx. and thus
804 * are known not to leave dangling reservations.
806 .globl fast_exception_return
807 fast_exception_return:
821 /* Load PPR from thread struct before we clear MSR:RI */
823 ld r2,PACACURRENT(r13)
824 ld r2,TASKTHREADPPR(r2)
825 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
828 * Clear RI before restoring r13. If we are returning to
829 * userspace and we take an exception after restoring r13,
830 * we end up corrupting the userspace r13 value.
832 ld r4,PACAKMSR(r13) /* Get kernel MSR without EE */
833 andc r4,r4,r0 /* r0 contains MSR_RI here */
836 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
838 std r3, PACATMSCRATCH(r13) /* Stash returned-to MSR */
841 * r13 is our per cpu area, only restore it if we are returning to
842 * userspace the value stored in the stack frame may belong to
848 mtspr SPRN_PPR,r2 /* Restore PPR */
849 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
850 ACCOUNT_CPU_USER_EXIT(r2, r4)
867 b . /* prevent speculative execution */
869 #endif /* CONFIG_PPC_BOOK3E */
872 * We are returning to a context with interrupts soft disabled.
874 * However, we may also about to hard enable, so we need to
875 * make sure that in this case, we also clear PACA_IRQ_HARD_DIS
876 * or that bit can get out of sync and bad things will happen
880 lbz r7,PACAIRQHAPPENED(r13)
883 rlwinm r7,r7,0,~PACA_IRQ_HARD_DIS
884 stb r7,PACAIRQHAPPENED(r13)
886 stb r0,PACASOFTIRQEN(r13);
891 * Something did happen, check if a re-emit is needed
892 * (this also clears paca->irq_happened)
894 restore_check_irq_replay:
895 /* XXX: We could implement a fast path here where we check
896 * for irq_happened being just 0x01, in which case we can
897 * clear it and return. That means that we would potentially
898 * miss a decrementer having wrapped all the way around.
900 * Still, this might be useful for things like hash_page
902 bl __check_irq_replay
904 beq restore_no_replay
907 * We need to re-emit an interrupt. We do so by re-using our
908 * existing exception frame. We first change the trap value,
909 * but we need to ensure we preserve the low nibble of it
917 * Then find the right handler and call it. Interrupts are
918 * still soft-disabled and we keep them that way.
922 addi r3,r1,STACK_FRAME_OVERHEAD;
925 1: cmpwi cr0,r3,0xe60
927 addi r3,r1,STACK_FRAME_OVERHEAD;
928 bl handle_hmi_exception
930 1: cmpwi cr0,r3,0x900
932 addi r3,r1,STACK_FRAME_OVERHEAD;
935 #ifdef CONFIG_PPC_DOORBELL
937 #ifdef CONFIG_PPC_BOOK3E
944 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
945 #endif /* CONFIG_PPC_BOOK3E */
947 addi r3,r1,STACK_FRAME_OVERHEAD;
948 bl doorbell_exception
950 #endif /* CONFIG_PPC_DOORBELL */
951 1: b ret_from_except /* What else to do here ? */
954 addi r3,r1,STACK_FRAME_OVERHEAD
955 bl unrecoverable_exception
958 #ifdef CONFIG_PPC_RTAS
960 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
961 * called with the MMU off.
963 * In addition, we need to be in 32b mode, at least for now.
965 * Note: r3 is an input parameter to rtas, so don't trash it...
970 stdu r1,-RTAS_FRAME_SIZE(r1) /* Save SP and create stack space. */
972 /* Because RTAS is running in 32b mode, it clobbers the high order half
973 * of all registers that it saves. We therefore save those registers
974 * RTAS might touch to the stack. (r0, r3-r13 are caller saved)
976 SAVE_GPR(2, r1) /* Save the TOC */
977 SAVE_GPR(13, r1) /* Save paca */
978 SAVE_8GPRS(14, r1) /* Save the non-volatiles */
979 SAVE_10GPRS(22, r1) /* ditto */
992 /* Temporary workaround to clear CR until RTAS can be modified to
999 /* There is no way it is acceptable to get here with interrupts enabled,
1000 * check it with the asm equivalent of WARN_ON
1002 lbz r0,PACASOFTIRQEN(r13)
1004 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
1007 /* Hard-disable interrupts */
1013 /* Unfortunately, the stack pointer and the MSR are also clobbered,
1014 * so they are saved in the PACA which allows us to restore
1015 * our original state after RTAS returns.
1018 std r6,PACASAVEDMSR(r13)
1020 /* Setup our real return addr */
1021 LOAD_REG_ADDR(r4,rtas_return_loc)
1022 clrldi r4,r4,2 /* convert to realmode address */
1026 ori r0,r0,MSR_EE|MSR_SE|MSR_BE|MSR_RI
1030 rldicr r9,r9,MSR_SF_LG,(63-MSR_SF_LG)
1031 ori r9,r9,MSR_IR|MSR_DR|MSR_FE0|MSR_FE1|MSR_FP|MSR_RI|MSR_LE
1033 sync /* disable interrupts so SRR0/1 */
1034 mtmsrd r0 /* don't get trashed */
1036 LOAD_REG_ADDR(r4, rtas)
1037 ld r5,RTASENTRY(r4) /* get the rtas->entry value */
1038 ld r4,RTASBASE(r4) /* get the rtas->base value */
1043 b . /* prevent speculative execution */
1048 /* relocation is off at this point */
1050 clrldi r4,r4,2 /* convert to realmode address */
1054 ld r3,(1f-0b)(r3) /* get &rtas_restore_regs */
1062 ld r1,PACAR1(r4) /* Restore our SP */
1063 ld r4,PACASAVEDMSR(r4) /* Restore our MSR */
1068 b . /* prevent speculative execution */
1071 1: .llong rtas_restore_regs
1074 /* relocation is on at this point */
1075 REST_GPR(2, r1) /* Restore the TOC */
1076 REST_GPR(13, r1) /* Restore paca */
1077 REST_8GPRS(14, r1) /* Restore the non-volatiles */
1078 REST_10GPRS(22, r1) /* ditto */
1093 addi r1,r1,RTAS_FRAME_SIZE /* Unstack our frame */
1094 ld r0,16(r1) /* get return address */
1097 blr /* return to caller */
1099 #endif /* CONFIG_PPC_RTAS */
1104 stdu r1,-PROM_FRAME_SIZE(r1) /* Save SP and create stack space */
1106 /* Because PROM is running in 32b mode, it clobbers the high order half
1107 * of all registers that it saves. We therefore save those registers
1108 * PROM might touch to the stack. (r0, r3-r13 are caller saved)
1119 /* Put PROM address in SRR0 */
1122 /* Setup our trampoline return addr in LR */
1125 addi r4,r4,(1f - 0b)
1128 /* Prepare a 32-bit mode big endian MSR
1130 #ifdef CONFIG_PPC_BOOK3E
1131 rlwinm r11,r11,0,1,31
1134 #else /* CONFIG_PPC_BOOK3E */
1135 LOAD_REG_IMMEDIATE(r12, MSR_SF | MSR_ISF | MSR_LE)
1139 #endif /* CONFIG_PPC_BOOK3E */
1141 1: /* Return from OF */
1144 /* Just make sure that r1 top 32 bits didn't get
1149 /* Restore the MSR (back to 64 bits) */
1154 /* Restore other registers */
1162 addi r1,r1,PROM_FRAME_SIZE
1167 #ifdef CONFIG_FUNCTION_TRACER
1168 #ifdef CONFIG_DYNAMIC_FTRACE
1176 #ifndef CC_USING_MPROFILE_KERNEL
1177 _GLOBAL_TOC(ftrace_caller)
1178 /* Taken from output of objdump from lib64/glibc */
1184 subi r3, r3, MCOUNT_INSN_SIZE
1189 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1190 .globl ftrace_graph_call
1193 _GLOBAL(ftrace_graph_stub)
1199 #else /* CC_USING_MPROFILE_KERNEL */
1202 * ftrace_caller() is the function that replaces _mcount() when ftrace is
1205 * We arrive here after a function A calls function B, and we are the trace
1206 * function for B. When we enter r1 points to A's stack frame, B has not yet
1207 * had a chance to allocate one yet.
1209 * Additionally r2 may point either to the TOC for A, or B, depending on
1210 * whether B did a TOC setup sequence before calling us.
1212 * On entry the LR points back to the _mcount() call site, and r0 holds the
1213 * saved LR as it was on entry to B, ie. the original return address at the
1216 * Our job is to save the register state into a struct pt_regs (on the stack)
1217 * and then arrange for the ftrace function to be called.
1219 _GLOBAL(ftrace_caller)
1220 /* Save the original return address in A's stack frame */
1223 /* Create our stack frame + pt_regs */
1224 stdu r1,-SWITCH_FRAME_SIZE(r1)
1226 /* Save all gprs to pt_regs */
1232 /* Load special regs for save below */
1238 /* Get the _mcount() call site out of LR */
1240 /* Save it as pt_regs->nip & pt_regs->link */
1244 /* Save callee's TOC in the ABI compliant location */
1246 ld r2,PACATOC(r13) /* get kernel TOC in r2 */
1248 addis r3,r2,function_trace_op@toc@ha
1249 addi r3,r3,function_trace_op@toc@l
1252 #ifdef CONFIG_LIVEPATCH
1253 mr r14,r7 /* remember old NIP */
1255 /* Calculate ip from nip-4 into r3 for call below */
1256 subi r3, r7, MCOUNT_INSN_SIZE
1258 /* Put the original return address in r4 as parent_ip */
1261 /* Save special regs */
1267 /* Load &pt_regs in r6 for call below */
1268 addi r6, r1 ,STACK_FRAME_OVERHEAD
1270 /* ftrace_call(r3, r4, r5, r6) */
1276 /* Load ctr with the possibly modified NIP */
1279 #ifdef CONFIG_LIVEPATCH
1280 cmpd r14,r3 /* has NIP been altered? */
1289 /* Restore callee's TOC */
1292 /* Pop our stack frame */
1293 addi r1, r1, SWITCH_FRAME_SIZE
1295 /* Restore original LR for return to B */
1299 #ifdef CONFIG_LIVEPATCH
1300 /* Based on the cmpd above, if the NIP was altered handle livepatch */
1301 bne- livepatch_handler
1304 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1306 .globl ftrace_graph_call
1309 _GLOBAL(ftrace_graph_stub)
1313 ld r0,LRSAVE(r1) /* restore callee's lr at _mcount site */
1315 bctr /* jump after _mcount site */
1316 #endif /* CC_USING_MPROFILE_KERNEL */
1318 _GLOBAL(ftrace_stub)
1321 #ifdef CONFIG_LIVEPATCH
1323 * This function runs in the mcount context, between two functions. As
1324 * such it can only clobber registers which are volatile and used in
1327 * We get here when a function A, calls another function B, but B has
1328 * been live patched with a new function C.
1331 * - we have no stack frame and can not allocate one
1332 * - LR points back to the original caller (in A)
1333 * - CTR holds the new NIP in C
1334 * - r0 & r12 are free
1336 * r0 can't be used as the base register for a DS-form load or store, so
1337 * we temporarily shuffle r1 (stack pointer) into r0 and then put it back.
1340 CURRENT_THREAD_INFO(r12, r1)
1342 /* Save stack pointer into r0 */
1345 /* Allocate 3 x 8 bytes */
1346 ld r1, TI_livepatch_sp(r12)
1348 std r1, TI_livepatch_sp(r12)
1350 /* Save toc & real LR on livepatch stack */
1355 /* Store stack end marker */
1356 lis r12, STACK_END_MAGIC@h
1357 ori r12, r12, STACK_END_MAGIC@l
1360 /* Restore real stack pointer */
1363 /* Put ctr in r12 for global entry and branch there */
1368 * Now we are returning from the patched function to the original
1369 * caller A. We are free to use r0 and r12, and we can use r2 until we
1373 CURRENT_THREAD_INFO(r12, r1)
1375 /* Save stack pointer into r0 */
1378 ld r1, TI_livepatch_sp(r12)
1380 /* Check stack marker hasn't been trashed */
1381 lis r2, STACK_END_MAGIC@h
1382 ori r2, r2, STACK_END_MAGIC@l
1385 EMIT_BUG_ENTRY 1b, __FILE__, __LINE__ - 1, 0
1387 /* Restore LR & toc from livepatch stack */
1392 /* Pop livepatch stack frame */
1393 CURRENT_THREAD_INFO(r12, r0)
1395 std r1, TI_livepatch_sp(r12)
1397 /* Restore real stack pointer */
1400 /* Return to original caller of live patched function */
1406 _GLOBAL_TOC(_mcount)
1407 /* Taken from output of objdump from lib64/glibc */
1414 subi r3, r3, MCOUNT_INSN_SIZE
1415 LOAD_REG_ADDR(r5,ftrace_trace_function)
1423 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1424 b ftrace_graph_caller
1429 _GLOBAL(ftrace_stub)
1432 #endif /* CONFIG_DYNAMIC_FTRACE */
1434 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1435 #ifndef CC_USING_MPROFILE_KERNEL
1436 _GLOBAL(ftrace_graph_caller)
1437 /* load r4 with local address */
1439 subi r4, r4, MCOUNT_INSN_SIZE
1441 /* Grab the LR out of the caller stack frame */
1445 bl prepare_ftrace_return
1449 * prepare_ftrace_return gives us the address we divert to.
1450 * Change the LR in the callers stack frame to this.
1460 #else /* CC_USING_MPROFILE_KERNEL */
1461 _GLOBAL(ftrace_graph_caller)
1462 /* with -mprofile-kernel, parameter regs are still alive at _mcount */
1472 /* Save callee's TOC in the ABI compliant location */
1474 ld r2, PACATOC(r13) /* get kernel TOC in r2 */
1476 mfctr r4 /* ftrace_caller has moved local addr here */
1478 mflr r3 /* ftrace_caller has restored LR from stack */
1479 subi r4, r4, MCOUNT_INSN_SIZE
1481 bl prepare_ftrace_return
1485 * prepare_ftrace_return gives us the address we divert to.
1486 * Change the LR to this.
1501 /* Restore callee's TOC */
1508 #endif /* CC_USING_MPROFILE_KERNEL */
1510 _GLOBAL(return_to_handler)
1511 /* need to save return values */
1521 * We might be called from a module.
1522 * Switch to our TOC to run inside the core kernel.
1526 bl ftrace_return_to_handler
1529 /* return value has real return address */
1538 /* Jump back to real return address */
1540 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1541 #endif /* CONFIG_FUNCTION_TRACER */