* Table is based upon EAS2.6 (Oct 1999)
*/
-#include <linux/config.h>
#include <asm/asmmacro.h>
#include <asm/break.h>
* (the "original") TLB miss, which may either be caused by an instruction
* fetch or a data access (or non-access).
*
- * What we do here is normal TLB miss handing for the _original_ miss, followed
- * by inserting the TLB entry for the virtual page table page that the VHPT
- * walker was attempting to access. The latter gets inserted as long
- * as both L1 and L2 have valid mappings for the faulting address.
- * The TLB entry for the original miss gets inserted only if
- * the L3 entry indicates that the page is present.
+ * What we do here is normal TLB miss handing for the _original_ miss,
+ * followed by inserting the TLB entry for the virtual page table page
+ * that the VHPT walker was attempting to access. The latter gets
+ * inserted as long as page table entry above pte level have valid
+ * mappings for the faulting address. The TLB entry for the original
+ * miss gets inserted only if the pte entry indicates that the page is
+ * present.
*
* do_page_fault gets invoked in the following cases:
* - the faulting virtual address uses unimplemented address bits
- * - the faulting virtual address has no L1, L2, or L3 mapping
+ * - the faulting virtual address has no valid page table mapping
*/
mov r16=cr.ifa // get address that caused the TLB miss
#ifdef CONFIG_HUGETLB_PAGE
shl r21=r16,3 // shift bit 60 into sign bit
shr.u r17=r16,61 // get the region number into r17
;;
- shr r22=r21,3
+ shr.u r22=r21,3
#ifdef CONFIG_HUGETLB_PAGE
extr.u r26=r25,2,6
;;
#endif
;;
cmp.eq p6,p7=5,r17 // is IFA pointing into to region 5?
- shr.u r18=r22,PGDIR_SHIFT // get bits 33-63 of the faulting address
+ shr.u r18=r22,PGDIR_SHIFT // get bottom portion of pgd index bit
;;
(p7) dep r17=r17,r19,(PAGE_SHIFT-3),3 // put region number bits in place
(p6) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT
(p7) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT-3
;;
-(p6) dep r17=r18,r19,3,(PAGE_SHIFT-3) // r17=PTA + IFA(33,42)*8
-(p7) dep r17=r18,r17,3,(PAGE_SHIFT-6) // r17=PTA + (((IFA(61,63) << 7) | IFA(33,39))*8)
+(p6) dep r17=r18,r19,3,(PAGE_SHIFT-3) // r17=pgd_offset for region 5
+(p7) dep r17=r18,r17,3,(PAGE_SHIFT-6) // r17=pgd_offset for region[0-4]
cmp.eq p7,p6=0,r21 // unused address bits all zeroes?
- shr.u r18=r22,PMD_SHIFT // shift L2 index into position
+#ifdef CONFIG_PGTABLE_4
+ shr.u r28=r22,PUD_SHIFT // shift pud index into position
+#else
+ shr.u r18=r22,PMD_SHIFT // shift pmd index into position
+#endif
+ ;;
+ ld8 r17=[r17] // get *pgd (may be 0)
;;
- ld8 r17=[r17] // fetch the L1 entry (may be 0)
+(p7) cmp.eq p6,p7=r17,r0 // was pgd_present(*pgd) == NULL?
+#ifdef CONFIG_PGTABLE_4
+ dep r28=r28,r17,3,(PAGE_SHIFT-3) // r28=pud_offset(pgd,addr)
;;
-(p7) cmp.eq p6,p7=r17,r0 // was L1 entry NULL?
- dep r17=r18,r17,3,(PAGE_SHIFT-3) // compute address of L2 page table entry
+ shr.u r18=r22,PMD_SHIFT // shift pmd index into position
+(p7) ld8 r29=[r28] // get *pud (may be 0)
+ ;;
+(p7) cmp.eq.or.andcm p6,p7=r29,r0 // was pud_present(*pud) == NULL?
+ dep r17=r18,r29,3,(PAGE_SHIFT-3) // r17=pmd_offset(pud,addr)
+#else
+ dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=pmd_offset(pgd,addr)
+#endif
;;
-(p7) ld8 r20=[r17] // fetch the L2 entry (may be 0)
- shr.u r19=r22,PAGE_SHIFT // shift L3 index into position
+(p7) ld8 r20=[r17] // get *pmd (may be 0)
+ shr.u r19=r22,PAGE_SHIFT // shift pte index into position
;;
-(p7) cmp.eq.or.andcm p6,p7=r20,r0 // was L2 entry NULL?
- dep r21=r19,r20,3,(PAGE_SHIFT-3) // compute address of L3 page table entry
+(p7) cmp.eq.or.andcm p6,p7=r20,r0 // was pmd_present(*pmd) == NULL?
+ dep r21=r19,r20,3,(PAGE_SHIFT-3) // r21=pte_offset(pmd,addr)
;;
-(p7) ld8 r18=[r21] // read the L3 PTE
- mov r19=cr.isr // cr.isr bit 0 tells us if this is an insn miss
+(p7) ld8 r18=[r21] // read *pte
+ mov r19=cr.isr // cr.isr bit 32 tells us if this is an insn miss
;;
(p7) tbit.z p6,p7=r18,_PAGE_P_BIT // page present bit cleared?
mov r22=cr.iha // get the VHPT address that caused the TLB miss
dv_serialize_data
/*
- * Re-check L2 and L3 pagetable. If they changed, we may have received a ptc.g
+ * Re-check pagetable entry. If they changed, we may have received a ptc.g
* between reading the pagetable and the "itc". If so, flush the entry we
- * inserted and retry.
+ * inserted and retry. At this point, we have:
+ *
+ * r28 = equivalent of pud_offset(pgd, ifa)
+ * r17 = equivalent of pmd_offset(pud, ifa)
+ * r21 = equivalent of pte_offset(pmd, ifa)
+ *
+ * r29 = *pud
+ * r20 = *pmd
+ * r18 = *pte
*/
- ld8 r25=[r21] // read L3 PTE again
- ld8 r26=[r17] // read L2 entry again
+ ld8 r25=[r21] // read *pte again
+ ld8 r26=[r17] // read *pmd again
+#ifdef CONFIG_PGTABLE_4
+ ld8 r19=[r28] // read *pud again
+#endif
+ cmp.ne p6,p7=r0,r0
;;
- cmp.ne p6,p7=r26,r20 // did L2 entry change
+ cmp.ne.or.andcm p6,p7=r26,r20 // did *pmd change
+#ifdef CONFIG_PGTABLE_4
+ cmp.ne.or.andcm p6,p7=r19,r29 // did *pud change
+#endif
mov r27=PAGE_SHIFT<<2
;;
(p6) ptc.l r22,r27 // purge PTE page translation
-(p7) cmp.ne.or.andcm p6,p7=r25,r18 // did L3 PTE change
+(p7) cmp.ne.or.andcm p6,p7=r25,r18 // did *pte change
;;
(p6) ptc.l r16,r27 // purge translation
#endif
ENTRY(itlb_miss)
DBG_FAULT(1)
/*
- * The ITLB handler accesses the L3 PTE via the virtually mapped linear
+ * The ITLB handler accesses the PTE via the virtually mapped linear
* page table. If a nested TLB miss occurs, we switch into physical
- * mode, walk the page table, and then re-execute the L3 PTE read
- * and go on normally after that.
+ * mode, walk the page table, and then re-execute the PTE read and
+ * go on normally after that.
*/
mov r16=cr.ifa // get virtual address
mov r29=b0 // save b0
mov r31=pr // save predicates
.itlb_fault:
- mov r17=cr.iha // get virtual address of L3 PTE
+ mov r17=cr.iha // get virtual address of PTE
movl r30=1f // load nested fault continuation point
;;
-1: ld8 r18=[r17] // read L3 PTE
+1: ld8 r18=[r17] // read *pte
;;
mov b0=r29
tbit.z p6,p0=r18,_PAGE_P_BIT // page present bit cleared?
*/
dv_serialize_data
- ld8 r19=[r17] // read L3 PTE again and see if same
+ ld8 r19=[r17] // read *pte again and see if same
mov r20=PAGE_SHIFT<<2 // setup page size for purge
;;
cmp.ne p7,p0=r18,r19
ENTRY(dtlb_miss)
DBG_FAULT(2)
/*
- * The DTLB handler accesses the L3 PTE via the virtually mapped linear
+ * The DTLB handler accesses the PTE via the virtually mapped linear
* page table. If a nested TLB miss occurs, we switch into physical
- * mode, walk the page table, and then re-execute the L3 PTE read
- * and go on normally after that.
+ * mode, walk the page table, and then re-execute the PTE read and
+ * go on normally after that.
*/
mov r16=cr.ifa // get virtual address
mov r29=b0 // save b0
mov r31=pr // save predicates
dtlb_fault:
- mov r17=cr.iha // get virtual address of L3 PTE
+ mov r17=cr.iha // get virtual address of PTE
movl r30=1f // load nested fault continuation point
;;
-1: ld8 r18=[r17] // read L3 PTE
+1: ld8 r18=[r17] // read *pte
;;
mov b0=r29
tbit.z p6,p0=r18,_PAGE_P_BIT // page present bit cleared?
*/
dv_serialize_data
- ld8 r19=[r17] // read L3 PTE again and see if same
+ ld8 r19=[r17] // read *pte again and see if same
mov r20=PAGE_SHIFT<<2 // setup page size for purge
;;
cmp.ne p7,p0=r18,r19
movl r19=(((1 << IA64_MAX_PHYS_BITS) - 1) & ~0xfff)
mov r21=cr.ipsr
mov r31=pr
+ mov r24=PERCPU_ADDR
;;
#ifdef CONFIG_DISABLE_VHPT
shr.u r22=r16,61 // get the region number into r21
(p8) mov r29=b0 // save b0
(p8) br.cond.dptk dtlb_fault
#endif
+ cmp.ge p10,p11=r16,r24 // access to per_cpu_data?
+ tbit.z p12,p0=r16,61 // access to region 6?
+ mov r25=PERCPU_PAGE_SHIFT << 2
+ mov r26=PERCPU_PAGE_SIZE
+ nop.m 0
+ nop.b 0
+ ;;
+(p10) mov r19=IA64_KR(PER_CPU_DATA)
+(p11) and r19=r19,r16 // clear non-ppn fields
extr.u r23=r21,IA64_PSR_CPL0_BIT,2 // extract psr.cpl
and r22=IA64_ISR_CODE_MASK,r20 // get the isr.code field
tbit.nz p6,p7=r20,IA64_ISR_SP_BIT // is speculation bit on?
- shr.u r18=r16,57 // move address bit 61 to bit 4
- and r19=r19,r16 // clear ed, reserved bits, and PTE control bits
tbit.nz p9,p0=r20,IA64_ISR_NA_BIT // is non-access bit on?
;;
- andcm r18=0x10,r18 // bit 4=~address-bit(61)
+(p10) sub r19=r19,r26
+(p10) mov cr.itir=r25
cmp.ne p8,p0=r0,r23
(p9) cmp.eq.or.andcm p6,p7=IA64_ISR_CODE_LFETCH,r22 // check isr.code field
+(p12) dep r17=-1,r17,4,1 // set ma=UC for region 6 addr
(p8) br.cond.spnt page_fault
dep r21=-1,r21,IA64_PSR_ED_BIT,1
- or r19=r19,r17 // insert PTE control bits into r19
;;
- or r19=r19,r18 // set bit 4 (uncached) if the access was to region 6
+ or r19=r19,r17 // insert PTE control bits into r19
(p6) mov cr.ipsr=r21
;;
(p7) itc.d r19 // insert the TLB entry
* r30: continuation address
* r31: saved pr
*
- * Output: r17: physical address of L3 PTE of faulting address
+ * Output: r17: physical address of PTE of faulting address
* r29: saved b0
* r30: continuation address
* r31: saved pr
(p6) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT
(p7) shr.u r21=r21,PGDIR_SHIFT+PAGE_SHIFT-3
;;
-(p6) dep r17=r18,r19,3,(PAGE_SHIFT-3) // r17=PTA + IFA(33,42)*8
-(p7) dep r17=r18,r17,3,(PAGE_SHIFT-6) // r17=PTA + (((IFA(61,63) << 7) | IFA(33,39))*8)
+(p6) dep r17=r18,r19,3,(PAGE_SHIFT-3) // r17=pgd_offset for region 5
+(p7) dep r17=r18,r17,3,(PAGE_SHIFT-6) // r17=pgd_offset for region[0-4]
cmp.eq p7,p6=0,r21 // unused address bits all zeroes?
- shr.u r18=r22,PMD_SHIFT // shift L2 index into position
+#ifdef CONFIG_PGTABLE_4
+ shr.u r18=r22,PUD_SHIFT // shift pud index into position
+#else
+ shr.u r18=r22,PMD_SHIFT // shift pmd index into position
+#endif
+ ;;
+ ld8 r17=[r17] // get *pgd (may be 0)
;;
- ld8 r17=[r17] // fetch the L1 entry (may be 0)
+(p7) cmp.eq p6,p7=r17,r0 // was pgd_present(*pgd) == NULL?
+ dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=p[u|m]d_offset(pgd,addr)
;;
-(p7) cmp.eq p6,p7=r17,r0 // was L1 entry NULL?
- dep r17=r18,r17,3,(PAGE_SHIFT-3) // compute address of L2 page table entry
+#ifdef CONFIG_PGTABLE_4
+(p7) ld8 r17=[r17] // get *pud (may be 0)
+ shr.u r18=r22,PMD_SHIFT // shift pmd index into position
;;
-(p7) ld8 r17=[r17] // fetch the L2 entry (may be 0)
- shr.u r19=r22,PAGE_SHIFT // shift L3 index into position
+(p7) cmp.eq.or.andcm p6,p7=r17,r0 // was pud_present(*pud) == NULL?
+ dep r17=r18,r17,3,(PAGE_SHIFT-3) // r17=pmd_offset(pud,addr)
;;
-(p7) cmp.eq.or.andcm p6,p7=r17,r0 // was L2 entry NULL?
- dep r17=r19,r17,3,(PAGE_SHIFT-3) // compute address of L3 page table entry
+#endif
+(p7) ld8 r17=[r17] // get *pmd (may be 0)
+ shr.u r19=r22,PAGE_SHIFT // shift pte index into position
+ ;;
+(p7) cmp.eq.or.andcm p6,p7=r17,r0 // was pmd_present(*pmd) == NULL?
+ dep r17=r19,r17,3,(PAGE_SHIFT-3) // r17=pte_offset(pmd,addr);
(p6) br.cond.spnt page_fault
mov b0=r30
br.sptk.many b0 // return to continuation point
;; // avoid RAW on r18
mov ar.ccv=r18 // set compare value for cmpxchg
or r25=_PAGE_D|_PAGE_A,r18 // set the dirty and accessed bits
+ tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit
;;
- cmpxchg8.acq r26=[r17],r25,ar.ccv
+(p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only update if page is present
mov r24=PAGE_SHIFT<<2
;;
- cmp.eq p6,p7=r26,r18
+(p6) cmp.eq p6,p7=r26,r18 // Only compare if page is present
;;
(p6) itc.d r25 // install updated PTE
;;
;;
mov ar.ccv=r18 // set compare value for cmpxchg
or r25=_PAGE_A,r18 // set the accessed bit
+ tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit
;;
- cmpxchg8.acq r26=[r17],r25,ar.ccv
+(p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only if page present
mov r24=PAGE_SHIFT<<2
;;
- cmp.eq p6,p7=r26,r18
+(p6) cmp.eq p6,p7=r26,r18 // Only if page present
;;
(p6) itc.i r25 // install updated PTE
;;
;; // avoid RAW on r18
mov ar.ccv=r18 // set compare value for cmpxchg
or r25=_PAGE_A,r18 // set the dirty bit
+ tbit.z p7,p6 = r18,_PAGE_P_BIT // Check present bit
;;
- cmpxchg8.acq r26=[r17],r25,ar.ccv
+(p6) cmpxchg8.acq r26=[r17],r25,ar.ccv // Only if page is present
mov r24=PAGE_SHIFT<<2
;;
- cmp.eq p6,p7=r26,r18
+(p6) cmp.eq p6,p7=r26,r18 // Only if page is present
;;
(p6) itc.d r25 // install updated PTE
/*
;;
SAVE_REST
;;
+ MCA_RECOVER_RANGE(interrupt)
alloc r14=ar.pfs,0,0,2,0 // must be first in an insn group
mov out0=cr.ivr // pass cr.ivr as first arg
add out1=16,sp // pass pointer to pt_regs as second arg
// 0x5a00 Entry 30 (size 16 bundles) Unaligned Reference (57)
ENTRY(unaligned_access)
DBG_FAULT(30)
- mov r16=cr.ipsr
mov r31=pr // prepare to save predicates
;;
br.sptk.many dispatch_unaligned_handler
projects / cascardo / linux.git / blobdiff