630 lines
18 KiB
ArmAsm
630 lines
18 KiB
ArmAsm
|
/* tlb-miss.S: TLB miss handlers
|
||
|
|
*
|
||
|
|
* Copyright (C) 2004 Red Hat, Inc. All Rights Reserved.
|
||
|
|
* Written by David Howells (dhowells@redhat.com)
|
||
|
|
*
|
||
|
|
* This program is free software; you can redistribute it and/or
|
||
|
|
* modify it under the terms of the GNU General Public License
|
||
|
|
* as published by the Free Software Foundation; either version
|
||
|
|
* 2 of the License, or (at your option) any later version.
|
||
|
|
*/
|
||
|
|
|
||
|
|
#include <linux/sys.h>
|
||
|
|
#include <linux/linkage.h>
|
||
|
|
#include <asm/page.h>
|
||
|
|
#include <asm/pgtable.h>
|
||
|
|
#include <asm/spr-regs.h>
|
||
|
|
|
||
|
|
.section .text..tlbmiss
|
||
|
|
.balign 4
|
||
|
|
|
||
|
|
.globl __entry_insn_mmu_miss
|
||
|
|
__entry_insn_mmu_miss:
|
||
|
|
break
|
||
|
|
nop
|
||
|
|
|
||
|
|
.globl __entry_insn_mmu_exception
|
||
|
|
__entry_insn_mmu_exception:
|
||
|
|
break
|
||
|
|
nop
|
||
|
|
|
||
|
|
.globl __entry_data_mmu_miss
|
||
|
|
__entry_data_mmu_miss:
|
||
|
|
break
|
||
|
|
nop
|
||
|
|
|
||
|
|
.globl __entry_data_mmu_exception
|
||
|
|
__entry_data_mmu_exception:
|
||
|
|
break
|
||
|
|
nop
|
||
|
|
|
||
|
|
###############################################################################
|
||
|
|
#
|
||
|
|
# handle a lookup failure of one sort or another in a kernel TLB handler
|
||
|
|
# On entry:
|
||
|
|
# GR29 - faulting address
|
||
|
|
# SCR2 - saved CCR
|
||
|
|
#
|
||
|
|
###############################################################################
|
||
|
|
.type __tlb_kernel_fault,@function
|
||
|
|
__tlb_kernel_fault:
|
||
|
|
# see if we're supposed to re-enable single-step mode upon return
|
||
|
|
sethi.p %hi(__break_tlb_miss_return_break),gr30
|
||
|
|
setlo %lo(__break_tlb_miss_return_break),gr30
|
||
|
|
movsg pcsr,gr31
|
||
|
|
|
||
|
|
subcc gr31,gr30,gr0,icc0
|
||
|
|
beq icc0,#0,__tlb_kernel_fault_sstep
|
||
|
|
|
||
|
|
movsg scr2,gr30
|
||
|
|
movgs gr30,ccr
|
||
|
|
movgs gr29,scr2 /* save EAR0 value */
|
||
|
|
sethi.p %hi(__kernel_current_task),gr29
|
||
|
|
setlo %lo(__kernel_current_task),gr29
|
||
|
|
ldi.p @(gr29,#0),gr29 /* restore GR29 */
|
||
|
|
|
||
|
|
bra __entry_kernel_handle_mmu_fault
|
||
|
|
|
||
|
|
# we've got to re-enable single-stepping
|
||
|
|
__tlb_kernel_fault_sstep:
|
||
|
|
sethi.p %hi(__break_tlb_miss_real_return_info),gr30
|
||
|
|
setlo %lo(__break_tlb_miss_real_return_info),gr30
|
||
|
|
lddi @(gr30,0),gr30
|
||
|
|
movgs gr30,pcsr
|
||
|
|
movgs gr31,psr
|
||
|
|
|
||
|
|
movsg scr2,gr30
|
||
|
|
movgs gr30,ccr
|
||
|
|
movgs gr29,scr2 /* save EAR0 value */
|
||
|
|
sethi.p %hi(__kernel_current_task),gr29
|
||
|
|
setlo %lo(__kernel_current_task),gr29
|
||
|
|
ldi.p @(gr29,#0),gr29 /* restore GR29 */
|
||
|
|
bra __entry_kernel_handle_mmu_fault_sstep
|
||
|
|
|
||
|
|
.size __tlb_kernel_fault, .-__tlb_kernel_fault
|
||
|
|
|
||
|
|
###############################################################################
|
||
|
|
#
|
||
|
|
# handle a lookup failure of one sort or another in a user TLB handler
|
||
|
|
# On entry:
|
||
|
|
# GR28 - faulting address
|
||
|
|
# SCR2 - saved CCR
|
||
|
|
#
|
||
|
|
###############################################################################
|
||
|
|
.type __tlb_user_fault,@function
|
||
|
|
__tlb_user_fault:
|
||
|
|
# see if we're supposed to re-enable single-step mode upon return
|
||
|
|
sethi.p %hi(__break_tlb_miss_return_break),gr30
|
||
|
|
setlo %lo(__break_tlb_miss_return_break),gr30
|
||
|
|
movsg pcsr,gr31
|
||
|
|
subcc gr31,gr30,gr0,icc0
|
||
|
|
beq icc0,#0,__tlb_user_fault_sstep
|
||
|
|
|
||
|
|
movsg scr2,gr30
|
||
|
|
movgs gr30,ccr
|
||
|
|
bra __entry_uspace_handle_mmu_fault
|
||
|
|
|
||
|
|
# we've got to re-enable single-stepping
|
||
|
|
__tlb_user_fault_sstep:
|
||
|
|
sethi.p %hi(__break_tlb_miss_real_return_info),gr30
|
||
|
|
setlo %lo(__break_tlb_miss_real_return_info),gr30
|
||
|
|
lddi @(gr30,0),gr30
|
||
|
|
movgs gr30,pcsr
|
||
|
|
movgs gr31,psr
|
||
|
|
movsg scr2,gr30
|
||
|
|
movgs gr30,ccr
|
||
|
|
bra __entry_uspace_handle_mmu_fault_sstep
|
||
|
|
|
||
|
|
.size __tlb_user_fault, .-__tlb_user_fault
|
||
|
|
|
||
|
|
###############################################################################
|
||
|
|
#
|
||
|
|
# Kernel instruction TLB miss handler
|
||
|
|
# On entry:
|
||
|
|
# GR1 - kernel stack pointer
|
||
|
|
# GR28 - saved exception frame pointer
|
||
|
|
# GR29 - faulting address
|
||
|
|
# GR31 - EAR0 ^ SCR0
|
||
|
|
# SCR0 - base of virtual range covered by cached PGE from last ITLB miss (or 0xffffffff)
|
||
|
|
# DAMR3 - mapped page directory
|
||
|
|
# DAMR4 - mapped page table as matched by SCR0
|
||
|
|
#
|
||
|
|
###############################################################################
|
||
|
|
.globl __entry_kernel_insn_tlb_miss
|
||
|
|
.type __entry_kernel_insn_tlb_miss,@function
|
||
|
|
__entry_kernel_insn_tlb_miss:
|
||
|
|
#if 0
|
||
|
|
sethi.p %hi(0xe1200004),gr30
|
||
|
|
setlo %lo(0xe1200004),gr30
|
||
|
|
st gr0,@(gr30,gr0)
|
||
|
|
sethi.p %hi(0xffc00100),gr30
|
||
|
|
setlo %lo(0xffc00100),gr30
|
||
|
|
sth gr30,@(gr30,gr0)
|
||
|
|
membar
|
||
|
|
#endif
|
||
|
|
|
||
|
|
movsg ccr,gr30 /* save CCR */
|
||
|
|
movgs gr30,scr2
|
||
|
|
|
||
|
|
# see if the cached page table mapping is appropriate
|
||
|
|
srlicc.p gr31,#26,gr0,icc0
|
||
|
|
setlos 0x3ffc,gr30
|
||
|
|
srli.p gr29,#12,gr31 /* use EAR0[25:14] as PTE index */
|
||
|
|
bne icc0,#0,__itlb_k_PTD_miss
|
||
|
|
|
||
|
|
__itlb_k_PTD_mapped:
|
||
|
|
# access the PTD with EAR0[25:14]
|
||
|
|
# - DAMLR4 points to the virtual address of the appropriate page table
|
||
|
|
# - the PTD holds 4096 PTEs
|
||
|
|
# - the PTD must be accessed uncached
|
||
|
|
# - the PTE must be marked accessed if it was valid
|
||
|
|
#
|
||
|
|
and gr31,gr30,gr31
|
||
|
|
movsg damlr4,gr30
|
||
|
|
add gr30,gr31,gr31
|
||
|
|
ldi @(gr31,#0),gr30 /* fetch the PTE */
|
||
|
|
andicc gr30,#_PAGE_PRESENT,gr0,icc0
|
||
|
|
ori.p gr30,#_PAGE_ACCESSED,gr30
|
||
|
|
beq icc0,#0,__tlb_kernel_fault /* jump if PTE invalid */
|
||
|
|
sti.p gr30,@(gr31,#0) /* update the PTE */
|
||
|
|
andi gr30,#~_PAGE_ACCESSED,gr30
|
||
|
|
|
||
|
|
# we're using IAMR1 as an extra TLB entry
|
||
|
|
# - punt the entry here (if valid) to the real TLB and then replace with the new PTE
|
||
|
|
# - need to check DAMR1 lest we cause an multiple-DAT-hit exception
|
||
|
|
# - IAMPR1 has no WP bit, and we mustn't lose WP information
|
||
|
|
movsg iampr1,gr31
|
||
|
|
andicc gr31,#xAMPRx_V,gr0,icc0
|
||
|
|
setlos.p 0xfffff000,gr31
|
||
|
|
beq icc0,#0,__itlb_k_nopunt /* punt not required */
|
||
|
|
|
||
|
|
movsg iamlr1,gr31
|
||
|
|
movgs gr31,tplr /* set TPLR.CXN */
|
||
|
|
tlbpr gr31,gr0,#4,#0 /* delete matches from TLB, IAMR1, DAMR1 */
|
||
|
|
|
||
|
|
movsg dampr1,gr31
|
||
|
|
ori gr31,#xAMPRx_V,gr31 /* entry was invalidated by tlbpr #4 */
|
||
|
|
movgs gr31,tppr
|
||
|
|
movsg iamlr1,gr31 /* set TPLR.CXN */
|
||
|
|
movgs gr31,tplr
|
||
|
|
tlbpr gr31,gr0,#2,#0 /* save to the TLB */
|
||
|
|
movsg tpxr,gr31 /* check the TLB write error flag */
|
||
|
|
andicc.p gr31,#TPXR_E,gr0,icc0
|
||
|
|
setlos #0xfffff000,gr31
|
||
|
|
bne icc0,#0,__tlb_kernel_fault
|
||
|
|
|
||
|
|
__itlb_k_nopunt:
|
||
|
|
|
||
|
|
# assemble the new TLB entry
|
||
|
|
and gr29,gr31,gr29
|
||
|
|
movsg cxnr,gr31
|
||
|
|
or gr29,gr31,gr29
|
||
|
|
movgs gr29,iamlr1 /* xAMLR = address | context number */
|
||
|
|
movgs gr30,iampr1
|
||
|
|
movgs gr29,damlr1
|
||
|
|
movgs gr30,dampr1
|
||
|
|
|
||
|
|
# return, restoring registers
|
||
|
|
movsg scr2,gr30
|
||
|
|
movgs gr30,ccr
|
||
|
|
sethi.p %hi(__kernel_current_task),gr29
|
||
|
|
setlo %lo(__kernel_current_task),gr29
|
||
|
|
ldi @(gr29,#0),gr29
|
||
|
|
rett #0
|
||
|
|
beq icc0,#3,0 /* prevent icache prefetch */
|
||
|
|
|
||
|
|
# the PTE we want wasn't in the PTD we have mapped, so we need to go looking for a more
|
||
|
|
# appropriate page table and map that instead
|
||
|
|
# - access the PGD with EAR0[31:26]
|
||
|
|
# - DAMLR3 points to the virtual address of the page directory
|
||
|
|
# - the PGD holds 64 PGEs and each PGE/PME points to a set of page tables
|
||
|
|
__itlb_k_PTD_miss:
|
||
|
|
srli gr29,#26,gr31 /* calculate PGE offset */
|
||
|
|
slli gr31,#8,gr31 /* and clear bottom bits */
|
||
|
|
|
||
|
|
movsg damlr3,gr30
|
||
|
|
ld @(gr31,gr30),gr30 /* access the PGE */
|
||
|
|
|
||
|
|
andicc.p gr30,#_PAGE_PRESENT,gr0,icc0
|
||
|
|
andicc gr30,#xAMPRx_SS,gr0,icc1
|
||
|
|
|
||
|
|
# map this PTD instead and record coverage address
|
||
|
|
ori.p gr30,#xAMPRx_L|xAMPRx_SS_16Kb|xAMPRx_S|xAMPRx_C|xAMPRx_V,gr30
|
||
|
|
beq icc0,#0,__tlb_kernel_fault /* jump if PGE not present */
|
||
|
|
slli.p gr31,#18,gr31
|
||
|
|
bne icc1,#0,__itlb_k_bigpage
|
||
|
|
movgs gr30,dampr4
|
||
|
|
movgs gr31,scr0
|
||
|
|
|
||
|
|
# we can now resume normal service
|
||
|
|
setlos 0x3ffc,gr30
|
||
|
|
srli.p gr29,#12,gr31 /* use EAR0[25:14] as PTE index */
|
||
|
|
bra __itlb_k_PTD_mapped
|
||
|
|
|
||
|
|
__itlb_k_bigpage:
|
||
|
|
break
|
||
|
|
nop
|
||
|
|
|
||
|
|
.size __entry_kernel_insn_tlb_miss, .-__entry_kernel_insn_tlb_miss
|
||
|
|
|
||
|
|
###############################################################################
|
||
|
|
#
|
||
|
|
# Kernel data TLB miss handler
|
||
|
|
# On entry:
|
||
|
|
# GR1 - kernel stack pointer
|
||
|
|
# GR28 - saved exception frame pointer
|
||
|
|
# GR29 - faulting address
|
||
|
|
# GR31 - EAR0 ^ SCR1
|
||
|
|
# SCR1 - base of virtual range covered by cached PGE from last DTLB miss (or 0xffffffff)
|
||
|
|
# DAMR3 - mapped page directory
|
||
|
|
# DAMR5 - mapped page table as matched by SCR1
|
||
|
|
#
|
||
|
|
###############################################################################
|
||
|
|
.globl __entry_kernel_data_tlb_miss
|
||
|
|
.type __entry_kernel_data_tlb_miss,@function
|
||
|
|
__entry_kernel_data_tlb_miss:
|
||
|
|
#if 0
|
||
|
|
sethi.p %hi(0xe1200004),gr30
|
||
|
|
setlo %lo(0xe1200004),gr30
|
||
|
|
st gr0,@(gr30,gr0)
|
||
|
|
sethi.p %hi(0xffc00100),gr30
|
||
|
|
setlo %lo(0xffc00100),gr30
|
||
|
|
sth gr30,@(gr30,gr0)
|
||
|
|
membar
|
||
|
|
#endif
|
||
|
|
|
||
|
|
movsg ccr,gr30 /* save CCR */
|
||
|
|
movgs gr30,scr2
|
||
|
|
|
||
|
|
# see if the cached page table mapping is appropriate
|
||
|
|
srlicc.p gr31,#26,gr0,icc0
|
||
|
|
setlos 0x3ffc,gr30
|
||
|
|
srli.p gr29,#12,gr31 /* use EAR0[25:14] as PTE index */
|
||
|
|
bne icc0,#0,__dtlb_k_PTD_miss
|
||
|
|
|
||
|
|
__dtlb_k_PTD_mapped:
|
||
|
|
# access the PTD with EAR0[25:14]
|
||
|
|
# - DAMLR5 points to the virtual address of the appropriate page table
|
||
|
|
# - the PTD holds 4096 PTEs
|
||
|
|
# - the PTD must be accessed uncached
|
||
|
|
# - the PTE must be marked accessed if it was valid
|
||
|
|
#
|
||
|
|
and gr31,gr30,gr31
|
||
|
|
movsg damlr5,gr30
|
||
|
|
add gr30,gr31,gr31
|
||
|
|
ldi @(gr31,#0),gr30 /* fetch the PTE */
|
||
|
|
andicc gr30,#_PAGE_PRESENT,gr0,icc0
|
||
|
|
ori.p gr30,#_PAGE_ACCESSED,gr30
|
||
|
|
beq icc0,#0,__tlb_kernel_fault /* jump if PTE invalid */
|
||
|
|
sti.p gr30,@(gr31,#0) /* update the PTE */
|
||
|
|
andi gr30,#~_PAGE_ACCESSED,gr30
|
||
|
|
|
||
|
|
# we're using DAMR1 as an extra TLB entry
|
||
|
|
# - punt the entry here (if valid) to the real TLB and then replace with the new PTE
|
||
|
|
# - need to check IAMR1 lest we cause an multiple-DAT-hit exception
|
||
|
|
movsg dampr1,gr31
|
||
|
|
andicc gr31,#xAMPRx_V,gr0,icc0
|
||
|
|
setlos.p 0xfffff000,gr31
|
||
|
|
beq icc0,#0,__dtlb_k_nopunt /* punt not required */
|
||
|
|
|
||
|
|
movsg damlr1,gr31
|
||
|
|
movgs gr31,tplr /* set TPLR.CXN */
|
||
|
|
tlbpr gr31,gr0,#4,#0 /* delete matches from TLB, IAMR1, DAMR1 */
|
||
|
|
|
||
|
|
movsg dampr1,gr31
|
||
|
|
ori gr31,#xAMPRx_V,gr31 /* entry was invalidated by tlbpr #4 */
|
||
|
|
movgs gr31,tppr
|
||
|
|
movsg damlr1,gr31 /* set TPLR.CXN */
|
||
|
|
movgs gr31,tplr
|
||
|
|
tlbpr gr31,gr0,#2,#0 /* save to the TLB */
|
||
|
|
movsg tpxr,gr31 /* check the TLB write error flag */
|
||
|
|
andicc.p gr31,#TPXR_E,gr0,icc0
|
||
|
|
setlos #0xfffff000,gr31
|
||
|
|
bne icc0,#0,__tlb_kernel_fault
|
||
|
|
|
||
|
|
__dtlb_k_nopunt:
|
||
|
|
|
||
|
|
# assemble the new TLB entry
|
||
|
|
and gr29,gr31,gr29
|
||
|
|
movsg cxnr,gr31
|
||
|
|
or gr29,gr31,gr29
|
||
|
|
movgs gr29,iamlr1 /* xAMLR = address | context number */
|
||
|
|
movgs gr30,iampr1
|
||
|
|
movgs gr29,damlr1
|
||
|
|
movgs gr30,dampr1
|
||
|
|
|
||
|
|
# return, restoring registers
|
||
|
|
movsg scr2,gr30
|
||
|
|
movgs gr30,ccr
|
||
|
|
sethi.p %hi(__kernel_current_task),gr29
|
||
|
|
setlo %lo(__kernel_current_task),gr29
|
||
|
|
ldi @(gr29,#0),gr29
|
||
|
|
rett #0
|
||
|
|
beq icc0,#3,0 /* prevent icache prefetch */
|
||
|
|
|
||
|
|
# the PTE we want wasn't in the PTD we have mapped, so we need to go looking for a more
|
||
|
|
# appropriate page table and map that instead
|
||
|
|
# - access the PGD with EAR0[31:26]
|
||
|
|
# - DAMLR3 points to the virtual address of the page directory
|
||
|
|
# - the PGD holds 64 PGEs and each PGE/PME points to a set of page tables
|
||
|
|
__dtlb_k_PTD_miss:
|
||
|
|
srli gr29,#26,gr31 /* calculate PGE offset */
|
||
|
|
slli gr31,#8,gr31 /* and clear bottom bits */
|
||
|
|
|
||
|
|
movsg damlr3,gr30
|
||
|
|
ld @(gr31,gr30),gr30 /* access the PGE */
|
||
|
|
|
||
|
|
andicc.p gr30,#_PAGE_PRESENT,gr0,icc0
|
||
|
|
andicc gr30,#xAMPRx_SS,gr0,icc1
|
||
|
|
|
||
|
|
# map this PTD instead and record coverage address
|
||
|
|
ori.p gr30,#xAMPRx_L|xAMPRx_SS_16Kb|xAMPRx_S|xAMPRx_C|xAMPRx_V,gr30
|
||
|
|
beq icc0,#0,__tlb_kernel_fault /* jump if PGE not present */
|
||
|
|
slli.p gr31,#18,gr31
|
||
|
|
bne icc1,#0,__dtlb_k_bigpage
|
||
|
|
movgs gr30,dampr5
|
||
|
|
movgs gr31,scr1
|
||
|
|
|
||
|
|
# we can now resume normal service
|
||
|
|
setlos 0x3ffc,gr30
|
||
|
|
srli.p gr29,#12,gr31 /* use EAR0[25:14] as PTE index */
|
||
|
|
bra __dtlb_k_PTD_mapped
|
||
|
|
|
||
|
|
__dtlb_k_bigpage:
|
||
|
|
break
|
||
|
|
nop
|
||
|
|
|
||
|
|
.size __entry_kernel_data_tlb_miss, .-__entry_kernel_data_tlb_miss
|
||
|
|
|
||
|
|
###############################################################################
|
||
|
|
#
|
||
|
|
# Userspace instruction TLB miss handler (with PGE prediction)
|
||
|
|
# On entry:
|
||
|
|
# GR28 - faulting address
|
||
|
|
# GR31 - EAR0 ^ SCR0
|
||
|
|
# SCR0 - base of virtual range covered by cached PGE from last ITLB miss (or 0xffffffff)
|
||
|
|
# DAMR3 - mapped page directory
|
||
|
|
# DAMR4 - mapped page table as matched by SCR0
|
||
|
|
#
|
||
|
|
###############################################################################
|
||
|
|
.globl __entry_user_insn_tlb_miss
|
||
|
|
.type __entry_user_insn_tlb_miss,@function
|
||
|
|
__entry_user_insn_tlb_miss:
|
||
|
|
#if 0
|
||
|
|
sethi.p %hi(0xe1200004),gr30
|
||
|
|
setlo %lo(0xe1200004),gr30
|
||
|
|
st gr0,@(gr30,gr0)
|
||
|
|
sethi.p %hi(0xffc00100),gr30
|
||
|
|
setlo %lo(0xffc00100),gr30
|
||
|
|
sth gr30,@(gr30,gr0)
|
||
|
|
membar
|
||
|
|
#endif
|
||
|
|
|
||
|
|
movsg ccr,gr30 /* save CCR */
|
||
|
|
movgs gr30,scr2
|
||
|
|
|
||
|
|
# see if the cached page table mapping is appropriate
|
||
|
|
srlicc.p gr31,#26,gr0,icc0
|
||
|
|
setlos 0x3ffc,gr30
|
||
|
|
srli.p gr28,#12,gr31 /* use EAR0[25:14] as PTE index */
|
||
|
|
bne icc0,#0,__itlb_u_PTD_miss
|
||
|
|
|
||
|
|
__itlb_u_PTD_mapped:
|
||
|
|
# access the PTD with EAR0[25:14]
|
||
|
|
# - DAMLR4 points to the virtual address of the appropriate page table
|
||
|
|
# - the PTD holds 4096 PTEs
|
||
|
|
# - the PTD must be accessed uncached
|
||
|
|
# - the PTE must be marked accessed if it was valid
|
||
|
|
#
|
||
|
|
and gr31,gr30,gr31
|
||
|
|
movsg damlr4,gr30
|
||
|
|
add gr30,gr31,gr31
|
||
|
|
ldi @(gr31,#0),gr30 /* fetch the PTE */
|
||
|
|
andicc gr30,#_PAGE_PRESENT,gr0,icc0
|
||
|
|
ori.p gr30,#_PAGE_ACCESSED,gr30
|
||
|
|
beq icc0,#0,__tlb_user_fault /* jump if PTE invalid */
|
||
|
|
sti.p gr30,@(gr31,#0) /* update the PTE */
|
||
|
|
andi gr30,#~_PAGE_ACCESSED,gr30
|
||
|
|
|
||
|
|
# we're using IAMR1/DAMR1 as an extra TLB entry
|
||
|
|
# - punt the entry here (if valid) to the real TLB and then replace with the new PTE
|
||
|
|
movsg dampr1,gr31
|
||
|
|
andicc gr31,#xAMPRx_V,gr0,icc0
|
||
|
|
setlos.p 0xfffff000,gr31
|
||
|
|
beq icc0,#0,__itlb_u_nopunt /* punt not required */
|
||
|
|
|
||
|
|
movsg dampr1,gr31
|
||
|
|
movgs gr31,tppr
|
||
|
|
movsg damlr1,gr31 /* set TPLR.CXN */
|
||
|
|
movgs gr31,tplr
|
||
|
|
tlbpr gr31,gr0,#2,#0 /* save to the TLB */
|
||
|
|
movsg tpxr,gr31 /* check the TLB write error flag */
|
||
|
|
andicc.p gr31,#TPXR_E,gr0,icc0
|
||
|
|
setlos #0xfffff000,gr31
|
||
|
|
bne icc0,#0,__tlb_user_fault
|
||
|
|
|
||
|
|
__itlb_u_nopunt:
|
||
|
|
|
||
|
|
# assemble the new TLB entry
|
||
|
|
and gr28,gr31,gr28
|
||
|
|
movsg cxnr,gr31
|
||
|
|
or gr28,gr31,gr28
|
||
|
|
movgs gr28,iamlr1 /* xAMLR = address | context number */
|
||
|
|
movgs gr30,iampr1
|
||
|
|
movgs gr28,damlr1
|
||
|
|
movgs gr30,dampr1
|
||
|
|
|
||
|
|
# return, restoring registers
|
||
|
|
movsg scr2,gr30
|
||
|
|
movgs gr30,ccr
|
||
|
|
rett #0
|
||
|
|
beq icc0,#3,0 /* prevent icache prefetch */
|
||
|
|
|
||
|
|
# the PTE we want wasn't in the PTD we have mapped, so we need to go looking for a more
|
||
|
|
# appropriate page table and map that instead
|
||
|
|
# - access the PGD with EAR0[31:26]
|
||
|
|
# - DAMLR3 points to the virtual address of the page directory
|
||
|
|
# - the PGD holds 64 PGEs and each PGE/PME points to a set of page tables
|
||
|
|
__itlb_u_PTD_miss:
|
||
|
|
srli gr28,#26,gr31 /* calculate PGE offset */
|
||
|
|
slli gr31,#8,gr31 /* and clear bottom bits */
|
||
|
|
|
||
|
|
movsg damlr3,gr30
|
||
|
|
ld @(gr31,gr30),gr30 /* access the PGE */
|
||
|
|
|
||
|
|
andicc.p gr30,#_PAGE_PRESENT,gr0,icc0
|
||
|
|
andicc gr30,#xAMPRx_SS,gr0,icc1
|
||
|
|
|
||
|
|
# map this PTD instead and record coverage address
|
||
|
|
ori.p gr30,#xAMPRx_L|xAMPRx_SS_16Kb|xAMPRx_S|xAMPRx_C|xAMPRx_V,gr30
|
||
|
|
beq icc0,#0,__tlb_user_fault /* jump if PGE not present */
|
||
|
|
slli.p gr31,#18,gr31
|
||
|
|
bne icc1,#0,__itlb_u_bigpage
|
||
|
|
movgs gr30,dampr4
|
||
|
|
movgs gr31,scr0
|
||
|
|
|
||
|
|
# we can now resume normal service
|
||
|
|
setlos 0x3ffc,gr30
|
||
|
|
srli.p gr28,#12,gr31 /* use EAR0[25:14] as PTE index */
|
||
|
|
bra __itlb_u_PTD_mapped
|
||
|
|
|
||
|
|
__itlb_u_bigpage:
|
||
|
|
break
|
||
|
|
nop
|
||
|
|
|
||
|
|
.size __entry_user_insn_tlb_miss, .-__entry_user_insn_tlb_miss
|
||
|
|
|
||
|
|
###############################################################################
|
||
|
|
#
|
||
|
|
# Userspace data TLB miss handler
|
||
|
|
# On entry:
|
||
|
|
# GR28 - faulting address
|
||
|
|
# GR31 - EAR0 ^ SCR1
|
||
|
|
# SCR1 - base of virtual range covered by cached PGE from last DTLB miss (or 0xffffffff)
|
||
|
|
# DAMR3 - mapped page directory
|
||
|
|
# DAMR5 - mapped page table as matched by SCR1
|
||
|
|
#
|
||
|
|
###############################################################################
|
||
|
|
.globl __entry_user_data_tlb_miss
|
||
|
|
.type __entry_user_data_tlb_miss,@function
|
||
|
|
__entry_user_data_tlb_miss:
|
||
|
|
#if 0
|
||
|
|
sethi.p %hi(0xe1200004),gr30
|
||
|
|
setlo %lo(0xe1200004),gr30
|
||
|
|
st gr0,@(gr30,gr0)
|
||
|
|
sethi.p %hi(0xffc00100),gr30
|
||
|
|
setlo %lo(0xffc00100),gr30
|
||
|
|
sth gr30,@(gr30,gr0)
|
||
|
|
membar
|
||
|
|
#endif
|
||
|
|
|
||
|
|
movsg ccr,gr30 /* save CCR */
|
||
|
|
movgs gr30,scr2
|
||
|
|
|
||
|
|
# see if the cached page table mapping is appropriate
|
||
|
|
srlicc.p gr31,#26,gr0,icc0
|
||
|
|
setlos 0x3ffc,gr30
|
||
|
|
srli.p gr28,#12,gr31 /* use EAR0[25:14] as PTE index */
|
||
|
|
bne icc0,#0,__dtlb_u_PTD_miss
|
||
|
|
|
||
|
|
__dtlb_u_PTD_mapped:
|
||
|
|
# access the PTD with EAR0[25:14]
|
||
|
|
# - DAMLR5 points to the virtual address of the appropriate page table
|
||
|
|
# - the PTD holds 4096 PTEs
|
||
|
|
# - the PTD must be accessed uncached
|
||
|
|
# - the PTE must be marked accessed if it was valid
|
||
|
|
#
|
||
|
|
and gr31,gr30,gr31
|
||
|
|
movsg damlr5,gr30
|
||
|
|
|
||
|
|
__dtlb_u_using_iPTD:
|
||
|
|
add gr30,gr31,gr31
|
||
|
|
ldi @(gr31,#0),gr30 /* fetch the PTE */
|
||
|
|
andicc gr30,#_PAGE_PRESENT,gr0,icc0
|
||
|
|
ori.p gr30,#_PAGE_ACCESSED,gr30
|
||
|
|
beq icc0,#0,__tlb_user_fault /* jump if PTE invalid */
|
||
|
|
sti.p gr30,@(gr31,#0) /* update the PTE */
|
||
|
|
andi gr30,#~_PAGE_ACCESSED,gr30
|
||
|
|
|
||
|
|
# we're using DAMR1 as an extra TLB entry
|
||
|
|
# - punt the entry here (if valid) to the real TLB and then replace with the new PTE
|
||
|
|
movsg dampr1,gr31
|
||
|
|
andicc gr31,#xAMPRx_V,gr0,icc0
|
||
|
|
setlos.p 0xfffff000,gr31
|
||
|
|
beq icc0,#0,__dtlb_u_nopunt /* punt not required */
|
||
|
|
|
||
|
|
movsg dampr1,gr31
|
||
|
|
movgs gr31,tppr
|
||
|
|
movsg damlr1,gr31 /* set TPLR.CXN */
|
||
|
|
movgs gr31,tplr
|
||
|
|
tlbpr gr31,gr0,#2,#0 /* save to the TLB */
|
||
|
|
movsg tpxr,gr31 /* check the TLB write error flag */
|
||
|
|
andicc.p gr31,#TPXR_E,gr0,icc0
|
||
|
|
setlos #0xfffff000,gr31
|
||
|
|
bne icc0,#0,__tlb_user_fault
|
||
|
|
|
||
|
|
__dtlb_u_nopunt:
|
||
|
|
|
||
|
|
# assemble the new TLB entry
|
||
|
|
and gr28,gr31,gr28
|
||
|
|
movsg cxnr,gr31
|
||
|
|
or gr28,gr31,gr28
|
||
|
|
movgs gr28,iamlr1 /* xAMLR = address | context number */
|
||
|
|
movgs gr30,iampr1
|
||
|
|
movgs gr28,damlr1
|
||
|
|
movgs gr30,dampr1
|
||
|
|
|
||
|
|
# return, restoring registers
|
||
|
|
movsg scr2,gr30
|
||
|
|
movgs gr30,ccr
|
||
|
|
rett #0
|
||
|
|
beq icc0,#3,0 /* prevent icache prefetch */
|
||
|
|
|
||
|
|
# the PTE we want wasn't in the PTD we have mapped, so we need to go looking for a more
|
||
|
|
# appropriate page table and map that instead
|
||
|
|
# - first of all, check the insn PGE cache - we may well get a hit there
|
||
|
|
# - access the PGD with EAR0[31:26]
|
||
|
|
# - DAMLR3 points to the virtual address of the page directory
|
||
|
|
# - the PGD holds 64 PGEs and each PGE/PME points to a set of page tables
|
||
|
|
__dtlb_u_PTD_miss:
|
||
|
|
movsg scr0,gr31 /* consult the insn-PGE-cache key */
|
||
|
|
xor gr28,gr31,gr31
|
||
|
|
srlicc gr31,#26,gr0,icc0
|
||
|
|
srli gr28,#12,gr31 /* use EAR0[25:14] as PTE index */
|
||
|
|
bne icc0,#0,__dtlb_u_iPGE_miss
|
||
|
|
|
||
|
|
# what we're looking for is covered by the insn-PGE-cache
|
||
|
|
setlos 0x3ffc,gr30
|
||
|
|
and gr31,gr30,gr31
|
||
|
|
movsg damlr4,gr30
|
||
|
|
bra __dtlb_u_using_iPTD
|
||
|
|
|
||
|
|
__dtlb_u_iPGE_miss:
|
||
|
|
srli gr28,#26,gr31 /* calculate PGE offset */
|
||
|
|
slli gr31,#8,gr31 /* and clear bottom bits */
|
||
|
|
|
||
|
|
movsg damlr3,gr30
|
||
|
|
ld @(gr31,gr30),gr30 /* access the PGE */
|
||
|
|
|
||
|
|
andicc.p gr30,#_PAGE_PRESENT,gr0,icc0
|
||
|
|
andicc gr30,#xAMPRx_SS,gr0,icc1
|
||
|
|
|
||
|
|
# map this PTD instead and record coverage address
|
||
|
|
ori.p gr30,#xAMPRx_L|xAMPRx_SS_16Kb|xAMPRx_S|xAMPRx_C|xAMPRx_V,gr30
|
||
|
|
beq icc0,#0,__tlb_user_fault /* jump if PGE not present */
|
||
|
|
slli.p gr31,#18,gr31
|
||
|
|
bne icc1,#0,__dtlb_u_bigpage
|
||
|
|
movgs gr30,dampr5
|
||
|
|
movgs gr31,scr1
|
||
|
|
|
||
|
|
# we can now resume normal service
|
||
|
|
setlos 0x3ffc,gr30
|
||
|
|
srli.p gr28,#12,gr31 /* use EAR0[25:14] as PTE index */
|
||
|
|
bra __dtlb_u_PTD_mapped
|
||
|
|
|
||
|
|
__dtlb_u_bigpage:
|
||
|
|
break
|
||
|
|
nop
|
||
|
|
|
||
|
|
.size __entry_user_data_tlb_miss, .-__entry_user_data_tlb_miss
|