[XTENSA] Add support for cache-aliasing
Add support for processors that have cache-aliasing issues, such as the Stretch S5000 processor. Cache-aliasing means that the size of the cache (for one way) is larger than the page size, thus, a page can end up in several places in cache depending on the virtual to physical translation. The method used here is to map a user page temporarily through the auto-refill way 0 and of of the DTLB. We probably will want to revisit this issue and use a better approach with kmap/kunmap. Signed-off-by: Chris Zankel <chris@zankel.net>
This commit is contained in:
Chris Zankel
parent
ff6fd46988
commit
6656920b0b
14 changed files with 822 additions and 420 deletions
|
|
@ -18,12 +18,13 @@
|
|||
#include <linux/stddef.h>
|
||||
#include <linux/thread_info.h>
|
||||
#include <linux/ptrace.h>
|
||||
#include <linux/mm.h>
|
||||
|
||||
#include <asm/ptrace.h>
|
||||
#include <asm/processor.h>
|
||||
#include <asm/uaccess.h>
|
||||
|
||||
#define DEFINE(sym, val) asm volatile("\n->" #sym " %0 " #val : : "i" (val))
|
||||
#define BLANK() asm volatile("\n->" : : )
|
||||
|
||||
int main(void)
|
||||
{
|
||||
|
|
@ -63,7 +64,6 @@ int main(void)
|
|||
DEFINE(PT_SIZE, sizeof(struct pt_regs));
|
||||
DEFINE(PT_AREG_END, offsetof (struct pt_regs, areg[XCHAL_NUM_AREGS]));
|
||||
DEFINE(PT_USER_SIZE, offsetof(struct pt_regs, areg[XCHAL_NUM_AREGS]));
|
||||
BLANK();
|
||||
|
||||
/* struct task_struct */
|
||||
DEFINE(TASK_PTRACE, offsetof (struct task_struct, ptrace));
|
||||
|
|
@ -73,27 +73,26 @@ int main(void)
|
|||
DEFINE(TASK_THREAD, offsetof (struct task_struct, thread));
|
||||
DEFINE(TASK_THREAD_INFO, offsetof (struct task_struct, stack));
|
||||
DEFINE(TASK_STRUCT_SIZE, sizeof (struct task_struct));
|
||||
BLANK();
|
||||
|
||||
/* struct thread_info (offset from start_struct) */
|
||||
DEFINE(THREAD_RA, offsetof (struct task_struct, thread.ra));
|
||||
DEFINE(THREAD_SP, offsetof (struct task_struct, thread.sp));
|
||||
DEFINE(THREAD_CP_SAVE, offsetof (struct task_struct, thread.cp_save));
|
||||
DEFINE(THREAD_CURRENT_DS, offsetof (struct task_struct, thread.current_ds));
|
||||
BLANK();
|
||||
|
||||
/* struct mm_struct */
|
||||
DEFINE(MM_USERS, offsetof(struct mm_struct, mm_users));
|
||||
DEFINE(MM_PGD, offsetof (struct mm_struct, pgd));
|
||||
DEFINE(MM_CONTEXT, offsetof (struct mm_struct, context));
|
||||
BLANK();
|
||||
DEFINE(PT_SINGLESTEP_BIT, PT_SINGLESTEP_BIT);
|
||||
|
||||
/* struct page */
|
||||
DEFINE(PAGE_FLAGS, offsetof(struct page, flags));
|
||||
|
||||
/* constants */
|
||||
DEFINE(_CLONE_VM, CLONE_VM);
|
||||
DEFINE(_CLONE_UNTRACED, CLONE_UNTRACED);
|
||||
DEFINE(PG_ARCH_1, PG_arch_1);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
* License. See the file "COPYING" in the main directory of this archive
|
||||
* for more details.
|
||||
*
|
||||
* Copyright (C) 2004-2005 by Tensilica Inc.
|
||||
* Copyright (C) 2004-2007 by Tensilica Inc.
|
||||
*
|
||||
* Chris Zankel <chris@zankel.net>
|
||||
*
|
||||
|
|
@ -169,7 +169,7 @@ _user_exception:
|
|||
* We have to save all registers up to the first '1' from
|
||||
* the right, except the current frame (bit 0).
|
||||
* Assume a2 is: 001001000110001
|
||||
* All regiser frames starting from the top fiel to the marked '1'
|
||||
* All register frames starting from the top field to the marked '1'
|
||||
* must be saved.
|
||||
*/
|
||||
|
||||
|
|
@ -1590,7 +1590,7 @@ ENTRY(fast_second_level_miss)
|
|||
* The messy computation for 'pteval' above really simplifies
|
||||
* into the following:
|
||||
*
|
||||
* pteval = ((pmdval - PAGE_OFFSET) & PAGE_MASK) | PAGE_KERNEL
|
||||
* pteval = ((pmdval - PAGE_OFFSET) & PAGE_MASK) | PAGE_DIRECTORY
|
||||
*/
|
||||
|
||||
movi a1, -PAGE_OFFSET
|
||||
|
|
@ -1602,7 +1602,7 @@ ENTRY(fast_second_level_miss)
|
|||
or a0, a0, a1 # ... | PAGE_DIRECTORY
|
||||
|
||||
/*
|
||||
* We utilize all three wired-ways (7-9( to hold pmd translations.
|
||||
* We utilize all three wired-ways (7-9) to hold pmd translations.
|
||||
* Memory regions are mapped to the DTLBs according to bits 28 and 29.
|
||||
* This allows to map the three most common regions to three different
|
||||
* DTLBs:
|
||||
|
|
@ -1652,6 +1652,73 @@ ENTRY(fast_second_level_miss)
|
|||
9: l32i a0, a1, TASK_ACTIVE_MM # unlikely case mm == 0
|
||||
j 8b
|
||||
|
||||
#if (DCACHE_WAY_SIZE > PAGE_SIZE)
|
||||
|
||||
2: /* Special case for cache aliasing.
|
||||
* We (should) only get here if a clear_user_page, copy_user_page
|
||||
* or the aliased cache flush functions got preemptively interrupted
|
||||
* by another task. Re-establish temporary mapping to the
|
||||
* TLBTEMP_BASE areas.
|
||||
*/
|
||||
|
||||
/* We shouldn't be in a double exception */
|
||||
|
||||
l32i a0, a2, PT_DEPC
|
||||
bgeui a0, VALID_DOUBLE_EXCEPTION_ADDRESS, 2f
|
||||
|
||||
/* Make sure the exception originated in the special functions */
|
||||
|
||||
movi a0, __tlbtemp_mapping_start
|
||||
rsr a3, EPC_1
|
||||
bltu a3, a0, 2f
|
||||
movi a0, __tlbtemp_mapping_end
|
||||
bgeu a3, a0, 2f
|
||||
|
||||
/* Check if excvaddr was in one of the TLBTEMP_BASE areas. */
|
||||
|
||||
movi a3, TLBTEMP_BASE_1
|
||||
rsr a0, EXCVADDR
|
||||
bltu a0, a3, 2f
|
||||
|
||||
addi a1, a0, -(2 << (DCACHE_ALIAS_ORDER + PAGE_SHIFT))
|
||||
bgeu a1, a3, 2f
|
||||
|
||||
/* Check if we have to restore an ITLB mapping. */
|
||||
|
||||
movi a1, __tlbtemp_mapping_itlb
|
||||
rsr a3, EPC_1
|
||||
sub a3, a3, a1
|
||||
|
||||
/* Calculate VPN */
|
||||
|
||||
movi a1, PAGE_MASK
|
||||
and a1, a1, a0
|
||||
|
||||
/* Jump for ITLB entry */
|
||||
|
||||
bgez a3, 1f
|
||||
|
||||
/* We can use up to two TLBTEMP areas, one for src and one for dst. */
|
||||
|
||||
extui a3, a0, PAGE_SHIFT + DCACHE_ALIAS_ORDER, 1
|
||||
add a1, a3, a1
|
||||
|
||||
/* PPN is in a6 for the first TLBTEMP area and in a7 for the second. */
|
||||
|
||||
mov a0, a6
|
||||
movnez a0, a7, a3
|
||||
j 3b
|
||||
|
||||
/* ITLB entry. We only use dst in a6. */
|
||||
|
||||
1: witlb a6, a1
|
||||
isync
|
||||
j 4b
|
||||
|
||||
|
||||
#endif // DCACHE_WAY_SIZE > PAGE_SIZE
|
||||
|
||||
|
||||
2: /* Invalid PGD, default exception handling */
|
||||
|
||||
movi a3, exc_table
|
||||
|
|
|
|||
|
|
@ -5,9 +5,5 @@
|
|||
# removes any old dependencies. DON'T put your own dependencies here
|
||||
# unless it's something special (ie not a .c file).
|
||||
#
|
||||
# Note 2! The CFLAGS definition is now in the main makefile...
|
||||
|
||||
obj-y := init.o fault.o tlb.o misc.o
|
||||
obj-m :=
|
||||
obj-n :=
|
||||
obj- :=
|
||||
obj-y := init.o fault.o tlb.o misc.o cache.o
|
||||
|
|
|
|||
256
arch/xtensa/mm/cache.c
Normal file
256
arch/xtensa/mm/cache.c
Normal file
|
|
@ -0,0 +1,256 @@
|
|||
/*
|
||||
* arch/xtensa/mm/cache.c
|
||||
*
|
||||
* This file is subject to the terms and conditions of the GNU General Public
|
||||
* License. See the file "COPYING" in the main directory of this archive
|
||||
* for more details.
|
||||
*
|
||||
* Copyright (C) 2001-2006 Tensilica Inc.
|
||||
*
|
||||
* Chris Zankel <chris@zankel.net>
|
||||
* Joe Taylor
|
||||
* Marc Gauthier
|
||||
*
|
||||
*/
|
||||
|
||||
#include <linux/init.h>
|
||||
#include <linux/signal.h>
|
||||
#include <linux/sched.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/errno.h>
|
||||
#include <linux/string.h>
|
||||
#include <linux/types.h>
|
||||
#include <linux/ptrace.h>
|
||||
#include <linux/bootmem.h>
|
||||
#include <linux/swap.h>
|
||||
#include <linux/pagemap.h>
|
||||
|
||||
#include <asm/pgtable.h>
|
||||
#include <asm/bootparam.h>
|
||||
#include <asm/mmu_context.h>
|
||||
#include <asm/tlb.h>
|
||||
#include <asm/tlbflush.h>
|
||||
#include <asm/page.h>
|
||||
#include <asm/pgalloc.h>
|
||||
#include <asm/pgtable.h>
|
||||
|
||||
//#define printd(x...) printk(x)
|
||||
#define printd(x...) do { } while(0)
|
||||
|
||||
/*
|
||||
* Note:
|
||||
* The kernel provides one architecture bit PG_arch_1 in the page flags that
|
||||
* can be used for cache coherency.
|
||||
*
|
||||
* I$-D$ coherency.
|
||||
*
|
||||
* The Xtensa architecture doesn't keep the instruction cache coherent with
|
||||
* the data cache. We use the architecture bit to indicate if the caches
|
||||
* are coherent. The kernel clears this bit whenever a page is added to the
|
||||
* page cache. At that time, the caches might not be in sync. We, therefore,
|
||||
* define this flag as 'clean' if set.
|
||||
*
|
||||
* D-cache aliasing.
|
||||
*
|
||||
* With cache aliasing, we have to always flush the cache when pages are
|
||||
* unmapped (see tlb_start_vma(). So, we use this flag to indicate a dirty
|
||||
* page.
|
||||
*
|
||||
*
|
||||
*
|
||||
*/
|
||||
|
||||
#if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK
|
||||
|
||||
/*
|
||||
* Any time the kernel writes to a user page cache page, or it is about to
|
||||
* read from a page cache page this routine is called.
|
||||
*
|
||||
*/
|
||||
|
||||
void flush_dcache_page(struct page *page)
|
||||
{
|
||||
struct address_space *mapping = page_mapping(page);
|
||||
|
||||
/*
|
||||
* If we have a mapping but the page is not mapped to user-space
|
||||
* yet, we simply mark this page dirty and defer flushing the
|
||||
* caches until update_mmu().
|
||||
*/
|
||||
|
||||
if (mapping && !mapping_mapped(mapping)) {
|
||||
if (!test_bit(PG_arch_1, &page->flags))
|
||||
set_bit(PG_arch_1, &page->flags);
|
||||
return;
|
||||
|
||||
} else {
|
||||
|
||||
unsigned long phys = page_to_phys(page);
|
||||
unsigned long temp = page->index << PAGE_SHIFT;
|
||||
unsigned long alias = !(DCACHE_ALIAS_EQ(temp, phys));
|
||||
unsigned long virt;
|
||||
|
||||
/*
|
||||
* Flush the page in kernel space and user space.
|
||||
* Note that we can omit that step if aliasing is not
|
||||
* an issue, but we do have to synchronize I$ and D$
|
||||
* if we have a mapping.
|
||||
*/
|
||||
|
||||
if (!alias && !mapping)
|
||||
return;
|
||||
|
||||
__flush_invalidate_dcache_page((long)page_address(page));
|
||||
|
||||
virt = TLBTEMP_BASE_1 + (temp & DCACHE_ALIAS_MASK);
|
||||
|
||||
if (alias)
|
||||
__flush_invalidate_dcache_page_alias(virt, phys);
|
||||
|
||||
if (mapping)
|
||||
__invalidate_icache_page_alias(virt, phys);
|
||||
}
|
||||
|
||||
/* There shouldn't be an entry in the cache for this page anymore. */
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* For now, flush the whole cache. FIXME??
|
||||
*/
|
||||
|
||||
void flush_cache_range(struct vm_area_struct* vma,
|
||||
unsigned long start, unsigned long end)
|
||||
{
|
||||
__flush_invalidate_dcache_all();
|
||||
__invalidate_icache_all();
|
||||
}
|
||||
|
||||
/*
|
||||
* Remove any entry in the cache for this page.
|
||||
*
|
||||
* Note that this function is only called for user pages, so use the
|
||||
* alias versions of the cache flush functions.
|
||||
*/
|
||||
|
||||
void flush_cache_page(struct vm_area_struct* vma, unsigned long address,
|
||||
unsigned long pfn)
|
||||
{
|
||||
/* Note that we have to use the 'alias' address to avoid multi-hit */
|
||||
|
||||
unsigned long phys = page_to_phys(pfn_to_page(pfn));
|
||||
unsigned long virt = TLBTEMP_BASE_1 + (address & DCACHE_ALIAS_MASK);
|
||||
|
||||
__flush_invalidate_dcache_page_alias(virt, phys);
|
||||
__invalidate_icache_page_alias(virt, phys);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
void
|
||||
update_mmu_cache(struct vm_area_struct * vma, unsigned long addr, pte_t pte)
|
||||
{
|
||||
unsigned long pfn = pte_pfn(pte);
|
||||
struct page *page;
|
||||
|
||||
if (!pfn_valid(pfn))
|
||||
return;
|
||||
|
||||
page = pfn_to_page(pfn);
|
||||
|
||||
/* Invalidate old entry in TLBs */
|
||||
|
||||
invalidate_itlb_mapping(addr);
|
||||
invalidate_dtlb_mapping(addr);
|
||||
|
||||
#if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK
|
||||
|
||||
if (!PageReserved(page) && test_bit(PG_arch_1, &page->flags)) {
|
||||
|
||||
unsigned long vaddr = TLBTEMP_BASE_1 + (addr & DCACHE_ALIAS_MASK);
|
||||
unsigned long paddr = (unsigned long) page_address(page);
|
||||
unsigned long phys = page_to_phys(page);
|
||||
|
||||
__flush_invalidate_dcache_page(paddr);
|
||||
|
||||
__flush_invalidate_dcache_page_alias(vaddr, phys);
|
||||
__invalidate_icache_page_alias(vaddr, phys);
|
||||
|
||||
clear_bit(PG_arch_1, &page->flags);
|
||||
}
|
||||
#else
|
||||
if (!PageReserved(page) && !test_bit(PG_arch_1, &page->flags)
|
||||
&& (vma->vm_flags & VM_EXEC) != 0) {
|
||||
unsigned long vaddr = addr & PAGE_MASK;
|
||||
__flush_dcache_page(vaddr);
|
||||
__invalidate_icache_page(vaddr);
|
||||
set_bit(PG_arch_1, &page->flags);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
/*
|
||||
* access_process_vm() has called get_user_pages(), which has done a
|
||||
* flush_dcache_page() on the page.
|
||||
*/
|
||||
|
||||
#if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK
|
||||
|
||||
void copy_to_user_page(struct vm_area_struct *vma, struct page *page,
|
||||
unsigned long vaddr, void *dst, const void *src,
|
||||
unsigned long len)
|
||||
{
|
||||
unsigned long phys = page_to_phys(page);
|
||||
unsigned long alias = !(DCACHE_ALIAS_EQ(vaddr, phys));
|
||||
|
||||
/* Flush and invalidate user page if aliased. */
|
||||
|
||||
if (alias) {
|
||||
unsigned long temp = TLBTEMP_BASE_1 + (vaddr & DCACHE_ALIAS_MASK);
|
||||
__flush_invalidate_dcache_page_alias(temp, phys);
|
||||
}
|
||||
|
||||
/* Copy data */
|
||||
|
||||
memcpy(dst, src, len);
|
||||
|
||||
/*
|
||||
* Flush and invalidate kernel page if aliased and synchronize
|
||||
* data and instruction caches for executable pages.
|
||||
*/
|
||||
|
||||
if (alias) {
|
||||
unsigned long temp = TLBTEMP_BASE_1 + (vaddr & DCACHE_ALIAS_MASK);
|
||||
|
||||
__flush_invalidate_dcache_range((unsigned long) dst, len);
|
||||
if ((vma->vm_flags & VM_EXEC) != 0) {
|
||||
__invalidate_icache_page_alias(temp, phys);
|
||||
}
|
||||
|
||||
} else if ((vma->vm_flags & VM_EXEC) != 0) {
|
||||
__flush_dcache_range((unsigned long)dst,len);
|
||||
__invalidate_icache_range((unsigned long) dst, len);
|
||||
}
|
||||
}
|
||||
|
||||
extern void copy_from_user_page(struct vm_area_struct *vma, struct page *page,
|
||||
unsigned long vaddr, void *dst, const void *src,
|
||||
unsigned long len)
|
||||
{
|
||||
unsigned long phys = page_to_phys(page);
|
||||
unsigned long alias = !(DCACHE_ALIAS_EQ(vaddr, phys));
|
||||
|
||||
/*
|
||||
* Flush user page if aliased.
|
||||
* (Note: a simply flush would be sufficient)
|
||||
*/
|
||||
|
||||
if (alias) {
|
||||
unsigned long temp = TLBTEMP_BASE_1 + (vaddr & DCACHE_ALIAS_MASK);
|
||||
__flush_invalidate_dcache_page_alias(temp, phys);
|
||||
}
|
||||
|
||||
memcpy(dst, src, len);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
@ -24,6 +24,8 @@
|
|||
unsigned long asid_cache = ASID_USER_FIRST;
|
||||
void bad_page_fault(struct pt_regs*, unsigned long, int);
|
||||
|
||||
#undef DEBUG_PAGE_FAULT
|
||||
|
||||
/*
|
||||
* This routine handles page faults. It determines the address,
|
||||
* and the problem, and then passes it off to one of the appropriate
|
||||
|
|
@ -64,7 +66,7 @@ void do_page_fault(struct pt_regs *regs)
|
|||
exccause == EXCCAUSE_ITLB_MISS ||
|
||||
exccause == EXCCAUSE_FETCH_CACHE_ATTRIBUTE) ? 1 : 0;
|
||||
|
||||
#if 0
|
||||
#ifdef DEBUG_PAGE_FAULT
|
||||
printk("[%s:%d:%08x:%d:%08x:%s%s]\n", current->comm, current->pid,
|
||||
address, exccause, regs->pc, is_write? "w":"", is_exec? "x":"");
|
||||
#endif
|
||||
|
|
@ -219,7 +221,7 @@ bad_page_fault(struct pt_regs *regs, unsigned long address, int sig)
|
|||
|
||||
/* Are we prepared to handle this kernel fault? */
|
||||
if ((entry = search_exception_tables(regs->pc)) != NULL) {
|
||||
#if 1
|
||||
#ifdef DEBUG_PAGE_FAULT
|
||||
printk(KERN_DEBUG "%s: Exception at pc=%#010lx (%lx)\n",
|
||||
current->comm, regs->pc, entry->fixup);
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -15,40 +15,24 @@
|
|||
* Kevin Chea
|
||||
*/
|
||||
|
||||
#include <linux/init.h>
|
||||
#include <linux/signal.h>
|
||||
#include <linux/sched.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/errno.h>
|
||||
#include <linux/string.h>
|
||||
#include <linux/types.h>
|
||||
#include <linux/ptrace.h>
|
||||
#include <linux/bootmem.h>
|
||||
#include <linux/swap.h>
|
||||
#include <linux/mman.h>
|
||||
#include <linux/nodemask.h>
|
||||
#include <linux/mm.h>
|
||||
#include <linux/slab.h>
|
||||
|
||||
#include <asm/pgtable.h>
|
||||
#include <asm/bootparam.h>
|
||||
#include <asm/mmu_context.h>
|
||||
#include <asm/tlb.h>
|
||||
#include <asm/tlbflush.h>
|
||||
#include <asm/page.h>
|
||||
#include <asm/pgalloc.h>
|
||||
#include <asm/pgtable.h>
|
||||
|
||||
|
||||
#define DEBUG 0
|
||||
|
||||
DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
|
||||
//static DEFINE_SPINLOCK(tlb_lock);
|
||||
|
||||
/*
|
||||
* This flag is used to indicate that the page was mapped and modified in
|
||||
* kernel space, so the cache is probably dirty at that address.
|
||||
* If cache aliasing is enabled and the page color mismatches, update_mmu_cache
|
||||
* synchronizes the caches if this bit is set.
|
||||
*/
|
||||
|
||||
#define PG_cache_clean PG_arch_1
|
||||
|
||||
/* References to section boundaries */
|
||||
|
||||
|
|
@ -323,228 +307,22 @@ void show_mem(void)
|
|||
printk("%d free pages\n", free);
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------------------- */
|
||||
struct kmem_cache *pgtable_cache __read_mostly;
|
||||
|
||||
#if (DCACHE_WAY_SIZE > PAGE_SIZE)
|
||||
|
||||
/*
|
||||
* With cache aliasing, the page color of the page in kernel space and user
|
||||
* space might mismatch. We temporarily map the page to a different virtual
|
||||
* address with the same color and clear the page there.
|
||||
*/
|
||||
|
||||
void clear_user_page(void *kaddr, unsigned long vaddr, struct page* page)
|
||||
static void pgd_ctor(void *addr, struct kmem_cache *cache, unsigned long flags)
|
||||
{
|
||||
pte_t* ptep = (pte_t*)addr;
|
||||
int i;
|
||||
|
||||
/* There shouldn't be any entries for this page. */
|
||||
for (i = 0; i < 1024; i++, ptep++)
|
||||
pte_clear(NULL, 0, ptep);
|
||||
|
||||
__flush_invalidate_dcache_page_phys(__pa(page_address(page)));
|
||||
|
||||
if (!PAGE_COLOR_EQ(vaddr, kaddr)) {
|
||||
unsigned long v, p;
|
||||
|
||||
/* Temporarily map page to DTLB_WAY_DCACHE_ALIAS0. */
|
||||
|
||||
spin_lock(&tlb_lock);
|
||||
|
||||
p = (unsigned long)pte_val((mk_pte(page,PAGE_KERNEL)));
|
||||
kaddr = (void*)PAGE_COLOR_MAP0(vaddr);
|
||||
v = (unsigned long)kaddr | DTLB_WAY_DCACHE_ALIAS0;
|
||||
__asm__ __volatile__("wdtlb %0,%1; dsync" : :"a" (p), "a" (v));
|
||||
|
||||
clear_page(kaddr);
|
||||
|
||||
spin_unlock(&tlb_lock);
|
||||
} else {
|
||||
clear_page(kaddr);
|
||||
}
|
||||
|
||||
/* We need to make sure that i$ and d$ are coherent. */
|
||||
|
||||
clear_bit(PG_cache_clean, &page->flags);
|
||||
}
|
||||
|
||||
/*
|
||||
* With cache aliasing, we have to make sure that the page color of the page
|
||||
* in kernel space matches that of the virtual user address before we read
|
||||
* the page. If the page color differ, we create a temporary DTLB entry with
|
||||
* the corrent page color and use this 'temporary' address as the source.
|
||||
* We then use the same approach as in clear_user_page and copy the data
|
||||
* to the kernel space and clear the PG_cache_clean bit to synchronize caches
|
||||
* later.
|
||||
*
|
||||
* Note:
|
||||
* Instead of using another 'way' for the temporary DTLB entry, we could
|
||||
* probably use the same entry that points to the kernel address (after
|
||||
* saving the original value and restoring it when we are done).
|
||||
*/
|
||||
|
||||
void copy_user_page(void* to, void* from, unsigned long vaddr,
|
||||
struct page* to_page)
|
||||
void __init pgtable_cache_init(void)
|
||||
{
|
||||
/* There shouldn't be any entries for the new page. */
|
||||
|
||||
__flush_invalidate_dcache_page_phys(__pa(page_address(to_page)));
|
||||
|
||||
spin_lock(&tlb_lock);
|
||||
|
||||
if (!PAGE_COLOR_EQ(vaddr, from)) {
|
||||
unsigned long v, p, t;
|
||||
|
||||
__asm__ __volatile__ ("pdtlb %1,%2; rdtlb1 %0,%1"
|
||||
: "=a"(p), "=a"(t) : "a"(from));
|
||||
from = (void*)PAGE_COLOR_MAP0(vaddr);
|
||||
v = (unsigned long)from | DTLB_WAY_DCACHE_ALIAS0;
|
||||
__asm__ __volatile__ ("wdtlb %0,%1; dsync" ::"a" (p), "a" (v));
|
||||
}
|
||||
|
||||
if (!PAGE_COLOR_EQ(vaddr, to)) {
|
||||
unsigned long v, p;
|
||||
|
||||
p = (unsigned long)pte_val((mk_pte(to_page,PAGE_KERNEL)));
|
||||
to = (void*)PAGE_COLOR_MAP1(vaddr);
|
||||
v = (unsigned long)to | DTLB_WAY_DCACHE_ALIAS1;
|
||||
__asm__ __volatile__ ("wdtlb %0,%1; dsync" ::"a" (p), "a" (v));
|
||||
}
|
||||
copy_page(to, from);
|
||||
|
||||
spin_unlock(&tlb_lock);
|
||||
|
||||
/* We need to make sure that i$ and d$ are coherent. */
|
||||
|
||||
clear_bit(PG_cache_clean, &to_page->flags);
|
||||
pgtable_cache = kmem_cache_create("pgd",
|
||||
PAGE_SIZE, PAGE_SIZE,
|
||||
SLAB_HWCACHE_ALIGN,
|
||||
pgd_ctor);
|
||||
}
|
||||
|
||||
|
||||
|
||||
/*
|
||||
* Any time the kernel writes to a user page cache page, or it is about to
|
||||
* read from a page cache page this routine is called.
|
||||
*
|
||||
* Note:
|
||||
* The kernel currently only provides one architecture bit in the page
|
||||
* flags that we use for I$/D$ coherency. Maybe, in future, we can
|
||||
* use a sepearte bit for deferred dcache aliasing:
|
||||
* If the page is not mapped yet, we only need to set a flag,
|
||||
* if mapped, we need to invalidate the page.
|
||||
*/
|
||||
// FIXME: we probably need this for WB caches not only for Page Coloring..
|
||||
|
||||
void flush_dcache_page(struct page *page)
|
||||
{
|
||||
unsigned long addr = __pa(page_address(page));
|
||||
struct address_space *mapping = page_mapping(page);
|
||||
|
||||
__flush_invalidate_dcache_page_phys(addr);
|
||||
|
||||
if (!test_bit(PG_cache_clean, &page->flags))
|
||||
return;
|
||||
|
||||
/* If this page hasn't been mapped, yet, handle I$/D$ coherency later.*/
|
||||
#if 0
|
||||
if (mapping && !mapping_mapped(mapping))
|
||||
clear_bit(PG_cache_clean, &page->flags);
|
||||
else
|
||||
#endif
|
||||
__invalidate_icache_page_phys(addr);
|
||||
}
|
||||
|
||||
void flush_cache_range(struct vm_area_struct* vma, unsigned long s,
|
||||
unsigned long e)
|
||||
{
|
||||
__flush_invalidate_cache_all();
|
||||
}
|
||||
|
||||
void flush_cache_page(struct vm_area_struct* vma, unsigned long address,
|
||||
unsigned long pfn)
|
||||
{
|
||||
struct page *page = pfn_to_page(pfn);
|
||||
|
||||
/* Remove any entry for the old mapping. */
|
||||
|
||||
if (current->active_mm == vma->vm_mm) {
|
||||
unsigned long addr = __pa(page_address(page));
|
||||
__flush_invalidate_dcache_page_phys(addr);
|
||||
if ((vma->vm_flags & VM_EXEC) != 0)
|
||||
__invalidate_icache_page_phys(addr);
|
||||
} else {
|
||||
BUG();
|
||||
}
|
||||
}
|
||||
|
||||
#endif /* (DCACHE_WAY_SIZE > PAGE_SIZE) */
|
||||
|
||||
|
||||
pte_t* pte_alloc_one_kernel (struct mm_struct* mm, unsigned long addr)
|
||||
{
|
||||
pte_t* pte = (pte_t*)__get_free_pages(GFP_KERNEL|__GFP_REPEAT, 0);
|
||||
if (likely(pte)) {
|
||||
pte_t* ptep = (pte_t*)(pte_val(*pte) + PAGE_OFFSET);
|
||||
int i;
|
||||
for (i = 0; i < 1024; i++, ptep++)
|
||||
pte_clear(mm, addr, ptep);
|
||||
}
|
||||
return pte;
|
||||
}
|
||||
|
||||
struct page* pte_alloc_one(struct mm_struct *mm, unsigned long addr)
|
||||
{
|
||||
struct page *page;
|
||||
|
||||
page = alloc_pages(GFP_KERNEL | __GFP_REPEAT, 0);
|
||||
|
||||
if (likely(page)) {
|
||||
pte_t* ptep = kmap_atomic(page, KM_USER0);
|
||||
int i;
|
||||
|
||||
for (i = 0; i < 1024; i++, ptep++)
|
||||
pte_clear(mm, addr, ptep);
|
||||
|
||||
kunmap_atomic(ptep, KM_USER0);
|
||||
}
|
||||
return page;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* Handle D$/I$ coherency.
|
||||
*
|
||||
* Note:
|
||||
* We only have one architecture bit for the page flags, so we cannot handle
|
||||
* cache aliasing, yet.
|
||||
*/
|
||||
|
||||
void
|
||||
update_mmu_cache(struct vm_area_struct * vma, unsigned long addr, pte_t pte)
|
||||
{
|
||||
unsigned long pfn = pte_pfn(pte);
|
||||
struct page *page;
|
||||
unsigned long vaddr = addr & PAGE_MASK;
|
||||
|
||||
if (!pfn_valid(pfn))
|
||||
return;
|
||||
|
||||
page = pfn_to_page(pfn);
|
||||
|
||||
invalidate_itlb_mapping(addr);
|
||||
invalidate_dtlb_mapping(addr);
|
||||
|
||||
/* We have a new mapping. Use it. */
|
||||
|
||||
write_dtlb_entry(pte, dtlb_probe(addr));
|
||||
|
||||
/* If the processor can execute from this page, synchronize D$/I$. */
|
||||
|
||||
if ((vma->vm_flags & VM_EXEC) != 0) {
|
||||
|
||||
write_itlb_entry(pte, itlb_probe(addr));
|
||||
|
||||
/* Synchronize caches, if not clean. */
|
||||
|
||||
if (!test_and_set_bit(PG_cache_clean, &page->flags)) {
|
||||
__flush_dcache_page(vaddr);
|
||||
__invalidate_icache_page(vaddr);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,29 +7,33 @@
|
|||
* License. See the file "COPYING" in the main directory of this archive
|
||||
* for more details.
|
||||
*
|
||||
* Copyright (C) 2001 - 2005 Tensilica Inc.
|
||||
* Copyright (C) 2001 - 2007 Tensilica Inc.
|
||||
*
|
||||
* Chris Zankel <chris@zankel.net>
|
||||
*/
|
||||
|
||||
/* Note: we might want to implement some of the loops as zero-overhead-loops,
|
||||
* where applicable and if supported by the processor.
|
||||
*/
|
||||
|
||||
#include <linux/linkage.h>
|
||||
#include <asm/page.h>
|
||||
#include <asm/pgtable.h>
|
||||
#include <asm/asmmacro.h>
|
||||
#include <asm/cacheasm.h>
|
||||
#include <asm/tlbflush.h>
|
||||
|
||||
/* clear_page (page) */
|
||||
|
||||
/*
|
||||
* clear_page and clear_user_page are the same for non-cache-aliased configs.
|
||||
*
|
||||
* clear_page (unsigned long page)
|
||||
* a2
|
||||
*/
|
||||
|
||||
ENTRY(clear_page)
|
||||
entry a1, 16
|
||||
addi a4, a2, PAGE_SIZE
|
||||
movi a3, 0
|
||||
|
||||
1: s32i a3, a2, 0
|
||||
movi a3, 0
|
||||
__loopi a2, a7, PAGE_SIZE, 32
|
||||
s32i a3, a2, 0
|
||||
s32i a3, a2, 4
|
||||
s32i a3, a2, 8
|
||||
s32i a3, a2, 12
|
||||
|
|
@ -37,42 +41,277 @@ ENTRY(clear_page)
|
|||
s32i a3, a2, 20
|
||||
s32i a3, a2, 24
|
||||
s32i a3, a2, 28
|
||||
addi a2, a2, 32
|
||||
blt a2, a4, 1b
|
||||
__endla a2, a7, 32
|
||||
|
||||
retw
|
||||
|
||||
/*
|
||||
* copy_page and copy_user_page are the same for non-cache-aliased configs.
|
||||
*
|
||||
* copy_page (void *to, void *from)
|
||||
* a2 a3
|
||||
* a2 a3
|
||||
*/
|
||||
|
||||
ENTRY(copy_page)
|
||||
entry a1, 16
|
||||
addi a4, a2, PAGE_SIZE
|
||||
|
||||
1: l32i a5, a3, 0
|
||||
l32i a6, a3, 4
|
||||
l32i a7, a3, 8
|
||||
s32i a5, a2, 0
|
||||
s32i a6, a2, 4
|
||||
s32i a7, a2, 8
|
||||
l32i a5, a3, 12
|
||||
l32i a6, a3, 16
|
||||
l32i a7, a3, 20
|
||||
s32i a5, a2, 12
|
||||
s32i a6, a2, 16
|
||||
s32i a7, a2, 20
|
||||
l32i a5, a3, 24
|
||||
l32i a6, a3, 28
|
||||
s32i a5, a2, 24
|
||||
s32i a6, a2, 28
|
||||
addi a2, a2, 32
|
||||
addi a3, a3, 32
|
||||
blt a2, a4, 1b
|
||||
__loopi a2, a4, PAGE_SIZE, 32
|
||||
|
||||
l32i a8, a3, 0
|
||||
l32i a9, a3, 4
|
||||
s32i a8, a2, 0
|
||||
s32i a9, a2, 4
|
||||
|
||||
l32i a8, a3, 8
|
||||
l32i a9, a3, 12
|
||||
s32i a8, a2, 8
|
||||
s32i a9, a2, 12
|
||||
|
||||
l32i a8, a3, 16
|
||||
l32i a9, a3, 20
|
||||
s32i a8, a2, 16
|
||||
s32i a9, a2, 20
|
||||
|
||||
l32i a8, a3, 24
|
||||
l32i a9, a3, 28
|
||||
s32i a8, a2, 24
|
||||
s32i a9, a2, 28
|
||||
|
||||
addi a2, a2, 32
|
||||
addi a3, a3, 32
|
||||
|
||||
__endl a2, a4
|
||||
|
||||
retw
|
||||
|
||||
/*
|
||||
* If we have to deal with cache aliasing, we use temporary memory mappings
|
||||
* to ensure that the source and destination pages have the same color as
|
||||
* the virtual address. We use way 0 and 1 for temporary mappings in such cases.
|
||||
*
|
||||
* The temporary DTLB entries shouldn't be flushed by interrupts, but are
|
||||
* flushed by preemptive task switches. Special code in the
|
||||
* fast_second_level_miss handler re-established the temporary mapping.
|
||||
* It requires that the PPNs for the destination and source addresses are
|
||||
* in a6, and a7, respectively.
|
||||
*/
|
||||
|
||||
/* TLB miss exceptions are treated special in the following region */
|
||||
|
||||
ENTRY(__tlbtemp_mapping_start)
|
||||
|
||||
#if (DCACHE_WAY_SIZE > PAGE_SIZE)
|
||||
|
||||
/*
|
||||
* clear_user_page (void *addr, unsigned long vaddr, struct page *page)
|
||||
* a2 a3 a4
|
||||
*/
|
||||
|
||||
ENTRY(clear_user_page)
|
||||
entry a1, 32
|
||||
|
||||
/* Mark page dirty and determine alias. */
|
||||
|
||||
movi a7, (1 << PG_ARCH_1)
|
||||
l32i a5, a4, PAGE_FLAGS
|
||||
xor a6, a2, a3
|
||||
extui a3, a3, PAGE_SHIFT, DCACHE_ALIAS_ORDER
|
||||
extui a6, a6, PAGE_SHIFT, DCACHE_ALIAS_ORDER
|
||||
or a5, a5, a7
|
||||
slli a3, a3, PAGE_SHIFT
|
||||
s32i a5, a4, PAGE_FLAGS
|
||||
|
||||
/* Skip setting up a temporary DTLB if not aliased. */
|
||||
|
||||
beqz a6, 1f
|
||||
|
||||
/* Invalidate kernel page. */
|
||||
|
||||
mov a10, a2
|
||||
call8 __invalidate_dcache_page
|
||||
|
||||
/* Setup a temporary DTLB with the color of the VPN */
|
||||
|
||||
movi a4, -PAGE_OFFSET + (PAGE_KERNEL | _PAGE_HW_WRITE)
|
||||
movi a5, TLBTEMP_BASE_1 # virt
|
||||
add a6, a2, a4 # ppn
|
||||
add a2, a5, a3 # add 'color'
|
||||
|
||||
wdtlb a6, a2
|
||||
dsync
|
||||
|
||||
1: movi a3, 0
|
||||
__loopi a2, a7, PAGE_SIZE, 32
|
||||
s32i a3, a2, 0
|
||||
s32i a3, a2, 4
|
||||
s32i a3, a2, 8
|
||||
s32i a3, a2, 12
|
||||
s32i a3, a2, 16
|
||||
s32i a3, a2, 20
|
||||
s32i a3, a2, 24
|
||||
s32i a3, a2, 28
|
||||
__endla a2, a7, 32
|
||||
|
||||
bnez a6, 1f
|
||||
retw
|
||||
|
||||
/* We need to invalidate the temporary idtlb entry, if any. */
|
||||
|
||||
1: addi a2, a2, -PAGE_SIZE
|
||||
idtlb a2
|
||||
dsync
|
||||
|
||||
retw
|
||||
|
||||
/*
|
||||
* copy_page_user (void *to, void *from, unsigned long vaddr, struct page *page)
|
||||
* a2 a3 a4 a5
|
||||
*/
|
||||
|
||||
ENTRY(copy_user_page)
|
||||
|
||||
entry a1, 32
|
||||
|
||||
/* Mark page dirty and determine alias for destination. */
|
||||
|
||||
movi a8, (1 << PG_ARCH_1)
|
||||
l32i a9, a5, PAGE_FLAGS
|
||||
xor a6, a2, a4
|
||||
xor a7, a3, a4
|
||||
extui a4, a4, PAGE_SHIFT, DCACHE_ALIAS_ORDER
|
||||
extui a6, a6, PAGE_SHIFT, DCACHE_ALIAS_ORDER
|
||||
extui a7, a7, PAGE_SHIFT, DCACHE_ALIAS_ORDER
|
||||
or a9, a9, a8
|
||||
slli a4, a4, PAGE_SHIFT
|
||||
s32i a9, a5, PAGE_FLAGS
|
||||
movi a5, -PAGE_OFFSET + (PAGE_KERNEL | _PAGE_HW_WRITE)
|
||||
|
||||
beqz a6, 1f
|
||||
|
||||
/* Invalidate dcache */
|
||||
|
||||
mov a10, a2
|
||||
call8 __invalidate_dcache_page
|
||||
|
||||
/* Setup a temporary DTLB with a matching color. */
|
||||
|
||||
movi a8, TLBTEMP_BASE_1 # base
|
||||
add a6, a2, a5 # ppn
|
||||
add a2, a8, a4 # add 'color'
|
||||
|
||||
wdtlb a6, a2
|
||||
dsync
|
||||
|
||||
/* Skip setting up a temporary DTLB for destination if not aliased. */
|
||||
|
||||
1: beqz a7, 1f
|
||||
|
||||
/* Setup a temporary DTLB with a matching color. */
|
||||
|
||||
movi a8, TLBTEMP_BASE_2 # base
|
||||
add a7, a3, a5 # ppn
|
||||
add a3, a8, a4
|
||||
addi a8, a3, 1 # way1
|
||||
|
||||
wdtlb a7, a8
|
||||
dsync
|
||||
|
||||
1: __loopi a2, a4, PAGE_SIZE, 32
|
||||
|
||||
l32i a8, a3, 0
|
||||
l32i a9, a3, 4
|
||||
s32i a8, a2, 0
|
||||
s32i a9, a2, 4
|
||||
|
||||
l32i a8, a3, 8
|
||||
l32i a9, a3, 12
|
||||
s32i a8, a2, 8
|
||||
s32i a9, a2, 12
|
||||
|
||||
l32i a8, a3, 16
|
||||
l32i a9, a3, 20
|
||||
s32i a8, a2, 16
|
||||
s32i a9, a2, 20
|
||||
|
||||
l32i a8, a3, 24
|
||||
l32i a9, a3, 28
|
||||
s32i a8, a2, 24
|
||||
s32i a9, a2, 28
|
||||
|
||||
addi a2, a2, 32
|
||||
addi a3, a3, 32
|
||||
|
||||
__endl a2, a4
|
||||
|
||||
/* We need to invalidate any temporary mapping! */
|
||||
|
||||
bnez a6, 1f
|
||||
bnez a7, 2f
|
||||
retw
|
||||
|
||||
1: addi a2, a2, -PAGE_SIZE
|
||||
idtlb a2
|
||||
dsync
|
||||
bnez a7, 2f
|
||||
retw
|
||||
|
||||
2: addi a3, a3, -PAGE_SIZE+1
|
||||
idtlb a3
|
||||
dsync
|
||||
|
||||
retw
|
||||
|
||||
#endif
|
||||
|
||||
#if (DCACHE_WAY_SIZE > PAGE_SIZE)
|
||||
|
||||
/*
|
||||
* void __flush_invalidate_dcache_page_alias (addr, phys)
|
||||
* a2 a3
|
||||
*/
|
||||
|
||||
ENTRY(__flush_invalidate_dcache_page_alias)
|
||||
entry sp, 16
|
||||
|
||||
movi a7, 0 # required for exception handler
|
||||
addi a6, a3, (PAGE_KERNEL | _PAGE_HW_WRITE)
|
||||
mov a4, a2
|
||||
wdtlb a6, a2
|
||||
dsync
|
||||
|
||||
___flush_invalidate_dcache_page a2 a3
|
||||
|
||||
idtlb a4
|
||||
dsync
|
||||
|
||||
retw
|
||||
|
||||
#endif
|
||||
|
||||
ENTRY(__tlbtemp_mapping_itlb)
|
||||
|
||||
#if (ICACHE_WAY_SIZE > PAGE_SIZE)
|
||||
|
||||
ENTRY(__invalidate_icache_page_alias)
|
||||
entry sp, 16
|
||||
|
||||
addi a6, a3, (PAGE_KERNEL | _PAGE_HW_WRITE)
|
||||
mov a4, a2
|
||||
witlb a6, a2
|
||||
isync
|
||||
|
||||
___invalidate_icache_page a2 a3
|
||||
|
||||
iitlb a4
|
||||
isync
|
||||
retw
|
||||
|
||||
#endif
|
||||
|
||||
/* End of special treatment in tlb miss exception */
|
||||
|
||||
ENTRY(__tlbtemp_mapping_end)
|
||||
|
||||
/*
|
||||
* void __invalidate_icache_page(ulong start)
|
||||
*/
|
||||
|
|
@ -121,8 +360,6 @@ ENTRY(__flush_dcache_page)
|
|||
dsync
|
||||
retw
|
||||
|
||||
|
||||
|
||||
/*
|
||||
* void __invalidate_icache_range(ulong start, ulong size)
|
||||
*/
|
||||
|
|
@ -168,7 +405,6 @@ ENTRY(__invalidate_dcache_range)
|
|||
|
||||
___invalidate_dcache_range a2 a3 a4
|
||||
|
||||
|
||||
retw
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -19,6 +19,15 @@
|
|||
|
||||
#define DCACHE_WAY_SIZE (XCHAL_DCACHE_SIZE/XCHAL_DCACHE_WAYS)
|
||||
#define ICACHE_WAY_SIZE (XCHAL_ICACHE_SIZE/XCHAL_ICACHE_WAYS)
|
||||
#define DCACHE_WAY_SHIFT (XCHAL_DCACHE_SETWIDTH + XCHAL_DCACHE_LINEWIDTH)
|
||||
#define ICACHE_WAY_SHIFT (XCHAL_ICACHE_SETWIDTH + XCHAL_ICACHE_LINEWIDTH)
|
||||
|
||||
/* Maximum cache size per way. */
|
||||
#if DCACHE_WAY_SIZE >= ICACHE_WAY_SIZE
|
||||
# define CACHE_WAY_SIZE DCACHE_WAY_SIZE
|
||||
#else
|
||||
# define CACHE_WAY_SIZE ICACHE_WAY_SIZE
|
||||
#endif
|
||||
|
||||
|
||||
#endif /* _XTENSA_CACHE_H */
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@
|
|||
* License. See the file "COPYING" in the main directory of this archive
|
||||
* for more details.
|
||||
*
|
||||
* (C) 2001 - 2006 Tensilica Inc.
|
||||
* (C) 2001 - 2007 Tensilica Inc.
|
||||
*/
|
||||
|
||||
#ifndef _XTENSA_CACHEFLUSH_H
|
||||
|
|
@ -18,10 +18,7 @@
|
|||
#include <asm/page.h>
|
||||
|
||||
/*
|
||||
* flush and invalidate data cache, invalidate instruction cache:
|
||||
*
|
||||
* __flush_invalidate_cache_all()
|
||||
* __flush_invalidate_cache_range(from,sze)
|
||||
* Lo-level routines for cache flushing.
|
||||
*
|
||||
* invalidate data or instruction cache:
|
||||
*
|
||||
|
|
@ -40,26 +37,39 @@
|
|||
* __flush_invalidate_dcache_all()
|
||||
* __flush_invalidate_dcache_page(adr)
|
||||
* __flush_invalidate_dcache_range(from,size)
|
||||
*
|
||||
* specials for cache aliasing:
|
||||
*
|
||||
* __flush_invalidate_dcache_page_alias(vaddr,paddr)
|
||||
* __invalidate_icache_page_alias(vaddr,paddr)
|
||||
*/
|
||||
|
||||
extern void __flush_invalidate_cache_all(void);
|
||||
extern void __flush_invalidate_cache_range(unsigned long, unsigned long);
|
||||
extern void __flush_invalidate_dcache_all(void);
|
||||
extern void __invalidate_dcache_all(void);
|
||||
extern void __invalidate_icache_all(void);
|
||||
|
||||
extern void __invalidate_dcache_page(unsigned long);
|
||||
extern void __invalidate_icache_page(unsigned long);
|
||||
extern void __invalidate_icache_range(unsigned long, unsigned long);
|
||||
extern void __invalidate_dcache_range(unsigned long, unsigned long);
|
||||
|
||||
|
||||
#if XCHAL_DCACHE_IS_WRITEBACK
|
||||
extern void __flush_invalidate_dcache_all(void);
|
||||
extern void __flush_dcache_page(unsigned long);
|
||||
extern void __flush_dcache_range(unsigned long, unsigned long);
|
||||
extern void __flush_invalidate_dcache_page(unsigned long);
|
||||
extern void __flush_invalidate_dcache_range(unsigned long, unsigned long);
|
||||
#else
|
||||
# define __flush_dcache_page(p) do { } while(0)
|
||||
# define __flush_invalidate_dcache_page(p) do { } while(0)
|
||||
# define __flush_invalidate_dcache_range(p,s) do { } while(0)
|
||||
# define __flush_dcache_range(p,s) do { } while(0)
|
||||
# define __flush_dcache_page(p) do { } while(0)
|
||||
# define __flush_invalidate_dcache_page(p) __invalidate_dcache_page(p)
|
||||
# define __flush_invalidate_dcache_range(p,s) __invalidate_dcache_range(p,s)
|
||||
#endif
|
||||
|
||||
#if (DCACHE_WAY_SIZE > PAGE_SIZE)
|
||||
extern void __flush_invalidate_dcache_page_alias(unsigned long, unsigned long);
|
||||
#endif
|
||||
#if (ICACHE_WAY_SIZE > PAGE_SIZE)
|
||||
extern void __invalidate_icache_page_alias(unsigned long, unsigned long);
|
||||
#endif
|
||||
|
||||
/*
|
||||
|
|
@ -71,17 +81,21 @@ extern void __flush_invalidate_dcache_range(unsigned long, unsigned long);
|
|||
* (see also Documentation/cachetlb.txt)
|
||||
*/
|
||||
|
||||
#if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK
|
||||
#if (DCACHE_WAY_SIZE > PAGE_SIZE)
|
||||
|
||||
#define flush_cache_all() __flush_invalidate_cache_all();
|
||||
#define flush_cache_mm(mm) __flush_invalidate_cache_all();
|
||||
#define flush_cache_dup_mm(mm) __flush_invalidate_cache_all();
|
||||
#define flush_cache_all() \
|
||||
do { \
|
||||
__flush_invalidate_dcache_all(); \
|
||||
__invalidate_icache_all(); \
|
||||
} while (0)
|
||||
|
||||
#define flush_cache_vmap(start,end) __flush_invalidate_cache_all();
|
||||
#define flush_cache_vunmap(start,end) __flush_invalidate_cache_all();
|
||||
#define flush_cache_mm(mm) flush_cache_all()
|
||||
#define flush_cache_dup_mm(mm) flush_cache_mm(mm)
|
||||
|
||||
#define flush_cache_vmap(start,end) flush_cache_all()
|
||||
#define flush_cache_vunmap(start,end) flush_cache_all()
|
||||
|
||||
extern void flush_dcache_page(struct page*);
|
||||
|
||||
extern void flush_cache_range(struct vm_area_struct*, ulong, ulong);
|
||||
extern void flush_cache_page(struct vm_area_struct*, unsigned long, unsigned long);
|
||||
|
||||
|
|
@ -101,24 +115,39 @@ extern void flush_cache_page(struct vm_area_struct*, unsigned long, unsigned lon
|
|||
|
||||
#endif
|
||||
|
||||
/* Ensure consistency between data and instruction cache. */
|
||||
#define flush_icache_range(start,end) \
|
||||
__invalidate_icache_range(start,(end)-(start))
|
||||
do { \
|
||||
__flush_dcache_range(start, (end) - (start)); \
|
||||
__invalidate_icache_range(start,(end) - (start)); \
|
||||
} while (0)
|
||||
|
||||
/* This is not required, see Documentation/cachetlb.txt */
|
||||
|
||||
#define flush_icache_page(vma,page) do { } while(0)
|
||||
#define flush_icache_page(vma,page) do { } while (0)
|
||||
|
||||
#define flush_dcache_mmap_lock(mapping) do { } while (0)
|
||||
#define flush_dcache_mmap_unlock(mapping) do { } while (0)
|
||||
|
||||
#if (DCACHE_WAY_SIZE > PAGE_SIZE)
|
||||
|
||||
#define copy_to_user_page(vma, page, vaddr, dst, src, len) \
|
||||
memcpy(dst, src, len)
|
||||
extern void copy_to_user_page(struct vm_area_struct*, struct page*,
|
||||
unsigned long, void*, const void*, unsigned long);
|
||||
extern void copy_from_user_page(struct vm_area_struct*, struct page*,
|
||||
unsigned long, void*, const void*, unsigned long);
|
||||
|
||||
#else
|
||||
|
||||
#define copy_to_user_page(vma, page, vaddr, dst, src, len) \
|
||||
do { \
|
||||
memcpy(dst, src, len); \
|
||||
__flush_dcache_range((unsigned long) dst, len); \
|
||||
__invalidate_icache_range((unsigned long) dst, len); \
|
||||
} while (0)
|
||||
|
||||
#define copy_from_user_page(vma, page, vaddr, dst, src, len) \
|
||||
memcpy(dst, src, len)
|
||||
|
||||
#endif
|
||||
|
||||
#endif /* __KERNEL__ */
|
||||
|
||||
#endif /* _XTENSA_CACHEFLUSH_H */
|
||||
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
#ifdef __KERNEL__
|
||||
#include <asm/byteorder.h>
|
||||
#include <asm/page.h>
|
||||
#include <linux/kernel.h>
|
||||
|
||||
#include <linux/types.h>
|
||||
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
|
||||
#include <asm/processor.h>
|
||||
#include <asm/types.h>
|
||||
#include <asm/cache.h>
|
||||
|
||||
/*
|
||||
* Fixed TLB translations in the processor.
|
||||
|
|
@ -39,6 +40,53 @@
|
|||
#define MAX_MEM_PFN XCHAL_KSEG_SIZE
|
||||
#define PGTABLE_START 0x80000000
|
||||
|
||||
/*
|
||||
* Cache aliasing:
|
||||
*
|
||||
* If the cache size for one way is greater than the page size, we have to
|
||||
* deal with cache aliasing. The cache index is wider than the page size:
|
||||
*
|
||||
* | |cache| cache index
|
||||
* | pfn |off| virtual address
|
||||
* |xxxx:X|zzz|
|
||||
* | : | |
|
||||
* | \ / | |
|
||||
* |trans.| |
|
||||
* | / \ | |
|
||||
* |yyyy:Y|zzz| physical address
|
||||
*
|
||||
* When the page number is translated to the physical page address, the lowest
|
||||
* bit(s) (X) that are part of the cache index are also translated (Y).
|
||||
* If this translation changes bit(s) (X), the cache index is also afected,
|
||||
* thus resulting in a different cache line than before.
|
||||
* The kernel does not provide a mechanism to ensure that the page color
|
||||
* (represented by this bit) remains the same when allocated or when pages
|
||||
* are remapped. When user pages are mapped into kernel space, the color of
|
||||
* the page might also change.
|
||||
*
|
||||
* We use the address space VMALLOC_END ... VMALLOC_END + DCACHE_WAY_SIZE * 2
|
||||
* to temporarily map a patch so we can match the color.
|
||||
*/
|
||||
|
||||
#if DCACHE_WAY_SIZE > PAGE_SIZE
|
||||
# define DCACHE_ALIAS_ORDER (DCACHE_WAY_SHIFT - PAGE_SHIFT)
|
||||
# define DCACHE_ALIAS_MASK (PAGE_MASK & (DCACHE_WAY_SIZE - 1))
|
||||
# define DCACHE_ALIAS(a) (((a) & DCACHE_ALIAS_MASK) >> PAGE_SHIFT)
|
||||
# define DCACHE_ALIAS_EQ(a,b) ((((a) ^ (b)) & DCACHE_ALIAS_MASK) == 0)
|
||||
#else
|
||||
# define DCACHE_ALIAS_ORDER 0
|
||||
#endif
|
||||
|
||||
#if ICACHE_WAY_SIZE > PAGE_SIZE
|
||||
# define ICACHE_ALIAS_ORDER (ICACHE_WAY_SHIFT - PAGE_SHIFT)
|
||||
# define ICACHE_ALIAS_MASK (PAGE_MASK & (ICACHE_WAY_SIZE - 1))
|
||||
# define ICACHE_ALIAS(a) (((a) & ICACHE_ALIAS_MASK) >> PAGE_SHIFT)
|
||||
# define ICACHE_ALIAS_EQ(a,b) ((((a) ^ (b)) & ICACHE_ALIAS_MASK) == 0)
|
||||
#else
|
||||
# define ICACHE_ALIAS_ORDER 0
|
||||
#endif
|
||||
|
||||
|
||||
#ifdef __ASSEMBLY__
|
||||
|
||||
#define __pgprot(x) (x)
|
||||
|
|
@ -90,11 +138,11 @@ extern void copy_page(void *to, void *from);
|
|||
* some extra work
|
||||
*/
|
||||
|
||||
#if (DCACHE_WAY_SIZE > PAGE_SIZE)
|
||||
void clear_user_page(void *addr, unsigned long vaddr, struct page* page);
|
||||
void copy_user_page(void *to,void* from,unsigned long vaddr,struct page* page);
|
||||
#if DCACHE_WAY_SIZE > PAGE_SIZE
|
||||
extern void clear_user_page(void*, unsigned long, struct page*);
|
||||
extern void copy_user_page(void*, void*, unsigned long, struct page*);
|
||||
#else
|
||||
# define clear_user_page(page,vaddr,pg) clear_page(page)
|
||||
# define clear_user_page(page, vaddr, pg) clear_page(page)
|
||||
# define copy_user_page(to, from, vaddr, pg) copy_page(to, from)
|
||||
#endif
|
||||
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
/*
|
||||
* linux/include/asm-xtensa/pgalloc.h
|
||||
* include/asm-xtensa/pgalloc.h
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License version 2 as
|
||||
* published by the Free Software Foundation.
|
||||
*
|
||||
* Copyright (C) 2001-2005 Tensilica Inc.
|
||||
* Copyright (C) 2001-2007 Tensilica Inc.
|
||||
*/
|
||||
|
||||
#ifndef _XTENSA_PGALLOC_H
|
||||
|
|
@ -13,103 +13,54 @@
|
|||
|
||||
#ifdef __KERNEL__
|
||||
|
||||
#include <linux/threads.h>
|
||||
#include <linux/highmem.h>
|
||||
#include <asm/processor.h>
|
||||
#include <asm/cacheflush.h>
|
||||
|
||||
|
||||
/* Cache aliasing:
|
||||
*
|
||||
* If the cache size for one way is greater than the page size, we have to
|
||||
* deal with cache aliasing. The cache index is wider than the page size:
|
||||
*
|
||||
* |cache |
|
||||
* |pgnum |page| virtual address
|
||||
* |xxxxxX|zzzz|
|
||||
* | | |
|
||||
* \ / | |
|
||||
* trans.| |
|
||||
* / \ | |
|
||||
* |yyyyyY|zzzz| physical address
|
||||
*
|
||||
* When the page number is translated to the physical page address, the lowest
|
||||
* bit(s) (X) that are also part of the cache index are also translated (Y).
|
||||
* If this translation changes this bit (X), the cache index is also afected,
|
||||
* thus resulting in a different cache line than before.
|
||||
* The kernel does not provide a mechanism to ensure that the page color
|
||||
* (represented by this bit) remains the same when allocated or when pages
|
||||
* are remapped. When user pages are mapped into kernel space, the color of
|
||||
* the page might also change.
|
||||
*
|
||||
* We use the address space VMALLOC_END ... VMALLOC_END + DCACHE_WAY_SIZE * 2
|
||||
* to temporarily map a patch so we can match the color.
|
||||
*/
|
||||
|
||||
#if (DCACHE_WAY_SIZE > PAGE_SIZE)
|
||||
# define PAGE_COLOR_MASK (PAGE_MASK & (DCACHE_WAY_SIZE-1))
|
||||
# define PAGE_COLOR(a) \
|
||||
(((unsigned long)(a)&PAGE_COLOR_MASK) >> PAGE_SHIFT)
|
||||
# define PAGE_COLOR_EQ(a,b) \
|
||||
((((unsigned long)(a) ^ (unsigned long)(b)) & PAGE_COLOR_MASK) == 0)
|
||||
# define PAGE_COLOR_MAP0(v) \
|
||||
(VMALLOC_END + ((unsigned long)(v) & PAGE_COLOR_MASK))
|
||||
# define PAGE_COLOR_MAP1(v) \
|
||||
(VMALLOC_END + ((unsigned long)(v) & PAGE_COLOR_MASK) + DCACHE_WAY_SIZE)
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Allocating and freeing a pmd is trivial: the 1-entry pmd is
|
||||
* inside the pgd, so has no extra memory associated with it.
|
||||
*/
|
||||
|
||||
#define pgd_free(pgd) free_page((unsigned long)(pgd))
|
||||
|
||||
#if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK
|
||||
|
||||
static inline void
|
||||
pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmdp, pte_t *pte)
|
||||
{
|
||||
pmd_val(*(pmdp)) = (unsigned long)(pte);
|
||||
__asm__ __volatile__ ("memw; dhwb %0, 0; dsync" :: "a" (pmdp));
|
||||
}
|
||||
|
||||
static inline void
|
||||
pmd_populate(struct mm_struct *mm, pmd_t *pmdp, struct page *page)
|
||||
{
|
||||
pmd_val(*(pmdp)) = (unsigned long)page_to_virt(page);
|
||||
__asm__ __volatile__ ("memw; dhwb %0, 0; dsync" :: "a" (pmdp));
|
||||
}
|
||||
|
||||
|
||||
|
||||
#else
|
||||
|
||||
# define pmd_populate_kernel(mm, pmdp, pte) \
|
||||
(pmd_val(*(pmdp)) = (unsigned long)(pte))
|
||||
# define pmd_populate(mm, pmdp, page) \
|
||||
(pmd_val(*(pmdp)) = (unsigned long)page_to_virt(page))
|
||||
|
||||
#endif
|
||||
#define pmd_populate_kernel(mm, pmdp, ptep) \
|
||||
(pmd_val(*(pmdp)) = ((unsigned long)ptep))
|
||||
#define pmd_populate(mm, pmdp, page) \
|
||||
(pmd_val(*(pmdp)) = ((unsigned long)page_to_virt(page)))
|
||||
|
||||
static inline pgd_t*
|
||||
pgd_alloc(struct mm_struct *mm)
|
||||
{
|
||||
pgd_t *pgd;
|
||||
|
||||
pgd = (pgd_t *)__get_free_pages(GFP_KERNEL|__GFP_ZERO, PGD_ORDER);
|
||||
|
||||
if (likely(pgd != NULL))
|
||||
__flush_dcache_page((unsigned long)pgd);
|
||||
|
||||
return pgd;
|
||||
return (pgd_t*) __get_free_pages(GFP_KERNEL | __GFP_ZERO, PGD_ORDER);
|
||||
}
|
||||
|
||||
extern pte_t* pte_alloc_one_kernel(struct mm_struct* mm, unsigned long addr);
|
||||
extern struct page* pte_alloc_one(struct mm_struct* mm, unsigned long addr);
|
||||
static inline void pgd_free(pgd_t *pgd)
|
||||
{
|
||||
free_page((unsigned long)pgd);
|
||||
}
|
||||
|
||||
#define pte_free_kernel(pte) free_page((unsigned long)pte)
|
||||
#define pte_free(pte) __free_page(pte)
|
||||
/* Use a slab cache for the pte pages (see also sparc64 implementation) */
|
||||
|
||||
extern struct kmem_cache *pgtable_cache;
|
||||
|
||||
static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm,
|
||||
unsigned long address)
|
||||
{
|
||||
return kmem_cache_alloc(pgtable_cache, GFP_KERNEL|__GFP_REPEAT);
|
||||
}
|
||||
|
||||
static inline struct page *pte_alloc_one(struct mm_struct *mm,
|
||||
unsigned long addr)
|
||||
{
|
||||
return virt_to_page(pte_alloc_one_kernel(mm, addr));
|
||||
}
|
||||
|
||||
static inline void pte_free_kernel(pte_t *pte)
|
||||
{
|
||||
kmem_cache_free(pgtable_cache, pte);
|
||||
}
|
||||
|
||||
static inline void pte_free(struct page *page)
|
||||
{
|
||||
kmem_cache_free(pgtable_cache, page_address(page));
|
||||
}
|
||||
|
||||
#endif /* __KERNEL__ */
|
||||
#endif /* _XTENSA_PGALLOC_H */
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/*
|
||||
* linux/include/asm-xtensa/pgtable.h
|
||||
* include/asm-xtensa/pgtable.h
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License version 2 as
|
||||
|
|
@ -60,16 +60,20 @@
|
|||
#define FIRST_USER_ADDRESS 0
|
||||
#define FIRST_USER_PGD_NR (FIRST_USER_ADDRESS >> PGDIR_SHIFT)
|
||||
|
||||
/* virtual memory area. We keep a distance to other memory regions to be
|
||||
/*
|
||||
* Virtual memory area. We keep a distance to other memory regions to be
|
||||
* on the safe side. We also use this area for cache aliasing.
|
||||
*/
|
||||
|
||||
// FIXME: virtual memory area must be configuration-dependent
|
||||
|
||||
#define VMALLOC_START 0xC0000000
|
||||
#define VMALLOC_END 0xC7FF0000
|
||||
#define VMALLOC_END 0xC6FEFFFF
|
||||
#define TLBTEMP_BASE_1 0xC6FF0000
|
||||
#define TLBTEMP_BASE_2 0xC6FF8000
|
||||
#define MODULE_START 0xC7000000
|
||||
#define MODULE_END 0xC7FFFFFF
|
||||
|
||||
/* Xtensa Linux config PTE layout (when present):
|
||||
/*
|
||||
* Xtensa Linux config PTE layout (when present):
|
||||
* 31-12: PPN
|
||||
* 11-6: Software
|
||||
* 5-4: RING
|
||||
|
|
@ -126,12 +130,13 @@
|
|||
#define PAGE_SHARED __pgprot(_PAGE_PRESENT | _PAGE_USER | _PAGE_WRITABLE)
|
||||
#define PAGE_SHARED_EXEC \
|
||||
__pgprot(_PAGE_PRESENT | _PAGE_USER | _PAGE_WRITABLE | _PAGE_HW_EXEC)
|
||||
#define PAGE_KERNEL __pgprot(_PAGE_PRESENT)
|
||||
#define PAGE_KERNEL __pgprot(_PAGE_PRESENT | _PAGE_HW_WRITE)
|
||||
#define PAGE_KERNEL_EXEC __pgprot(_PAGE_PRESENT|_PAGE_HW_WRITE|_PAGE_HW_EXEC)
|
||||
|
||||
#if (DCACHE_WAY_SIZE > PAGE_SIZE)
|
||||
# define _PAGE_DIRECTORY (_PAGE_VALID | _PAGE_ACCESSED | _PAGE_HW_WRITE)
|
||||
# define _PAGE_DIRECTORY (_PAGE_VALID | _PAGE_ACCESSED)
|
||||
#else
|
||||
# define _PAGE_DIRECTORY (_PAGE_VALID|_PAGE_ACCESSED|_PAGE_HW_WRITE|_PAGE_CA_WB)
|
||||
# define _PAGE_DIRECTORY (_PAGE_VALID | _PAGE_ACCESSED | _PAGE_CA_WB)
|
||||
#endif
|
||||
|
||||
#else /* no mmu */
|
||||
|
|
@ -244,6 +249,10 @@ static inline pte_t pte_modify(pte_t pte, pgprot_t newprot)
|
|||
static inline void update_pte(pte_t *ptep, pte_t pteval)
|
||||
{
|
||||
*ptep = pteval;
|
||||
#if (DCACHE_WAY_SIZE > PAGE_SIZE) && XCHAL_DCACHE_IS_WRITEBACK
|
||||
__asm__ __volatile__ ("dhwb %0, 0" :: "a" (ptep));
|
||||
#endif
|
||||
|
||||
}
|
||||
|
||||
struct mm_struct;
|
||||
|
|
@ -383,13 +392,12 @@ extern void update_mmu_cache(struct vm_area_struct * vma,
|
|||
* remap a physical page `pfn' of size `size' with page protection `prot'
|
||||
* into virtual address `from'
|
||||
*/
|
||||
|
||||
#define io_remap_pfn_range(vma,from,pfn,size,prot) \
|
||||
remap_pfn_range(vma, from, pfn, size, prot)
|
||||
|
||||
|
||||
/* No page table caches to init */
|
||||
|
||||
#define pgtable_cache_init() do { } while (0)
|
||||
extern void pgtable_cache_init(void);
|
||||
|
||||
typedef pte_t *pte_addr_t;
|
||||
|
||||
|
|
|
|||
|
|
@ -11,14 +11,36 @@
|
|||
#ifndef _XTENSA_TLB_H
|
||||
#define _XTENSA_TLB_H
|
||||
|
||||
#define tlb_start_vma(tlb,vma) do { } while (0)
|
||||
#define tlb_end_vma(tlb,vma) do { } while (0)
|
||||
#define __tlb_remove_tlb_entry(tlb,pte,addr) do { } while (0)
|
||||
#include <asm/cache.h>
|
||||
#include <asm/page.h>
|
||||
|
||||
#if (DCACHE_WAY_SIZE <= PAGE_SIZE)
|
||||
|
||||
/* Note, read http://lkml.org/lkml/2004/1/15/6 */
|
||||
|
||||
# define tlb_start_vma(tlb,vma) do { } while (0)
|
||||
# define tlb_end_vma(tlb,vma) do { } while (0)
|
||||
|
||||
#else
|
||||
|
||||
# define tlb_start_vma(tlb, vma) \
|
||||
do { \
|
||||
if (!tlb->fullmm) \
|
||||
flush_cache_range(vma, vma->vm_start, vma->vm_end); \
|
||||
} while(0)
|
||||
|
||||
# define tlb_end_vma(tlb, vma) \
|
||||
do { \
|
||||
if (!tlb->fullmm) \
|
||||
flush_tlb_range(vma, vma->vm_start, vma->vm_end); \
|
||||
} while(0)
|
||||
|
||||
#endif
|
||||
|
||||
#define __tlb_remove_tlb_entry(tlb,pte,addr) do { } while (0)
|
||||
#define tlb_flush(tlb) flush_tlb_mm((tlb)->mm)
|
||||
|
||||
#include <asm-generic/tlb.h>
|
||||
#include <asm/page.h>
|
||||
|
||||
#define __pte_free_tlb(tlb,pte) pte_free(pte)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue