[MIPS] Fix aliasing bug in copy_to_user_page / copy_from_user_page
The current implementation uses a sequence of a cacheflush and a copy. This is racy in case of a multithreaded debuggee and renders GDB virtually unusable. Aside this fixes a performance hog rendering access to /proc/cmdline very slow and resulting in a enough cache stalls for the 34K AP/SP programming model to make the bare metal code on the non-Linux VPE miss RT deadlines. The main part of this patch was originally written by Ralf Baechle; Atushi Nemoto did the the debugging. Signed-off-by: Atsushi Nemoto <anemo@mba.ocn.ne.jp> Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
This commit is contained in:
parent
224dc50ece
commit
f8829caee3
5 changed files with 190 additions and 28 deletions
|
@ -30,11 +30,34 @@
|
|||
#include <asm/cachectl.h>
|
||||
#include <asm/cpu.h>
|
||||
#include <asm/dma.h>
|
||||
#include <asm/kmap_types.h>
|
||||
#include <asm/mmu_context.h>
|
||||
#include <asm/sections.h>
|
||||
#include <asm/pgtable.h>
|
||||
#include <asm/pgalloc.h>
|
||||
#include <asm/tlb.h>
|
||||
#include <asm/fixmap.h>
|
||||
|
||||
/* Atomicity and interruptability */
|
||||
#ifdef CONFIG_MIPS_MT_SMTC
|
||||
|
||||
#include <asm/mipsmtregs.h>
|
||||
|
||||
#define ENTER_CRITICAL(flags) \
|
||||
{ \
|
||||
unsigned int mvpflags; \
|
||||
local_irq_save(flags);\
|
||||
mvpflags = dvpe()
|
||||
#define EXIT_CRITICAL(flags) \
|
||||
evpe(mvpflags); \
|
||||
local_irq_restore(flags); \
|
||||
}
|
||||
#else
|
||||
|
||||
#define ENTER_CRITICAL(flags) local_irq_save(flags)
|
||||
#define EXIT_CRITICAL(flags) local_irq_restore(flags)
|
||||
|
||||
#endif /* CONFIG_MIPS_MT_SMTC */
|
||||
|
||||
DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
|
||||
|
||||
|
@ -80,13 +103,142 @@ unsigned long setup_zero_pages(void)
|
|||
return 1UL << order;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_HIGHMEM
|
||||
pte_t *kmap_pte;
|
||||
pgprot_t kmap_prot;
|
||||
/*
|
||||
* These are almost like kmap_atomic / kunmap_atmic except they take an
|
||||
* additional address argument as the hint.
|
||||
*/
|
||||
|
||||
#define kmap_get_fixmap_pte(vaddr) \
|
||||
pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr)), (vaddr))
|
||||
|
||||
#ifdef CONFIG_MIPS_MT_SMTC
|
||||
static pte_t *kmap_coherent_pte;
|
||||
static void __init kmap_coherent_init(void)
|
||||
{
|
||||
unsigned long vaddr;
|
||||
|
||||
/* cache the first coherent kmap pte */
|
||||
vaddr = __fix_to_virt(FIX_CMAP_BEGIN);
|
||||
kmap_coherent_pte = kmap_get_fixmap_pte(vaddr);
|
||||
}
|
||||
#else
|
||||
static inline void kmap_coherent_init(void) {}
|
||||
#endif
|
||||
|
||||
static inline void *kmap_coherent(struct page *page, unsigned long addr)
|
||||
{
|
||||
enum fixed_addresses idx;
|
||||
unsigned long vaddr, flags, entrylo;
|
||||
unsigned long old_ctx;
|
||||
pte_t pte;
|
||||
int tlbidx;
|
||||
|
||||
inc_preempt_count();
|
||||
idx = (addr >> PAGE_SHIFT) & (FIX_N_COLOURS - 1);
|
||||
#ifdef CONFIG_MIPS_MT_SMTC
|
||||
idx += FIX_N_COLOURS * smp_processor_id();
|
||||
#endif
|
||||
vaddr = __fix_to_virt(FIX_CMAP_END - idx);
|
||||
pte = mk_pte(page, PAGE_KERNEL);
|
||||
#if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32_R1)
|
||||
entrylo = pte.pte_high;
|
||||
#else
|
||||
entrylo = pte_val(pte) >> 6;
|
||||
#endif
|
||||
|
||||
ENTER_CRITICAL(flags);
|
||||
old_ctx = read_c0_entryhi();
|
||||
write_c0_entryhi(vaddr & (PAGE_MASK << 1));
|
||||
write_c0_entrylo0(entrylo);
|
||||
write_c0_entrylo1(entrylo);
|
||||
#ifdef CONFIG_MIPS_MT_SMTC
|
||||
set_pte(kmap_coherent_pte - (FIX_CMAP_END - idx), pte);
|
||||
/* preload TLB instead of local_flush_tlb_one() */
|
||||
mtc0_tlbw_hazard();
|
||||
tlb_probe();
|
||||
tlb_probe_hazard();
|
||||
tlbidx = read_c0_index();
|
||||
mtc0_tlbw_hazard();
|
||||
if (tlbidx < 0)
|
||||
tlb_write_random();
|
||||
else
|
||||
tlb_write_indexed();
|
||||
#else
|
||||
tlbidx = read_c0_wired();
|
||||
write_c0_wired(tlbidx + 1);
|
||||
write_c0_index(tlbidx);
|
||||
mtc0_tlbw_hazard();
|
||||
tlb_write_indexed();
|
||||
#endif
|
||||
tlbw_use_hazard();
|
||||
write_c0_entryhi(old_ctx);
|
||||
EXIT_CRITICAL(flags);
|
||||
|
||||
return (void*) vaddr;
|
||||
}
|
||||
|
||||
#define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1)))
|
||||
|
||||
static inline void kunmap_coherent(struct page *page)
|
||||
{
|
||||
#ifndef CONFIG_MIPS_MT_SMTC
|
||||
unsigned int wired;
|
||||
unsigned long flags, old_ctx;
|
||||
|
||||
ENTER_CRITICAL(flags);
|
||||
old_ctx = read_c0_entryhi();
|
||||
wired = read_c0_wired() - 1;
|
||||
write_c0_wired(wired);
|
||||
write_c0_index(wired);
|
||||
write_c0_entryhi(UNIQUE_ENTRYHI(wired));
|
||||
write_c0_entrylo0(0);
|
||||
write_c0_entrylo1(0);
|
||||
mtc0_tlbw_hazard();
|
||||
tlb_write_indexed();
|
||||
tlbw_use_hazard();
|
||||
write_c0_entryhi(old_ctx);
|
||||
EXIT_CRITICAL(flags);
|
||||
#endif
|
||||
dec_preempt_count();
|
||||
preempt_check_resched();
|
||||
}
|
||||
|
||||
void copy_to_user_page(struct vm_area_struct *vma,
|
||||
struct page *page, unsigned long vaddr, void *dst, const void *src,
|
||||
unsigned long len)
|
||||
{
|
||||
if (cpu_has_dc_aliases) {
|
||||
void *vto = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
|
||||
memcpy(vto, src, len);
|
||||
kunmap_coherent(page);
|
||||
} else
|
||||
memcpy(dst, src, len);
|
||||
if ((vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc)
|
||||
flush_cache_page(vma, vaddr, page_to_pfn(page));
|
||||
}
|
||||
|
||||
EXPORT_SYMBOL(copy_to_user_page);
|
||||
|
||||
void copy_from_user_page(struct vm_area_struct *vma,
|
||||
struct page *page, unsigned long vaddr, void *dst, const void *src,
|
||||
unsigned long len)
|
||||
{
|
||||
if (cpu_has_dc_aliases) {
|
||||
void *vfrom =
|
||||
kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
|
||||
memcpy(dst, vfrom, len);
|
||||
kunmap_coherent(page);
|
||||
} else
|
||||
memcpy(dst, src, len);
|
||||
}
|
||||
|
||||
EXPORT_SYMBOL(copy_from_user_page);
|
||||
|
||||
|
||||
#ifdef CONFIG_HIGHMEM
|
||||
pte_t *kmap_pte;
|
||||
pgprot_t kmap_prot;
|
||||
|
||||
static void __init kmap_init(void)
|
||||
{
|
||||
unsigned long kmap_vstart;
|
||||
|
@ -97,11 +249,12 @@ static void __init kmap_init(void)
|
|||
|
||||
kmap_prot = PAGE_KERNEL;
|
||||
}
|
||||
#endif /* CONFIG_HIGHMEM */
|
||||
|
||||
#ifdef CONFIG_32BIT
|
||||
void __init fixrange_init(unsigned long start, unsigned long end,
|
||||
pgd_t *pgd_base)
|
||||
{
|
||||
#if defined(CONFIG_HIGHMEM) || defined(CONFIG_MIPS_MT_SMTC)
|
||||
pgd_t *pgd;
|
||||
pud_t *pud;
|
||||
pmd_t *pmd;
|
||||
|
@ -122,7 +275,7 @@ void __init fixrange_init(unsigned long start, unsigned long end,
|
|||
for (; (k < PTRS_PER_PMD) && (vaddr != end); pmd++, k++) {
|
||||
if (pmd_none(*pmd)) {
|
||||
pte = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE);
|
||||
set_pmd(pmd, __pmd(pte));
|
||||
set_pmd(pmd, __pmd((unsigned long)pte));
|
||||
if (pte != pte_offset_kernel(pmd, 0))
|
||||
BUG();
|
||||
}
|
||||
|
@ -132,9 +285,8 @@ void __init fixrange_init(unsigned long start, unsigned long end,
|
|||
}
|
||||
j = 0;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
#endif /* CONFIG_32BIT */
|
||||
#endif /* CONFIG_HIGHMEM */
|
||||
|
||||
#ifndef CONFIG_NEED_MULTIPLE_NODES
|
||||
extern void pagetable_init(void);
|
||||
|
@ -175,6 +327,7 @@ void __init paging_init(void)
|
|||
#ifdef CONFIG_HIGHMEM
|
||||
kmap_init();
|
||||
#endif
|
||||
kmap_coherent_init();
|
||||
|
||||
max_dma = virt_to_phys((char *)MAX_DMA_ADDRESS) >> PAGE_SHIFT;
|
||||
low = max_low_pfn;
|
||||
|
|
|
@ -31,9 +31,10 @@ void pgd_init(unsigned long page)
|
|||
|
||||
void __init pagetable_init(void)
|
||||
{
|
||||
#ifdef CONFIG_HIGHMEM
|
||||
unsigned long vaddr;
|
||||
pgd_t *pgd, *pgd_base;
|
||||
pgd_t *pgd_base;
|
||||
#ifdef CONFIG_HIGHMEM
|
||||
pgd_t *pgd;
|
||||
pud_t *pud;
|
||||
pmd_t *pmd;
|
||||
pte_t *pte;
|
||||
|
@ -44,7 +45,6 @@ void __init pagetable_init(void)
|
|||
pgd_init((unsigned long)swapper_pg_dir
|
||||
+ sizeof(pgd_t) * USER_PTRS_PER_PGD);
|
||||
|
||||
#ifdef CONFIG_HIGHMEM
|
||||
pgd_base = swapper_pg_dir;
|
||||
|
||||
/*
|
||||
|
@ -53,6 +53,7 @@ void __init pagetable_init(void)
|
|||
vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
|
||||
fixrange_init(vaddr, 0, pgd_base);
|
||||
|
||||
#ifdef CONFIG_HIGHMEM
|
||||
/*
|
||||
* Permanent kmaps:
|
||||
*/
|
||||
|
|
|
@ -8,6 +8,7 @@
|
|||
*/
|
||||
#include <linux/init.h>
|
||||
#include <linux/mm.h>
|
||||
#include <asm/fixmap.h>
|
||||
#include <asm/pgtable.h>
|
||||
|
||||
void pgd_init(unsigned long page)
|
||||
|
@ -52,7 +53,17 @@ void pmd_init(unsigned long addr, unsigned long pagetable)
|
|||
|
||||
void __init pagetable_init(void)
|
||||
{
|
||||
unsigned long vaddr;
|
||||
pgd_t *pgd_base;
|
||||
|
||||
/* Initialize the entire pgd. */
|
||||
pgd_init((unsigned long)swapper_pg_dir);
|
||||
pmd_init((unsigned long)invalid_pmd_table, (unsigned long)invalid_pte_table);
|
||||
|
||||
pgd_base = swapper_pg_dir;
|
||||
/*
|
||||
* Fixed mappings:
|
||||
*/
|
||||
vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
|
||||
fixrange_init(vaddr, 0, pgd_base);
|
||||
}
|
||||
|
|
|
@ -55,24 +55,13 @@ extern void (*flush_icache_range)(unsigned long start, unsigned long end);
|
|||
#define flush_cache_vmap(start, end) flush_cache_all()
|
||||
#define flush_cache_vunmap(start, end) flush_cache_all()
|
||||
|
||||
static inline void copy_to_user_page(struct vm_area_struct *vma,
|
||||
extern void copy_to_user_page(struct vm_area_struct *vma,
|
||||
struct page *page, unsigned long vaddr, void *dst, const void *src,
|
||||
unsigned long len)
|
||||
{
|
||||
if (cpu_has_dc_aliases)
|
||||
flush_cache_page(vma, vaddr, page_to_pfn(page));
|
||||
memcpy(dst, src, len);
|
||||
__flush_icache_page(vma, page);
|
||||
}
|
||||
unsigned long len);
|
||||
|
||||
static inline void copy_from_user_page(struct vm_area_struct *vma,
|
||||
extern void copy_from_user_page(struct vm_area_struct *vma,
|
||||
struct page *page, unsigned long vaddr, void *dst, const void *src,
|
||||
unsigned long len)
|
||||
{
|
||||
if (cpu_has_dc_aliases)
|
||||
flush_cache_page(vma, vaddr, page_to_pfn(page));
|
||||
memcpy(dst, src, len);
|
||||
}
|
||||
unsigned long len);
|
||||
|
||||
extern void (*flush_cache_sigtramp)(unsigned long addr);
|
||||
extern void (*flush_icache_all)(void);
|
||||
|
|
|
@ -45,8 +45,16 @@
|
|||
* fix-mapped?
|
||||
*/
|
||||
enum fixed_addresses {
|
||||
#define FIX_N_COLOURS 8
|
||||
FIX_CMAP_BEGIN,
|
||||
#ifdef CONFIG_MIPS_MT_SMTC
|
||||
FIX_CMAP_END = FIX_CMAP_BEGIN + (FIX_N_COLOURS * NR_CPUS),
|
||||
#else
|
||||
FIX_CMAP_END = FIX_CMAP_BEGIN + FIX_N_COLOURS,
|
||||
#endif
|
||||
#ifdef CONFIG_HIGHMEM
|
||||
FIX_KMAP_BEGIN, /* reserved pte's for temporary kernel mappings */
|
||||
/* reserved pte's for temporary kernel mappings */
|
||||
FIX_KMAP_BEGIN = FIX_CMAP_END + 1,
|
||||
FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*NR_CPUS)-1,
|
||||
#endif
|
||||
__end_of_fixed_addresses
|
||||
|
@ -70,9 +78,9 @@ extern void __set_fixmap (enum fixed_addresses idx,
|
|||
* at the top of mem..
|
||||
*/
|
||||
#if defined(CONFIG_CPU_TX39XX) || defined(CONFIG_CPU_TX49XX)
|
||||
#define FIXADDR_TOP (0xff000000UL - 0x2000)
|
||||
#define FIXADDR_TOP ((unsigned long)(long)(int)(0xff000000 - 0x20000))
|
||||
#else
|
||||
#define FIXADDR_TOP (0xffffe000UL)
|
||||
#define FIXADDR_TOP ((unsigned long)(long)(int)0xfffe0000)
|
||||
#endif
|
||||
#define FIXADDR_SIZE (__end_of_fixed_addresses << PAGE_SHIFT)
|
||||
#define FIXADDR_START (FIXADDR_TOP - FIXADDR_SIZE)
|
||||
|
|
Loading…
Reference in a new issue