4150e50bf5
The flush_tlb_page() function is used on user address ranges when PTEs
(or PMDs/PUDs for huge pages) were changed (attributes or clearing). For
such cases, it is more efficient to invalidate only the last level of
the TLB with the "tlbi vale1is" instruction.
In the TLB shoot-down case, the TLB caching of the intermediate page
table levels (pmd, pud, pgd) is handled by __flush_tlb_pgtable() via the
__(pte|pmd|pud)_free_tlb() functions and it is not deferred to
tlb_finish_mmu() (as of commit 285994a62c - "arm64: Invalidate the TLB
corresponding to intermediate page table levels"). The tlb_flush()
function only needs to invalidate the TLB for the last level of page
tables; the __flush_tlb_range() function gains a fourth argument for
last level TLBI.
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
Cc: Will Deacon <will.deacon@arm.com>
Signed-off-by: Will Deacon <will.deacon@arm.com>
79 lines
2.1 KiB
C
79 lines
2.1 KiB
C
/*
|
|
* Based on arch/arm/include/asm/tlb.h
|
|
*
|
|
* Copyright (C) 2002 Russell King
|
|
* Copyright (C) 2012 ARM Ltd.
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
* published by the Free Software Foundation.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
#ifndef __ASM_TLB_H
|
|
#define __ASM_TLB_H
|
|
|
|
#include <linux/pagemap.h>
|
|
#include <linux/swap.h>
|
|
|
|
#ifdef CONFIG_HAVE_RCU_TABLE_FREE
|
|
|
|
#define tlb_remove_entry(tlb, entry) tlb_remove_table(tlb, entry)
|
|
static inline void __tlb_remove_table(void *_table)
|
|
{
|
|
free_page_and_swap_cache((struct page *)_table);
|
|
}
|
|
#else
|
|
#define tlb_remove_entry(tlb, entry) tlb_remove_page(tlb, entry)
|
|
#endif /* CONFIG_HAVE_RCU_TABLE_FREE */
|
|
|
|
#include <asm-generic/tlb.h>
|
|
|
|
static inline void tlb_flush(struct mmu_gather *tlb)
|
|
{
|
|
if (tlb->fullmm) {
|
|
flush_tlb_mm(tlb->mm);
|
|
} else {
|
|
struct vm_area_struct vma = { .vm_mm = tlb->mm, };
|
|
/*
|
|
* The intermediate page table levels are already handled by
|
|
* the __(pte|pmd|pud)_free_tlb() functions, so last level
|
|
* TLBI is sufficient here.
|
|
*/
|
|
__flush_tlb_range(&vma, tlb->start, tlb->end, true);
|
|
}
|
|
}
|
|
|
|
static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
|
|
unsigned long addr)
|
|
{
|
|
__flush_tlb_pgtable(tlb->mm, addr);
|
|
pgtable_page_dtor(pte);
|
|
tlb_remove_entry(tlb, pte);
|
|
}
|
|
|
|
#if CONFIG_PGTABLE_LEVELS > 2
|
|
static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
|
|
unsigned long addr)
|
|
{
|
|
__flush_tlb_pgtable(tlb->mm, addr);
|
|
tlb_remove_entry(tlb, virt_to_page(pmdp));
|
|
}
|
|
#endif
|
|
|
|
#if CONFIG_PGTABLE_LEVELS > 3
|
|
static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp,
|
|
unsigned long addr)
|
|
{
|
|
__flush_tlb_pgtable(tlb->mm, addr);
|
|
tlb_remove_entry(tlb, virt_to_page(pudp));
|
|
}
|
|
#endif
|
|
|
|
#endif
|