285994a62c
The ARM architecture allows the caching of intermediate page table levels and page table freeing requires a sequence like: pmd_clear() TLB invalidation pte page freeing With commit5e5f6dc105(arm64: mm: enable HAVE_RCU_TABLE_FREE logic), the page table freeing batching was moved from tlb_remove_page() to tlb_remove_table(). The former takes care of TLB invalidation as this is also shared with pte clearing and page cache page freeing. The latter, however, does not invalidate the TLBs for intermediate page table levels as it probably relies on the architecture code to do it if required. When the mm->mm_users < 2, tlb_remove_table() does not do any batching and page table pages are freed before tlb_finish_mmu() which performs the actual TLB invalidation. This patch introduces __tlb_flush_pgtable() for arm64 and calls it from the {pte,pmd,pud}_free_tlb() directly without relying on deferred page table freeing. Fixes:5e5f6dc105arm64: mm: enable HAVE_RCU_TABLE_FREE logic Reported-by: Jon Masters <jcm@redhat.com> Tested-by: Jon Masters <jcm@redhat.com> Tested-by: Steve Capper <steve.capper@linaro.org> Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
74 lines
2 KiB
C
74 lines
2 KiB
C
/*
|
|
* Based on arch/arm/include/asm/tlb.h
|
|
*
|
|
* Copyright (C) 2002 Russell King
|
|
* Copyright (C) 2012 ARM Ltd.
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
* published by the Free Software Foundation.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*/
|
|
#ifndef __ASM_TLB_H
|
|
#define __ASM_TLB_H
|
|
|
|
#include <linux/pagemap.h>
|
|
#include <linux/swap.h>
|
|
|
|
#ifdef CONFIG_HAVE_RCU_TABLE_FREE
|
|
|
|
#define tlb_remove_entry(tlb, entry) tlb_remove_table(tlb, entry)
|
|
static inline void __tlb_remove_table(void *_table)
|
|
{
|
|
free_page_and_swap_cache((struct page *)_table);
|
|
}
|
|
#else
|
|
#define tlb_remove_entry(tlb, entry) tlb_remove_page(tlb, entry)
|
|
#endif /* CONFIG_HAVE_RCU_TABLE_FREE */
|
|
|
|
#include <asm-generic/tlb.h>
|
|
|
|
static inline void tlb_flush(struct mmu_gather *tlb)
|
|
{
|
|
if (tlb->fullmm) {
|
|
flush_tlb_mm(tlb->mm);
|
|
} else {
|
|
struct vm_area_struct vma = { .vm_mm = tlb->mm, };
|
|
flush_tlb_range(&vma, tlb->start, tlb->end);
|
|
}
|
|
}
|
|
|
|
static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte,
|
|
unsigned long addr)
|
|
{
|
|
__flush_tlb_pgtable(tlb->mm, addr);
|
|
pgtable_page_dtor(pte);
|
|
tlb_remove_entry(tlb, pte);
|
|
}
|
|
|
|
#if CONFIG_ARM64_PGTABLE_LEVELS > 2
|
|
static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp,
|
|
unsigned long addr)
|
|
{
|
|
__flush_tlb_pgtable(tlb->mm, addr);
|
|
tlb_remove_entry(tlb, virt_to_page(pmdp));
|
|
}
|
|
#endif
|
|
|
|
#if CONFIG_ARM64_PGTABLE_LEVELS > 3
|
|
static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp,
|
|
unsigned long addr)
|
|
{
|
|
__flush_tlb_pgtable(tlb->mm, addr);
|
|
tlb_remove_entry(tlb, virt_to_page(pudp));
|
|
}
|
|
#endif
|
|
|
|
#endif
|