mm: introduce ctor/dtor at PGD level
Following on from the introduction of P4D-level ctor/dtor, let's finish the job and introduce ctor/dtor at PGD level. The incurred improvement in page accounting is minimal - the main motivation is to create a single, generic place where construction/destruction hooks can be added for all page table pages. This patch should cover all architectures and all configurations where PGDs are one or more regular pages. This excludes any configuration where PGDs are allocated from a kmem_cache object. Link: https://lkml.kernel.org/r/20250103184415.2744423-7-kevin.brodsky@arm.com Signed-off-by: Kevin Brodsky <kevin.brodsky@arm.com> Acked-by: Dave Hansen <dave.hansen@linux.intel.com> Acked-by: Qi Zheng <zhengqi.arch@bytedance.com> Cc: Andy Lutomirski <luto@kernel.org> Cc: Catalin Marinas <catalin.marinas@arm.com> Cc: Ingo Molnar <mingo@elte.hu> Cc: Linus Walleij <linus.walleij@linaro.org> Cc: Matthew Wilcox (Oracle) <willy@infradead.org> Cc: Mike Rapoport (Microsoft) <rppt@kernel.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Ryan Roberts <ryan.roberts@arm.com> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Will Deacon <will@kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
This commit is contained in:
committed by
Andrew Morton
parent
a9b3c355c2
commit
d95936a226
@@ -73,7 +73,7 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t pgtable)
|
||||
|
||||
static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
|
||||
{
|
||||
pagetable_free(virt_to_ptdesc(pgd));
|
||||
pagetable_dtor_free(virt_to_ptdesc(pgd));
|
||||
}
|
||||
|
||||
static inline pgd_t *pgd_alloc(struct mm_struct *mm)
|
||||
@@ -84,6 +84,7 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm)
|
||||
|
||||
if (!ptdesc)
|
||||
return NULL;
|
||||
pagetable_pgd_ctor(ptdesc);
|
||||
new_pgd = ptdesc_address(ptdesc);
|
||||
|
||||
memcpy(new_pgd, swapper_pg_dir, PTRS_PER_PGD * sizeof(pgd_t));
|
||||
|
||||
@@ -169,6 +169,9 @@ void *get_pointer_table(int type)
|
||||
case TABLE_PMD:
|
||||
pagetable_pmd_ctor(virt_to_ptdesc(page));
|
||||
break;
|
||||
case TABLE_PGD:
|
||||
pagetable_pgd_ctor(virt_to_ptdesc(page));
|
||||
break;
|
||||
}
|
||||
|
||||
mmu_page_ctor(page);
|
||||
@@ -207,8 +210,7 @@ int free_pointer_table(void *table, int type)
|
||||
/* all tables in page are free, free page */
|
||||
list_del(dp);
|
||||
mmu_page_dtor((void *)page);
|
||||
if (type == TABLE_PTE || type == TABLE_PMD)
|
||||
pagetable_dtor(virt_to_ptdesc((void *)page));
|
||||
pagetable_dtor(virt_to_ptdesc((void *)page));
|
||||
free_page (page);
|
||||
return 1;
|
||||
} else if (ptable_list[type].next != dp) {
|
||||
|
||||
@@ -130,11 +130,18 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd)
|
||||
|
||||
static inline pgd_t *pgd_alloc(struct mm_struct *mm)
|
||||
{
|
||||
return (pgd_t *) crst_table_alloc(mm);
|
||||
unsigned long *table = crst_table_alloc(mm);
|
||||
|
||||
if (!table)
|
||||
return NULL;
|
||||
pagetable_pgd_ctor(virt_to_ptdesc(table));
|
||||
|
||||
return (pgd_t *) table;
|
||||
}
|
||||
|
||||
static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
|
||||
{
|
||||
pagetable_dtor(virt_to_ptdesc(pgd));
|
||||
crst_table_free(mm, (unsigned long *) pgd);
|
||||
}
|
||||
|
||||
|
||||
@@ -271,6 +271,7 @@ static inline pgd_t *__pgd_alloc_noprof(struct mm_struct *mm, unsigned int order
|
||||
if (!ptdesc)
|
||||
return NULL;
|
||||
|
||||
pagetable_pgd_ctor(ptdesc);
|
||||
return ptdesc_address(ptdesc);
|
||||
}
|
||||
#define __pgd_alloc(...) alloc_hooks(__pgd_alloc_noprof(__VA_ARGS__))
|
||||
@@ -280,7 +281,7 @@ static inline void __pgd_free(struct mm_struct *mm, pgd_t *pgd)
|
||||
struct ptdesc *ptdesc = virt_to_ptdesc(pgd);
|
||||
|
||||
BUG_ON((unsigned long)pgd & (PAGE_SIZE-1));
|
||||
pagetable_free(ptdesc);
|
||||
pagetable_dtor_free(ptdesc);
|
||||
}
|
||||
|
||||
#ifndef __HAVE_ARCH_PGD_FREE
|
||||
|
||||
@@ -3159,6 +3159,11 @@ static inline void pagetable_p4d_ctor(struct ptdesc *ptdesc)
|
||||
__pagetable_ctor(ptdesc);
|
||||
}
|
||||
|
||||
static inline void pagetable_pgd_ctor(struct ptdesc *ptdesc)
|
||||
{
|
||||
__pagetable_ctor(ptdesc);
|
||||
}
|
||||
|
||||
extern void __init pagecache_init(void);
|
||||
extern void free_initmem(void);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user