From: David Gibson Ok, addressed several more obvious silly bugs. The context_reload thing is a bit more interesting - I've added another kinda-ugly hook so that we call the sparc specific code in hugetlb_prefault(). There may be a more sensible way to do this, but I don't know enough sparc to really do anything sensible with it. Cc: William Lee Irwin III Signed-off-by: Andrew Morton --- arch/sh/mm/hugetlbpage.c | 8 +++---- arch/sparc64/mm/hugetlbpage.c | 45 +++++++++++++++++++++++++++++++++++++----- include/asm-sparc64/page.h | 1 include/linux/hugetlb.h | 6 +++++ mm/hugetlb.c | 2 + 5 files changed, 53 insertions(+), 9 deletions(-) diff -puN arch/sh/mm/hugetlbpage.c~hugepage-consolidation-fix-fix arch/sh/mm/hugetlbpage.c --- 25/arch/sh/mm/hugetlbpage.c~hugepage-consolidation-fix-fix Wed May 4 12:51:47 2005 +++ 25-akpm/arch/sh/mm/hugetlbpage.c Wed May 4 12:51:47 2005 @@ -56,8 +56,8 @@ pte_t *huge_pte_offset(struct mm_struct #define mk_pte_huge(entry) do { pte_val(entry) |= _PAGE_SZHUGE; } while (0) -static void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t entry) +void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t entry) { int i; @@ -78,9 +78,9 @@ pte_t huge_ptep_get_and_clear(struct mm_ entry = *ptep; for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) { - pte_clear(mm, addr, pte); + pte_clear(mm, addr, ptep); addr += PAGE_SIZE; - pte++; + ptep++; } return entry; diff -puN arch/sparc64/mm/hugetlbpage.c~hugepage-consolidation-fix-fix arch/sparc64/mm/hugetlbpage.c --- 25/arch/sparc64/mm/hugetlbpage.c~hugepage-consolidation-fix-fix Wed May 4 12:51:47 2005 +++ 25-akpm/arch/sparc64/mm/hugetlbpage.c Wed May 4 12:51:47 2005 @@ -62,8 +62,8 @@ pte_t *huge_pte_offset(struct mm_struct #define mk_pte_huge(entry) do { pte_val(entry) |= _PAGE_SZHUGE; } while (0) -static void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, - pte_t *ptep, pte_t entry) +void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, + pte_t *ptep, pte_t entry) { int i; @@ -75,7 +75,8 @@ static void set_huge_pte_at(struct mm_st } } -pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) +pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, + pte_t *ptep) { pte_t entry; int i; @@ -83,8 +84,9 @@ pte_t huge_ptep_get_and_clear(struct mm_ entry = *ptep; for (i = 0; i < (1 << HUGETLB_PAGE_ORDER); i++) { - pte_clear(pte); - pte++; + pte_clear(mm, addr, ptep); + addr += PAGE_SIZE; + ptep++; } return entry; @@ -126,3 +128,36 @@ static void context_reload(void *__data) if (mm == current->mm) load_secondary_context(mm); } + +void hugetlb_prefault_arch_hook(struct mm_struct *mm) +{ + /* On UltraSPARC-III+ and later, configure the second half of + * the Data-TLB for huge pages. + */ + if (tlb_type == cheetah_plus) { + unsigned long ctx; + + spin_lock(&ctx_alloc_lock); + ctx = mm->context.sparc64_ctx_val; + ctx &= ~CTX_PGSZ_MASK; + ctx |= CTX_PGSZ_BASE << CTX_PGSZ0_SHIFT; + ctx |= CTX_PGSZ_HUGE << CTX_PGSZ1_SHIFT; + + if (ctx != mm->context.sparc64_ctx_val) { + /* When changing the page size fields, we + * must perform a context flush so that no + * stale entries match. This flush must + * occur with the original context register + * settings. + */ + do_flush_tlb_mm(mm); + + /* Reload the context register of all processors + * also executing in this address space. + */ + mm->context.sparc64_ctx_val = ctx; + on_each_cpu(context_reload, mm, 0, 0); + } + spin_unlock(&ctx_alloc_lock); + } +} diff -puN include/asm-sparc64/page.h~hugepage-consolidation-fix-fix include/asm-sparc64/page.h --- 25/include/asm-sparc64/page.h~hugepage-consolidation-fix-fix Wed May 4 12:51:47 2005 +++ 25-akpm/include/asm-sparc64/page.h Wed May 4 12:51:47 2005 @@ -96,6 +96,7 @@ typedef unsigned long pgprot_t; #define HPAGE_MASK (~(HPAGE_SIZE - 1UL)) #define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) #define ARCH_HAS_SETCLEAR_HUGE_PTE +#define ARCH_HAS_HUGETLB_PREFAULT_HOOK #endif #define TASK_UNMAPPED_BASE (test_thread_flag(TIF_32BIT) ? \ diff -puN include/linux/hugetlb.h~hugepage-consolidation-fix-fix include/linux/hugetlb.h --- 25/include/linux/hugetlb.h~hugepage-consolidation-fix-fix Wed May 4 12:51:47 2005 +++ 25-akpm/include/linux/hugetlb.h Wed May 4 12:51:47 2005 @@ -64,6 +64,12 @@ pte_t huge_ptep_get_and_clear(struct mm_ pte_t *ptep); #endif +#ifndef ARCH_HAS_HUGETLB_PREFAULT_HOOK +#define hugetlb_prefault_arch_hook(mm) do { } while (0) +#else +void hugetlb_prefault_arch_hook(struct mm_struct *mm); +#endif + #ifndef ARCH_HAS_HUGETLB_CLEAN_STALE_PGTABLE #define hugetlb_clean_stale_pgtable(pte) BUG() #else diff -puN mm/hugetlb.c~hugepage-consolidation-fix-fix mm/hugetlb.c --- 25/mm/hugetlb.c~hugepage-consolidation-fix-fix Wed May 4 12:51:47 2005 +++ 25-akpm/mm/hugetlb.c Wed May 4 12:51:47 2005 @@ -339,6 +339,8 @@ int hugetlb_prefault(struct address_spac BUG_ON(vma->vm_start & ~HPAGE_MASK); BUG_ON(vma->vm_end & ~HPAGE_MASK); + hugetlb_prefault_arch_hook(mm); + spin_lock(&mm->page_table_lock); for (addr = vma->vm_start; addr < vma->vm_end; addr += HPAGE_SIZE) { unsigned long idx; _