From: Ingo Molnar Date: Fri, 3 Jul 2009 08:29:37 -0500 Subject: mm: page_alloc: rt-friendly per-cpu pages rt-friendly per-cpu pages: convert the irqs-off per-cpu locking method into a preemptible, explicit-per-cpu-locks method. Contains fixes from: Peter Zijlstra Thomas Gleixner Signed-off-by: Ingo Molnar Signed-off-by: Thomas Gleixner --- mm/page_alloc.c | 57 ++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 18 deletions(-) Index: linux-stable/mm/page_alloc.c =================================================================== --- linux-stable.orig/mm/page_alloc.c +++ linux-stable/mm/page_alloc.c @@ -58,6 +58,7 @@ #include #include #include +#include #include #include @@ -216,6 +217,18 @@ EXPORT_SYMBOL(nr_node_ids); EXPORT_SYMBOL(nr_online_nodes); #endif +static DEFINE_LOCAL_IRQ_LOCK(pa_lock); + +#ifdef CONFIG_PREEMPT_RT_BASE +# define cpu_lock_irqsave(cpu, flags) \ + spin_lock_irqsave(&per_cpu(pa_lock, cpu).lock, flags) +# define cpu_unlock_irqrestore(cpu, flags) \ + spin_unlock_irqrestore(&per_cpu(pa_lock, cpu).lock, flags) +#else +# define cpu_lock_irqsave(cpu, flags) local_irq_save(flags) +# define cpu_unlock_irqrestore(cpu, flags) local_irq_restore(flags) +#endif + int page_group_by_mobility_disabled __read_mostly; /* @@ -726,13 +739,13 @@ static void __free_pages_ok(struct page if (!free_pages_prepare(page, order)) return; - local_irq_save(flags); + local_lock_irqsave(pa_lock, flags); if (unlikely(wasMlocked)) free_page_mlock(page); __count_vm_events(PGFREE, 1 << order); free_one_page(page_zone(page), page, order, get_pageblock_migratetype(page)); - local_irq_restore(flags); + local_unlock_irqrestore(pa_lock, flags); } void __meminit __free_pages_bootmem(struct page *page, unsigned int order) @@ -1157,7 +1170,7 @@ void drain_zone_pages(struct zone *zone, unsigned long flags; int to_drain; - local_irq_save(flags); + local_lock_irqsave(pa_lock, flags); if (pcp->count >= pcp->batch) to_drain = pcp->batch; else @@ -1166,7 +1179,7 @@ void drain_zone_pages(struct zone *zone, free_pcppages_bulk(zone, to_drain, pcp); pcp->count -= to_drain; } - local_irq_restore(flags); + local_unlock_irqrestore(pa_lock, flags); } #endif @@ -1186,7 +1199,7 @@ static void drain_pages(unsigned int cpu struct per_cpu_pageset *pset; struct per_cpu_pages *pcp; - local_irq_save(flags); + cpu_lock_irqsave(cpu, flags); pset = per_cpu_ptr(zone->pageset, cpu); pcp = &pset->pcp; @@ -1194,7 +1207,7 @@ static void drain_pages(unsigned int cpu free_pcppages_bulk(zone, pcp->count, pcp); pcp->count = 0; } - local_irq_restore(flags); + cpu_unlock_irqrestore(cpu, flags); } } @@ -1247,7 +1260,12 @@ void drain_all_pages(void) else cpumask_clear_cpu(cpu, &cpus_with_pcps); } +#ifndef CONFIG_PREEMPT_RT_BASE on_each_cpu_mask(&cpus_with_pcps, drain_local_pages, NULL, 1); +#else + for_each_cpu(cpu, &cpus_with_pcps) + drain_pages(cpu); +#endif } #ifdef CONFIG_HIBERNATION @@ -1303,7 +1321,7 @@ void free_hot_cold_page(struct page *pag migratetype = get_pageblock_migratetype(page); set_page_private(page, migratetype); - local_irq_save(flags); + local_lock_irqsave(pa_lock, flags); if (unlikely(wasMlocked)) free_page_mlock(page); __count_vm_event(PGFREE); @@ -1335,7 +1353,7 @@ void free_hot_cold_page(struct page *pag } out: - local_irq_restore(flags); + local_unlock_irqrestore(pa_lock, flags); } /* @@ -1447,7 +1465,7 @@ again: struct per_cpu_pages *pcp; struct list_head *list; - local_irq_save(flags); + local_lock_irqsave(pa_lock, flags); pcp = &this_cpu_ptr(zone->pageset)->pcp; list = &pcp->lists[migratetype]; if (list_empty(list)) { @@ -1479,17 +1497,19 @@ again: */ WARN_ON_ONCE(order > 1); } - spin_lock_irqsave(&zone->lock, flags); + local_spin_lock_irqsave(pa_lock, &zone->lock, flags); page = __rmqueue(zone, order, migratetype); - spin_unlock(&zone->lock); - if (!page) + if (!page) { + spin_unlock(&zone->lock); goto failed; + } __mod_zone_page_state(zone, NR_FREE_PAGES, -(1 << order)); + spin_unlock(&zone->lock); } __count_zone_vm_events(PGALLOC, zone, 1 << order); zone_statistics(preferred_zone, zone, gfp_flags); - local_irq_restore(flags); + local_unlock_irqrestore(pa_lock, flags); VM_BUG_ON(bad_range(zone, page)); if (prep_new_page(page, order, gfp_flags)) @@ -1497,7 +1517,7 @@ again: return page; failed: - local_irq_restore(flags); + local_unlock_irqrestore(pa_lock, flags); return NULL; } @@ -4992,6 +5012,7 @@ static int page_alloc_cpu_notify(struct void __init page_alloc_init(void) { hotcpu_notifier(page_alloc_cpu_notify, 0); + local_irq_lock_init(pa_lock); } /* @@ -5871,11 +5892,11 @@ static int __meminit __zone_pcp_update(v pset = per_cpu_ptr(zone->pageset, cpu); pcp = &pset->pcp; - local_irq_save(flags); + cpu_lock_irqsave(cpu, flags); if (pcp->count > 0) free_pcppages_bulk(zone, pcp->count, pcp); setup_pageset(pset, batch); - local_irq_restore(flags); + cpu_unlock_irqrestore(cpu, flags); } return 0; } @@ -5892,12 +5913,12 @@ void zone_pcp_reset(struct zone *zone) unsigned long flags; /* avoid races with drain_pages() */ - local_irq_save(flags); + local_lock_irqsave(pa_lock, flags); if (zone->pageset != &boot_pageset) { free_percpu(zone->pageset); zone->pageset = &boot_pageset; } - local_irq_restore(flags); + local_unlock_irqrestore(pa_lock, flags); } /*