diff options
| author | Mike Galbraith <umgwanakikbuti@gmail.com> | 2014-05-02 13:13:34 +0200 |
|---|---|---|
| committer | Sebastian Andrzej Siewior <bigeasy@linutronix.de> | 2016-02-13 00:36:10 +0100 |
| commit | 489797f2951364b8bec8baa843aa02af6b2c7ef7 (patch) | |
| tree | 28ce154ac9958c47fbd2970893493b9c6a19fae0 | |
| parent | b429a3edaee5673132f515c6c68e33118f3a4473 (diff) | |
| download | rt-linux-489797f2951364b8bec8baa843aa02af6b2c7ef7.tar.gz | |
stomp-machine: use lg_global_trylock_relax() to dead with stop_cpus_lock lglock
If the stop machinery is called from inactive CPU we cannot use
lg_global_lock(), because some other stomp machine invocation might be
in progress and the lock can be contended. We cannot schedule from this
context, so use the lovely new lg_global_trylock_relax() primitive to
do what we used to do via one mutex_trylock()/cpu_relax() loop. We
now do that trylock()/relax() across an entire herd of locks. Joy.
Signed-off-by: Mike Galbraith <umgwanakikbuti@gmail.com>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
| -rw-r--r-- | kernel/stop_machine.c | 25 |
1 files changed, 15 insertions, 10 deletions
diff --git a/kernel/stop_machine.c b/kernel/stop_machine.c index f33914197161e..2c5acc882bada 100644 --- a/kernel/stop_machine.c +++ b/kernel/stop_machine.c @@ -276,7 +276,7 @@ int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void * struct cpu_stop_work work1, work2; struct multi_stop_data msdata; - preempt_disable(); + preempt_disable_nort(); msdata = (struct multi_stop_data){ .fn = fn, .data = arg, @@ -296,11 +296,11 @@ int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void * if (cpu1 > cpu2) swap(cpu1, cpu2); if (cpu_stop_queue_two_works(cpu1, &work1, cpu2, &work2)) { - preempt_enable(); + preempt_enable_nort(); return -ENOENT; } - preempt_enable(); + preempt_enable_nort(); wait_for_stop_done(&done); @@ -333,17 +333,20 @@ static DEFINE_MUTEX(stop_cpus_mutex); static void queue_stop_cpus_work(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg, - struct cpu_stop_done *done) + struct cpu_stop_done *done, bool inactive) { struct cpu_stop_work *work; unsigned int cpu; /* - * Disable preemption while queueing to avoid getting - * preempted by a stopper which might wait for other stoppers - * to enter @fn which can lead to deadlock. + * Make sure that all work is queued on all cpus before + * any of the cpus can execute it. */ - lg_global_lock(&stop_cpus_lock); + if (!inactive) + lg_global_lock(&stop_cpus_lock); + else + lg_global_trylock_relax(&stop_cpus_lock); + for_each_cpu(cpu, cpumask) { work = &per_cpu(cpu_stopper.stop_work, cpu); work->fn = fn; @@ -360,7 +363,7 @@ static int __stop_cpus(const struct cpumask *cpumask, struct cpu_stop_done done; cpu_stop_init_done(&done, cpumask_weight(cpumask)); - queue_stop_cpus_work(cpumask, fn, arg, &done); + queue_stop_cpus_work(cpumask, fn, arg, &done, false); wait_for_stop_done(&done); return done.executed ? done.ret : -ENOENT; } @@ -558,6 +561,8 @@ static int __init cpu_stop_init(void) INIT_LIST_HEAD(&stopper->works); } + lg_lock_init(&stop_cpus_lock, "stop_cpus_lock"); + BUG_ON(smpboot_register_percpu_thread(&cpu_stop_threads)); stop_machine_unpark(raw_smp_processor_id()); stop_machine_initialized = true; @@ -654,7 +659,7 @@ int stop_machine_from_inactive_cpu(cpu_stop_fn_t fn, void *data, set_state(&msdata, MULTI_STOP_PREPARE); cpu_stop_init_done(&done, num_active_cpus()); queue_stop_cpus_work(cpu_active_mask, multi_cpu_stop, &msdata, - &done); + &done, true); ret = multi_cpu_stop(&msdata); /* Busy wait for completion. */ |