1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
|
From 3e69feb2525a857452ab975ee369be685faaa1d4 Mon Sep 17 00:00:00 2001
From: Ingo Molnar <mingo@elte.hu>
Date: Fri, 3 Jul 2009 08:44:01 -0500
Subject: [PATCH] softirq: per cpu assumptions fixes
commit 33fe41da821c6c542bcaa612a46695425525fdc6 in tip.
Signed-off-by: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Paul Gortmaker <paul.gortmaker@windriver.com>
---
kernel/hrtimer.c | 45 ++++++++++++++++++++++++++-------------------
kernel/sched_fair.c | 2 +-
kernel/timer.c | 2 +-
3 files changed, 28 insertions(+), 21 deletions(-)
diff --git a/kernel/hrtimer.c b/kernel/hrtimer.c
index c586357..82efa41 100644
--- a/kernel/hrtimer.c
+++ b/kernel/hrtimer.c
@@ -499,9 +499,9 @@ static inline int hrtimer_is_hres_enabled(void)
/*
* Is the high resolution mode active ?
*/
-static inline int hrtimer_hres_active(void)
+static inline int hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)
{
- return __get_cpu_var(hrtimer_bases).hres_active;
+ return cpu_base->hres_active;
}
/*
@@ -611,11 +611,11 @@ static int hrtimer_reprogram(struct hrtimer *timer,
*/
static void retrigger_next_event(void *arg)
{
- struct hrtimer_cpu_base *base;
+ struct hrtimer_cpu_base *base = &__get_cpu_var(hrtimer_bases);
struct timespec realtime_offset;
unsigned long seq;
- if (!hrtimer_hres_active())
+ if (!hrtimer_hres_active(base))
return;
do {
@@ -625,8 +625,6 @@ static void retrigger_next_event(void *arg)
-wall_to_monotonic.tv_nsec);
} while (read_raw_seqretry(&xtime_lock, seq));
- base = &__get_cpu_var(hrtimer_bases);
-
/* Adjust CLOCK_REALTIME offset */
raw_spin_lock(&base->lock);
base->clock_base[CLOCK_REALTIME].offset =
@@ -709,10 +707,8 @@ static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,
/*
* Switch to high resolution mode
*/
-static int hrtimer_switch_to_hres(void)
+static int hrtimer_switch_to_hres(struct hrtimer_cpu_base *base)
{
- int cpu = smp_processor_id();
- struct hrtimer_cpu_base *base = &per_cpu(hrtimer_bases, cpu);
unsigned long flags;
if (base->hres_active)
@@ -723,7 +719,7 @@ static int hrtimer_switch_to_hres(void)
if (tick_init_highres()) {
local_irq_restore(flags);
printk(KERN_WARNING "Could not switch to high resolution "
- "mode on CPU %d\n", cpu);
+ "mode on CPU %d\n", raw_smp_processor_id());
return 0;
}
base->hres_active = 1;
@@ -740,9 +736,15 @@ static int hrtimer_switch_to_hres(void)
#else
-static inline int hrtimer_hres_active(void) { return 0; }
+static inline int hrtimer_hres_active(struct hrtimer_cpu_base *base)
+{
+ return 0;
+}
static inline int hrtimer_is_hres_enabled(void) { return 0; }
-static inline int hrtimer_switch_to_hres(void) { return 0; }
+static inline int hrtimer_switch_to_hres(struct hrtimer_cpu_base *base)
+{
+ return 0;
+}
static inline void
hrtimer_force_reprogram(struct hrtimer_cpu_base *base, int skip_equal) { }
static inline int hrtimer_enqueue_reprogram(struct hrtimer *timer,
@@ -940,7 +942,7 @@ static void __remove_hrtimer(struct hrtimer *timer,
base->first = rb_next(&timer->node);
#ifdef CONFIG_HIGH_RES_TIMERS
/* Reprogram the clock event device. if enabled */
- if (reprogram && hrtimer_hres_active()) {
+ if (reprogram && hrtimer_hres_active(base->cpu_base)) {
ktime_t expires;
expires = ktime_sub(hrtimer_get_expires(timer),
@@ -1153,7 +1155,7 @@ ktime_t hrtimer_get_next_event(void)
raw_spin_lock_irqsave(&cpu_base->lock, flags);
- if (!hrtimer_hres_active()) {
+ if (!hrtimer_hres_active(cpu_base)) {
for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++, base++) {
struct hrtimer *timer;
@@ -1398,9 +1400,11 @@ retry:
*/
static void __hrtimer_peek_ahead_timers(void)
{
+ struct hrtimer_cpu_base *cpu_base;
struct tick_device *td;
- if (!hrtimer_hres_active())
+ cpu_base = &__get_cpu_var(hrtimer_bases);
+ if (!hrtimer_hres_active(cpu_base))
return;
td = &__get_cpu_var(tick_cpu_device);
@@ -1446,7 +1450,9 @@ static inline void __hrtimer_peek_ahead_timers(void) { }
*/
void hrtimer_run_pending(void)
{
- if (hrtimer_hres_active())
+ struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases);
+
+ if (hrtimer_hres_active(cpu_base))
return;
/*
@@ -1458,7 +1464,7 @@ void hrtimer_run_pending(void)
* deadlock vs. xtime_lock.
*/
if (tick_check_oneshot_change(!hrtimer_is_hres_enabled()))
- hrtimer_switch_to_hres();
+ hrtimer_switch_to_hres(cpu_base);
}
/*
@@ -1467,11 +1473,12 @@ void hrtimer_run_pending(void)
void hrtimer_run_queues(void)
{
struct rb_node *node;
- struct hrtimer_cpu_base *cpu_base = &__get_cpu_var(hrtimer_bases);
+ struct hrtimer_cpu_base *cpu_base;
struct hrtimer_clock_base *base;
int index, gettime = 1;
- if (hrtimer_hres_active())
+ cpu_base = &per_cpu(hrtimer_bases, raw_smp_processor_id());
+ if (hrtimer_hres_active(cpu_base))
return;
for (index = 0; index < HRTIMER_MAX_CLOCK_BASES; index++) {
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index b944efc..b873769 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -3438,7 +3438,7 @@ out:
*/
static void run_rebalance_domains(struct softirq_action *h)
{
- int this_cpu = smp_processor_id();
+ int this_cpu = raw_smp_processor_id();
struct rq *this_rq = cpu_rq(this_cpu);
enum cpu_idle_type idle = this_rq->idle_at_tick ?
CPU_IDLE : CPU_NOT_IDLE;
diff --git a/kernel/timer.c b/kernel/timer.c
index c9820fa..f793f67 100644
--- a/kernel/timer.c
+++ b/kernel/timer.c
@@ -1276,7 +1276,7 @@ void update_process_times(int user_tick)
*/
static void run_timer_softirq(struct softirq_action *h)
{
- struct tvec_base *base = __get_cpu_var(tvec_bases);
+ struct tvec_base *base = per_cpu(tvec_bases, raw_smp_processor_id());
printk_tick();
hrtimer_run_pending();
--
1.7.0.4
|
