]>
Commit | Line | Data |
---|---|---|
1 | /* | |
2 | * Copyright (2004) Linus Torvalds | |
3 | * | |
4 | * Author: Zwane Mwaikambo <zwane@fsmlabs.com> | |
5 | * | |
6 | * Copyright (2004, 2005) Ingo Molnar | |
7 | * | |
8 | * This file contains the spinlock/rwlock implementations for the | |
9 | * SMP and the DEBUG_SPINLOCK cases. (UP-nondebug inlines them) | |
10 | * | |
11 | * Note that some architectures have special knowledge about the | |
12 | * stack frames of these functions in their profile_pc. If you | |
13 | * change anything significant here that could change the stack | |
14 | * frame contact the architecture maintainers. | |
15 | */ | |
16 | ||
17 | #include <linux/linkage.h> | |
18 | #include <linux/preempt.h> | |
19 | #include <linux/spinlock.h> | |
20 | #include <linux/interrupt.h> | |
21 | #include <linux/debug_locks.h> | |
22 | #include <linux/module.h> | |
23 | ||
24 | /* | |
25 | * If lockdep is enabled then we use the non-preemption spin-ops | |
26 | * even on CONFIG_PREEMPT, because lockdep assumes that interrupts are | |
27 | * not re-enabled during lock-acquire (which the preempt-spin-ops do): | |
28 | */ | |
29 | #if !defined(CONFIG_GENERIC_LOCKBREAK) || defined(CONFIG_DEBUG_LOCK_ALLOC) | |
30 | /* | |
31 | * The __lock_function inlines are taken from | |
32 | * include/linux/spinlock_api_smp.h | |
33 | */ | |
34 | #else | |
35 | #define raw_read_can_lock(l) read_can_lock(l) | |
36 | #define raw_write_can_lock(l) write_can_lock(l) | |
37 | /* | |
38 | * We build the __lock_function inlines here. They are too large for | |
39 | * inlining all over the place, but here is only one user per function | |
40 | * which embedds them into the calling _lock_function below. | |
41 | * | |
42 | * This could be a long-held lock. We both prepare to spin for a long | |
43 | * time (making _this_ CPU preemptable if possible), and we also signal | |
44 | * towards that other CPU that it should break the lock ASAP. | |
45 | */ | |
46 | #define BUILD_LOCK_OPS(op, locktype) \ | |
47 | void __lockfunc __raw_##op##_lock(locktype##_t *lock) \ | |
48 | { \ | |
49 | for (;;) { \ | |
50 | preempt_disable(); \ | |
51 | if (likely(do_raw_##op##_trylock(lock))) \ | |
52 | break; \ | |
53 | preempt_enable(); \ | |
54 | \ | |
55 | if (!(lock)->break_lock) \ | |
56 | (lock)->break_lock = 1; \ | |
57 | while (!raw_##op##_can_lock(lock) && (lock)->break_lock)\ | |
58 | arch_##op##_relax(&lock->raw_lock); \ | |
59 | } \ | |
60 | (lock)->break_lock = 0; \ | |
61 | } \ | |
62 | \ | |
63 | unsigned long __lockfunc __raw_##op##_lock_irqsave(locktype##_t *lock) \ | |
64 | { \ | |
65 | unsigned long flags; \ | |
66 | \ | |
67 | for (;;) { \ | |
68 | preempt_disable(); \ | |
69 | local_irq_save(flags); \ | |
70 | if (likely(do_raw_##op##_trylock(lock))) \ | |
71 | break; \ | |
72 | local_irq_restore(flags); \ | |
73 | preempt_enable(); \ | |
74 | \ | |
75 | if (!(lock)->break_lock) \ | |
76 | (lock)->break_lock = 1; \ | |
77 | while (!raw_##op##_can_lock(lock) && (lock)->break_lock)\ | |
78 | arch_##op##_relax(&lock->raw_lock); \ | |
79 | } \ | |
80 | (lock)->break_lock = 0; \ | |
81 | return flags; \ | |
82 | } \ | |
83 | \ | |
84 | void __lockfunc __raw_##op##_lock_irq(locktype##_t *lock) \ | |
85 | { \ | |
86 | _raw_##op##_lock_irqsave(lock); \ | |
87 | } \ | |
88 | \ | |
89 | void __lockfunc __raw_##op##_lock_bh(locktype##_t *lock) \ | |
90 | { \ | |
91 | unsigned long flags; \ | |
92 | \ | |
93 | /* */ \ | |
94 | /* Careful: we must exclude softirqs too, hence the */ \ | |
95 | /* irq-disabling. We use the generic preemption-aware */ \ | |
96 | /* function: */ \ | |
97 | /**/ \ | |
98 | flags = _raw_##op##_lock_irqsave(lock); \ | |
99 | local_bh_disable(); \ | |
100 | local_irq_restore(flags); \ | |
101 | } \ | |
102 | ||
103 | /* | |
104 | * Build preemption-friendly versions of the following | |
105 | * lock-spinning functions: | |
106 | * | |
107 | * __[spin|read|write]_lock() | |
108 | * __[spin|read|write]_lock_irq() | |
109 | * __[spin|read|write]_lock_irqsave() | |
110 | * __[spin|read|write]_lock_bh() | |
111 | */ | |
112 | BUILD_LOCK_OPS(spin, raw_spinlock); | |
113 | BUILD_LOCK_OPS(read, rwlock); | |
114 | BUILD_LOCK_OPS(write, rwlock); | |
115 | ||
116 | #endif | |
117 | ||
118 | #ifndef CONFIG_INLINE_SPIN_TRYLOCK | |
119 | int __lockfunc _raw_spin_trylock(raw_spinlock_t *lock) | |
120 | { | |
121 | return __raw_spin_trylock(lock); | |
122 | } | |
123 | EXPORT_SYMBOL(_raw_spin_trylock); | |
124 | #endif | |
125 | ||
126 | #ifndef CONFIG_INLINE_SPIN_TRYLOCK_BH | |
127 | int __lockfunc _raw_spin_trylock_bh(raw_spinlock_t *lock) | |
128 | { | |
129 | return __raw_spin_trylock_bh(lock); | |
130 | } | |
131 | EXPORT_SYMBOL(_raw_spin_trylock_bh); | |
132 | #endif | |
133 | ||
134 | #ifndef CONFIG_INLINE_SPIN_LOCK | |
135 | void __lockfunc _raw_spin_lock(raw_spinlock_t *lock) | |
136 | { | |
137 | __raw_spin_lock(lock); | |
138 | } | |
139 | EXPORT_SYMBOL(_raw_spin_lock); | |
140 | #endif | |
141 | ||
142 | #ifndef CONFIG_INLINE_SPIN_LOCK_IRQSAVE | |
143 | unsigned long __lockfunc _raw_spin_lock_irqsave(raw_spinlock_t *lock) | |
144 | { | |
145 | return __raw_spin_lock_irqsave(lock); | |
146 | } | |
147 | EXPORT_SYMBOL(_raw_spin_lock_irqsave); | |
148 | #endif | |
149 | ||
150 | #ifndef CONFIG_INLINE_SPIN_LOCK_IRQ | |
151 | void __lockfunc _raw_spin_lock_irq(raw_spinlock_t *lock) | |
152 | { | |
153 | __raw_spin_lock_irq(lock); | |
154 | } | |
155 | EXPORT_SYMBOL(_raw_spin_lock_irq); | |
156 | #endif | |
157 | ||
158 | #ifndef CONFIG_INLINE_SPIN_LOCK_BH | |
159 | void __lockfunc _raw_spin_lock_bh(raw_spinlock_t *lock) | |
160 | { | |
161 | __raw_spin_lock_bh(lock); | |
162 | } | |
163 | EXPORT_SYMBOL(_raw_spin_lock_bh); | |
164 | #endif | |
165 | ||
166 | #ifndef CONFIG_INLINE_SPIN_UNLOCK | |
167 | void __lockfunc _raw_spin_unlock(raw_spinlock_t *lock) | |
168 | { | |
169 | __raw_spin_unlock(lock); | |
170 | } | |
171 | EXPORT_SYMBOL(_raw_spin_unlock); | |
172 | #endif | |
173 | ||
174 | #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQRESTORE | |
175 | void __lockfunc _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) | |
176 | { | |
177 | __raw_spin_unlock_irqrestore(lock, flags); | |
178 | } | |
179 | EXPORT_SYMBOL(_raw_spin_unlock_irqrestore); | |
180 | #endif | |
181 | ||
182 | #ifndef CONFIG_INLINE_SPIN_UNLOCK_IRQ | |
183 | void __lockfunc _raw_spin_unlock_irq(raw_spinlock_t *lock) | |
184 | { | |
185 | __raw_spin_unlock_irq(lock); | |
186 | } | |
187 | EXPORT_SYMBOL(_raw_spin_unlock_irq); | |
188 | #endif | |
189 | ||
190 | #ifndef CONFIG_INLINE_SPIN_UNLOCK_BH | |
191 | void __lockfunc _raw_spin_unlock_bh(raw_spinlock_t *lock) | |
192 | { | |
193 | __raw_spin_unlock_bh(lock); | |
194 | } | |
195 | EXPORT_SYMBOL(_raw_spin_unlock_bh); | |
196 | #endif | |
197 | ||
198 | #ifndef CONFIG_INLINE_READ_TRYLOCK | |
199 | int __lockfunc _raw_read_trylock(rwlock_t *lock) | |
200 | { | |
201 | return __raw_read_trylock(lock); | |
202 | } | |
203 | EXPORT_SYMBOL(_raw_read_trylock); | |
204 | #endif | |
205 | ||
206 | #ifndef CONFIG_INLINE_READ_LOCK | |
207 | void __lockfunc _raw_read_lock(rwlock_t *lock) | |
208 | { | |
209 | __raw_read_lock(lock); | |
210 | } | |
211 | EXPORT_SYMBOL(_raw_read_lock); | |
212 | #endif | |
213 | ||
214 | #ifndef CONFIG_INLINE_READ_LOCK_IRQSAVE | |
215 | unsigned long __lockfunc _raw_read_lock_irqsave(rwlock_t *lock) | |
216 | { | |
217 | return __raw_read_lock_irqsave(lock); | |
218 | } | |
219 | EXPORT_SYMBOL(_raw_read_lock_irqsave); | |
220 | #endif | |
221 | ||
222 | #ifndef CONFIG_INLINE_READ_LOCK_IRQ | |
223 | void __lockfunc _raw_read_lock_irq(rwlock_t *lock) | |
224 | { | |
225 | __raw_read_lock_irq(lock); | |
226 | } | |
227 | EXPORT_SYMBOL(_raw_read_lock_irq); | |
228 | #endif | |
229 | ||
230 | #ifndef CONFIG_INLINE_READ_LOCK_BH | |
231 | void __lockfunc _raw_read_lock_bh(rwlock_t *lock) | |
232 | { | |
233 | __raw_read_lock_bh(lock); | |
234 | } | |
235 | EXPORT_SYMBOL(_raw_read_lock_bh); | |
236 | #endif | |
237 | ||
238 | #ifndef CONFIG_INLINE_READ_UNLOCK | |
239 | void __lockfunc _raw_read_unlock(rwlock_t *lock) | |
240 | { | |
241 | __raw_read_unlock(lock); | |
242 | } | |
243 | EXPORT_SYMBOL(_raw_read_unlock); | |
244 | #endif | |
245 | ||
246 | #ifndef CONFIG_INLINE_READ_UNLOCK_IRQRESTORE | |
247 | void __lockfunc _raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) | |
248 | { | |
249 | __raw_read_unlock_irqrestore(lock, flags); | |
250 | } | |
251 | EXPORT_SYMBOL(_raw_read_unlock_irqrestore); | |
252 | #endif | |
253 | ||
254 | #ifndef CONFIG_INLINE_READ_UNLOCK_IRQ | |
255 | void __lockfunc _raw_read_unlock_irq(rwlock_t *lock) | |
256 | { | |
257 | __raw_read_unlock_irq(lock); | |
258 | } | |
259 | EXPORT_SYMBOL(_raw_read_unlock_irq); | |
260 | #endif | |
261 | ||
262 | #ifndef CONFIG_INLINE_READ_UNLOCK_BH | |
263 | void __lockfunc _raw_read_unlock_bh(rwlock_t *lock) | |
264 | { | |
265 | __raw_read_unlock_bh(lock); | |
266 | } | |
267 | EXPORT_SYMBOL(_raw_read_unlock_bh); | |
268 | #endif | |
269 | ||
270 | #ifndef CONFIG_INLINE_WRITE_TRYLOCK | |
271 | int __lockfunc _raw_write_trylock(rwlock_t *lock) | |
272 | { | |
273 | return __raw_write_trylock(lock); | |
274 | } | |
275 | EXPORT_SYMBOL(_raw_write_trylock); | |
276 | #endif | |
277 | ||
278 | #ifndef CONFIG_INLINE_WRITE_LOCK | |
279 | void __lockfunc _raw_write_lock(rwlock_t *lock) | |
280 | { | |
281 | __raw_write_lock(lock); | |
282 | } | |
283 | EXPORT_SYMBOL(_raw_write_lock); | |
284 | #endif | |
285 | ||
286 | #ifndef CONFIG_INLINE_WRITE_LOCK_IRQSAVE | |
287 | unsigned long __lockfunc _raw_write_lock_irqsave(rwlock_t *lock) | |
288 | { | |
289 | return __raw_write_lock_irqsave(lock); | |
290 | } | |
291 | EXPORT_SYMBOL(_raw_write_lock_irqsave); | |
292 | #endif | |
293 | ||
294 | #ifndef CONFIG_INLINE_WRITE_LOCK_IRQ | |
295 | void __lockfunc _raw_write_lock_irq(rwlock_t *lock) | |
296 | { | |
297 | __raw_write_lock_irq(lock); | |
298 | } | |
299 | EXPORT_SYMBOL(_raw_write_lock_irq); | |
300 | #endif | |
301 | ||
302 | #ifndef CONFIG_INLINE_WRITE_LOCK_BH | |
303 | void __lockfunc _raw_write_lock_bh(rwlock_t *lock) | |
304 | { | |
305 | __raw_write_lock_bh(lock); | |
306 | } | |
307 | EXPORT_SYMBOL(_raw_write_lock_bh); | |
308 | #endif | |
309 | ||
310 | #ifndef CONFIG_INLINE_WRITE_UNLOCK | |
311 | void __lockfunc _raw_write_unlock(rwlock_t *lock) | |
312 | { | |
313 | __raw_write_unlock(lock); | |
314 | } | |
315 | EXPORT_SYMBOL(_raw_write_unlock); | |
316 | #endif | |
317 | ||
318 | #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQRESTORE | |
319 | void __lockfunc _raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) | |
320 | { | |
321 | __raw_write_unlock_irqrestore(lock, flags); | |
322 | } | |
323 | EXPORT_SYMBOL(_raw_write_unlock_irqrestore); | |
324 | #endif | |
325 | ||
326 | #ifndef CONFIG_INLINE_WRITE_UNLOCK_IRQ | |
327 | void __lockfunc _raw_write_unlock_irq(rwlock_t *lock) | |
328 | { | |
329 | __raw_write_unlock_irq(lock); | |
330 | } | |
331 | EXPORT_SYMBOL(_raw_write_unlock_irq); | |
332 | #endif | |
333 | ||
334 | #ifndef CONFIG_INLINE_WRITE_UNLOCK_BH | |
335 | void __lockfunc _raw_write_unlock_bh(rwlock_t *lock) | |
336 | { | |
337 | __raw_write_unlock_bh(lock); | |
338 | } | |
339 | EXPORT_SYMBOL(_raw_write_unlock_bh); | |
340 | #endif | |
341 | ||
342 | #ifdef CONFIG_DEBUG_LOCK_ALLOC | |
343 | ||
344 | void __lockfunc _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass) | |
345 | { | |
346 | preempt_disable(); | |
347 | spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); | |
348 | LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); | |
349 | } | |
350 | EXPORT_SYMBOL(_raw_spin_lock_nested); | |
351 | ||
352 | unsigned long __lockfunc _raw_spin_lock_irqsave_nested(raw_spinlock_t *lock, | |
353 | int subclass) | |
354 | { | |
355 | unsigned long flags; | |
356 | ||
357 | local_irq_save(flags); | |
358 | preempt_disable(); | |
359 | spin_acquire(&lock->dep_map, subclass, 0, _RET_IP_); | |
360 | LOCK_CONTENDED_FLAGS(lock, do_raw_spin_trylock, do_raw_spin_lock, | |
361 | do_raw_spin_lock_flags, &flags); | |
362 | return flags; | |
363 | } | |
364 | EXPORT_SYMBOL(_raw_spin_lock_irqsave_nested); | |
365 | ||
366 | void __lockfunc _raw_spin_lock_nest_lock(raw_spinlock_t *lock, | |
367 | struct lockdep_map *nest_lock) | |
368 | { | |
369 | preempt_disable(); | |
370 | spin_acquire_nest(&lock->dep_map, 0, 0, nest_lock, _RET_IP_); | |
371 | LOCK_CONTENDED(lock, do_raw_spin_trylock, do_raw_spin_lock); | |
372 | } | |
373 | EXPORT_SYMBOL(_raw_spin_lock_nest_lock); | |
374 | ||
375 | #endif | |
376 | ||
377 | notrace int in_lock_functions(unsigned long addr) | |
378 | { | |
379 | /* Linker adds these: start and end of __lockfunc functions */ | |
380 | extern char __lock_text_start[], __lock_text_end[]; | |
381 | ||
382 | return addr >= (unsigned long)__lock_text_start | |
383 | && addr < (unsigned long)__lock_text_end; | |
384 | } | |
385 | EXPORT_SYMBOL(in_lock_functions); |