]>
Commit | Line | Data |
---|---|---|
1 | /* | |
2 | * Generic helpers for smp ipi calls | |
3 | * | |
4 | * (C) Jens Axboe <jens.axboe@oracle.com> 2008 | |
5 | */ | |
6 | #include <linux/rcupdate.h> | |
7 | #include <linux/rculist.h> | |
8 | #include <linux/kernel.h> | |
9 | #include <linux/module.h> | |
10 | #include <linux/percpu.h> | |
11 | #include <linux/init.h> | |
12 | #include <linux/smp.h> | |
13 | #include <linux/cpu.h> | |
14 | ||
15 | static DEFINE_PER_CPU(struct call_single_queue, call_single_queue); | |
16 | ||
17 | static struct { | |
18 | struct list_head queue; | |
19 | raw_spinlock_t lock; | |
20 | } call_function __cacheline_aligned_in_smp = | |
21 | { | |
22 | .queue = LIST_HEAD_INIT(call_function.queue), | |
23 | .lock = __RAW_SPIN_LOCK_UNLOCKED(call_function.lock), | |
24 | }; | |
25 | ||
26 | enum { | |
27 | CSD_FLAG_LOCK = 0x01, | |
28 | }; | |
29 | ||
30 | struct call_function_data { | |
31 | struct call_single_data csd; | |
32 | atomic_t refs; | |
33 | cpumask_var_t cpumask; | |
34 | }; | |
35 | ||
36 | struct call_single_queue { | |
37 | struct list_head list; | |
38 | raw_spinlock_t lock; | |
39 | }; | |
40 | ||
41 | static DEFINE_PER_CPU(struct call_function_data, cfd_data); | |
42 | ||
43 | static int | |
44 | hotplug_cfd(struct notifier_block *nfb, unsigned long action, void *hcpu) | |
45 | { | |
46 | long cpu = (long)hcpu; | |
47 | struct call_function_data *cfd = &per_cpu(cfd_data, cpu); | |
48 | ||
49 | switch (action) { | |
50 | case CPU_UP_PREPARE: | |
51 | case CPU_UP_PREPARE_FROZEN: | |
52 | if (!zalloc_cpumask_var_node(&cfd->cpumask, GFP_KERNEL, | |
53 | cpu_to_node(cpu))) | |
54 | return NOTIFY_BAD; | |
55 | break; | |
56 | ||
57 | #ifdef CONFIG_HOTPLUG_CPU | |
58 | case CPU_UP_CANCELED: | |
59 | case CPU_UP_CANCELED_FROZEN: | |
60 | ||
61 | case CPU_DEAD: | |
62 | case CPU_DEAD_FROZEN: | |
63 | free_cpumask_var(cfd->cpumask); | |
64 | break; | |
65 | #endif | |
66 | }; | |
67 | ||
68 | return NOTIFY_OK; | |
69 | } | |
70 | ||
71 | static struct notifier_block __cpuinitdata hotplug_cfd_notifier = { | |
72 | .notifier_call = hotplug_cfd, | |
73 | }; | |
74 | ||
75 | static int __cpuinit init_call_single_data(void) | |
76 | { | |
77 | void *cpu = (void *)(long)smp_processor_id(); | |
78 | int i; | |
79 | ||
80 | for_each_possible_cpu(i) { | |
81 | struct call_single_queue *q = &per_cpu(call_single_queue, i); | |
82 | ||
83 | raw_spin_lock_init(&q->lock); | |
84 | INIT_LIST_HEAD(&q->list); | |
85 | } | |
86 | ||
87 | hotplug_cfd(&hotplug_cfd_notifier, CPU_UP_PREPARE, cpu); | |
88 | register_cpu_notifier(&hotplug_cfd_notifier); | |
89 | ||
90 | return 0; | |
91 | } | |
92 | early_initcall(init_call_single_data); | |
93 | ||
94 | /* | |
95 | * csd_lock/csd_unlock used to serialize access to per-cpu csd resources | |
96 | * | |
97 | * For non-synchronous ipi calls the csd can still be in use by the | |
98 | * previous function call. For multi-cpu calls its even more interesting | |
99 | * as we'll have to ensure no other cpu is observing our csd. | |
100 | */ | |
101 | static void csd_lock_wait(struct call_single_data *data) | |
102 | { | |
103 | while (data->flags & CSD_FLAG_LOCK) | |
104 | cpu_relax(); | |
105 | } | |
106 | ||
107 | static void csd_lock(struct call_single_data *data) | |
108 | { | |
109 | csd_lock_wait(data); | |
110 | data->flags = CSD_FLAG_LOCK; | |
111 | ||
112 | /* | |
113 | * prevent CPU from reordering the above assignment | |
114 | * to ->flags with any subsequent assignments to other | |
115 | * fields of the specified call_single_data structure: | |
116 | */ | |
117 | smp_mb(); | |
118 | } | |
119 | ||
120 | static void csd_unlock(struct call_single_data *data) | |
121 | { | |
122 | WARN_ON(!(data->flags & CSD_FLAG_LOCK)); | |
123 | ||
124 | /* | |
125 | * ensure we're all done before releasing data: | |
126 | */ | |
127 | smp_mb(); | |
128 | ||
129 | data->flags &= ~CSD_FLAG_LOCK; | |
130 | } | |
131 | ||
132 | /* | |
133 | * Insert a previously allocated call_single_data element | |
134 | * for execution on the given CPU. data must already have | |
135 | * ->func, ->info, and ->flags set. | |
136 | */ | |
137 | static | |
138 | void generic_exec_single(int cpu, struct call_single_data *data, int wait) | |
139 | { | |
140 | struct call_single_queue *dst = &per_cpu(call_single_queue, cpu); | |
141 | unsigned long flags; | |
142 | int ipi; | |
143 | ||
144 | raw_spin_lock_irqsave(&dst->lock, flags); | |
145 | ipi = list_empty(&dst->list); | |
146 | list_add_tail(&data->list, &dst->list); | |
147 | raw_spin_unlock_irqrestore(&dst->lock, flags); | |
148 | ||
149 | /* | |
150 | * The list addition should be visible before sending the IPI | |
151 | * handler locks the list to pull the entry off it because of | |
152 | * normal cache coherency rules implied by spinlocks. | |
153 | * | |
154 | * If IPIs can go out of order to the cache coherency protocol | |
155 | * in an architecture, sufficient synchronisation should be added | |
156 | * to arch code to make it appear to obey cache coherency WRT | |
157 | * locking and barrier primitives. Generic code isn't really | |
158 | * equipped to do the right thing... | |
159 | */ | |
160 | if (ipi) | |
161 | arch_send_call_function_single_ipi(cpu); | |
162 | ||
163 | if (wait) | |
164 | csd_lock_wait(data); | |
165 | } | |
166 | ||
167 | /* | |
168 | * Invoked by arch to handle an IPI for call function. Must be called with | |
169 | * interrupts disabled. | |
170 | */ | |
171 | void generic_smp_call_function_interrupt(void) | |
172 | { | |
173 | struct call_function_data *data; | |
174 | int cpu = smp_processor_id(); | |
175 | ||
176 | /* | |
177 | * Shouldn't receive this interrupt on a cpu that is not yet online. | |
178 | */ | |
179 | WARN_ON_ONCE(!cpu_online(cpu)); | |
180 | ||
181 | /* | |
182 | * Ensure entry is visible on call_function_queue after we have | |
183 | * entered the IPI. See comment in smp_call_function_many. | |
184 | * If we don't have this, then we may miss an entry on the list | |
185 | * and never get another IPI to process it. | |
186 | */ | |
187 | smp_mb(); | |
188 | ||
189 | /* | |
190 | * It's ok to use list_for_each_rcu() here even though we may | |
191 | * delete 'pos', since list_del_rcu() doesn't clear ->next | |
192 | */ | |
193 | list_for_each_entry_rcu(data, &call_function.queue, csd.list) { | |
194 | int refs; | |
195 | ||
196 | if (!cpumask_test_and_clear_cpu(cpu, data->cpumask)) | |
197 | continue; | |
198 | ||
199 | data->csd.func(data->csd.info); | |
200 | ||
201 | refs = atomic_dec_return(&data->refs); | |
202 | WARN_ON(refs < 0); | |
203 | if (!refs) { | |
204 | raw_spin_lock(&call_function.lock); | |
205 | list_del_rcu(&data->csd.list); | |
206 | raw_spin_unlock(&call_function.lock); | |
207 | } | |
208 | ||
209 | if (refs) | |
210 | continue; | |
211 | ||
212 | csd_unlock(&data->csd); | |
213 | } | |
214 | ||
215 | } | |
216 | ||
217 | /* | |
218 | * Invoked by arch to handle an IPI for call function single. Must be | |
219 | * called from the arch with interrupts disabled. | |
220 | */ | |
221 | void generic_smp_call_function_single_interrupt(void) | |
222 | { | |
223 | struct call_single_queue *q = &__get_cpu_var(call_single_queue); | |
224 | unsigned int data_flags; | |
225 | LIST_HEAD(list); | |
226 | ||
227 | /* | |
228 | * Shouldn't receive this interrupt on a cpu that is not yet online. | |
229 | */ | |
230 | WARN_ON_ONCE(!cpu_online(smp_processor_id())); | |
231 | ||
232 | raw_spin_lock(&q->lock); | |
233 | list_replace_init(&q->list, &list); | |
234 | raw_spin_unlock(&q->lock); | |
235 | ||
236 | while (!list_empty(&list)) { | |
237 | struct call_single_data *data; | |
238 | ||
239 | data = list_entry(list.next, struct call_single_data, list); | |
240 | list_del(&data->list); | |
241 | ||
242 | /* | |
243 | * 'data' can be invalid after this call if flags == 0 | |
244 | * (when called through generic_exec_single()), | |
245 | * so save them away before making the call: | |
246 | */ | |
247 | data_flags = data->flags; | |
248 | ||
249 | data->func(data->info); | |
250 | ||
251 | /* | |
252 | * Unlocked CSDs are valid through generic_exec_single(): | |
253 | */ | |
254 | if (data_flags & CSD_FLAG_LOCK) | |
255 | csd_unlock(data); | |
256 | } | |
257 | } | |
258 | ||
259 | static DEFINE_PER_CPU(struct call_single_data, csd_data); | |
260 | ||
261 | /* | |
262 | * smp_call_function_single - Run a function on a specific CPU | |
263 | * @func: The function to run. This must be fast and non-blocking. | |
264 | * @info: An arbitrary pointer to pass to the function. | |
265 | * @wait: If true, wait until function has completed on other CPUs. | |
266 | * | |
267 | * Returns 0 on success, else a negative status code. | |
268 | */ | |
269 | int smp_call_function_single(int cpu, void (*func) (void *info), void *info, | |
270 | int wait) | |
271 | { | |
272 | struct call_single_data d = { | |
273 | .flags = 0, | |
274 | }; | |
275 | unsigned long flags; | |
276 | int this_cpu; | |
277 | int err = 0; | |
278 | ||
279 | /* | |
280 | * prevent preemption and reschedule on another processor, | |
281 | * as well as CPU removal | |
282 | */ | |
283 | this_cpu = get_cpu(); | |
284 | ||
285 | /* | |
286 | * Can deadlock when called with interrupts disabled. | |
287 | * We allow cpu's that are not yet online though, as no one else can | |
288 | * send smp call function interrupt to this cpu and as such deadlocks | |
289 | * can't happen. | |
290 | */ | |
291 | WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled() | |
292 | && !oops_in_progress); | |
293 | ||
294 | if (cpu == this_cpu) { | |
295 | local_irq_save(flags); | |
296 | func(info); | |
297 | local_irq_restore(flags); | |
298 | } else { | |
299 | if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) { | |
300 | struct call_single_data *data = &d; | |
301 | ||
302 | if (!wait) | |
303 | data = &__get_cpu_var(csd_data); | |
304 | ||
305 | csd_lock(data); | |
306 | ||
307 | data->func = func; | |
308 | data->info = info; | |
309 | generic_exec_single(cpu, data, wait); | |
310 | } else { | |
311 | err = -ENXIO; /* CPU not online */ | |
312 | } | |
313 | } | |
314 | ||
315 | put_cpu(); | |
316 | ||
317 | return err; | |
318 | } | |
319 | EXPORT_SYMBOL(smp_call_function_single); | |
320 | ||
321 | /* | |
322 | * smp_call_function_any - Run a function on any of the given cpus | |
323 | * @mask: The mask of cpus it can run on. | |
324 | * @func: The function to run. This must be fast and non-blocking. | |
325 | * @info: An arbitrary pointer to pass to the function. | |
326 | * @wait: If true, wait until function has completed. | |
327 | * | |
328 | * Returns 0 on success, else a negative status code (if no cpus were online). | |
329 | * Note that @wait will be implicitly turned on in case of allocation failures, | |
330 | * since we fall back to on-stack allocation. | |
331 | * | |
332 | * Selection preference: | |
333 | * 1) current cpu if in @mask | |
334 | * 2) any cpu of current node if in @mask | |
335 | * 3) any other online cpu in @mask | |
336 | */ | |
337 | int smp_call_function_any(const struct cpumask *mask, | |
338 | void (*func)(void *info), void *info, int wait) | |
339 | { | |
340 | unsigned int cpu; | |
341 | const struct cpumask *nodemask; | |
342 | int ret; | |
343 | ||
344 | /* Try for same CPU (cheapest) */ | |
345 | cpu = get_cpu(); | |
346 | if (cpumask_test_cpu(cpu, mask)) | |
347 | goto call; | |
348 | ||
349 | /* Try for same node. */ | |
350 | nodemask = cpumask_of_node(cpu); | |
351 | for (cpu = cpumask_first_and(nodemask, mask); cpu < nr_cpu_ids; | |
352 | cpu = cpumask_next_and(cpu, nodemask, mask)) { | |
353 | if (cpu_online(cpu)) | |
354 | goto call; | |
355 | } | |
356 | ||
357 | /* Any online will do: smp_call_function_single handles nr_cpu_ids. */ | |
358 | cpu = cpumask_any_and(mask, cpu_online_mask); | |
359 | call: | |
360 | ret = smp_call_function_single(cpu, func, info, wait); | |
361 | put_cpu(); | |
362 | return ret; | |
363 | } | |
364 | EXPORT_SYMBOL_GPL(smp_call_function_any); | |
365 | ||
366 | /** | |
367 | * __smp_call_function_single(): Run a function on another CPU | |
368 | * @cpu: The CPU to run on. | |
369 | * @data: Pre-allocated and setup data structure | |
370 | * | |
371 | * Like smp_call_function_single(), but allow caller to pass in a | |
372 | * pre-allocated data structure. Useful for embedding @data inside | |
373 | * other structures, for instance. | |
374 | */ | |
375 | void __smp_call_function_single(int cpu, struct call_single_data *data, | |
376 | int wait) | |
377 | { | |
378 | csd_lock(data); | |
379 | ||
380 | /* | |
381 | * Can deadlock when called with interrupts disabled. | |
382 | * We allow cpu's that are not yet online though, as no one else can | |
383 | * send smp call function interrupt to this cpu and as such deadlocks | |
384 | * can't happen. | |
385 | */ | |
386 | WARN_ON_ONCE(cpu_online(smp_processor_id()) && wait && irqs_disabled() | |
387 | && !oops_in_progress); | |
388 | ||
389 | generic_exec_single(cpu, data, wait); | |
390 | } | |
391 | ||
392 | /** | |
393 | * smp_call_function_many(): Run a function on a set of other CPUs. | |
394 | * @mask: The set of cpus to run on (only runs on online subset). | |
395 | * @func: The function to run. This must be fast and non-blocking. | |
396 | * @info: An arbitrary pointer to pass to the function. | |
397 | * @wait: If true, wait (atomically) until function has completed | |
398 | * on other CPUs. | |
399 | * | |
400 | * If @wait is true, then returns once @func has returned. | |
401 | * | |
402 | * You must not call this function with disabled interrupts or from a | |
403 | * hardware interrupt handler or from a bottom half handler. Preemption | |
404 | * must be disabled when calling this function. | |
405 | */ | |
406 | void smp_call_function_many(const struct cpumask *mask, | |
407 | void (*func)(void *), void *info, bool wait) | |
408 | { | |
409 | struct call_function_data *data; | |
410 | unsigned long flags; | |
411 | int cpu, next_cpu, this_cpu = smp_processor_id(); | |
412 | ||
413 | /* | |
414 | * Can deadlock when called with interrupts disabled. | |
415 | * We allow cpu's that are not yet online though, as no one else can | |
416 | * send smp call function interrupt to this cpu and as such deadlocks | |
417 | * can't happen. | |
418 | */ | |
419 | WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled() | |
420 | && !oops_in_progress); | |
421 | ||
422 | /* So, what's a CPU they want? Ignoring this one. */ | |
423 | cpu = cpumask_first_and(mask, cpu_online_mask); | |
424 | if (cpu == this_cpu) | |
425 | cpu = cpumask_next_and(cpu, mask, cpu_online_mask); | |
426 | ||
427 | /* No online cpus? We're done. */ | |
428 | if (cpu >= nr_cpu_ids) | |
429 | return; | |
430 | ||
431 | /* Do we have another CPU which isn't us? */ | |
432 | next_cpu = cpumask_next_and(cpu, mask, cpu_online_mask); | |
433 | if (next_cpu == this_cpu) | |
434 | next_cpu = cpumask_next_and(next_cpu, mask, cpu_online_mask); | |
435 | ||
436 | /* Fastpath: do that cpu by itself. */ | |
437 | if (next_cpu >= nr_cpu_ids) { | |
438 | smp_call_function_single(cpu, func, info, wait); | |
439 | return; | |
440 | } | |
441 | ||
442 | data = &__get_cpu_var(cfd_data); | |
443 | csd_lock(&data->csd); | |
444 | ||
445 | data->csd.func = func; | |
446 | data->csd.info = info; | |
447 | cpumask_and(data->cpumask, mask, cpu_online_mask); | |
448 | cpumask_clear_cpu(this_cpu, data->cpumask); | |
449 | atomic_set(&data->refs, cpumask_weight(data->cpumask)); | |
450 | ||
451 | raw_spin_lock_irqsave(&call_function.lock, flags); | |
452 | /* | |
453 | * Place entry at the _HEAD_ of the list, so that any cpu still | |
454 | * observing the entry in generic_smp_call_function_interrupt() | |
455 | * will not miss any other list entries: | |
456 | */ | |
457 | list_add_rcu(&data->csd.list, &call_function.queue); | |
458 | raw_spin_unlock_irqrestore(&call_function.lock, flags); | |
459 | ||
460 | /* | |
461 | * Make the list addition visible before sending the ipi. | |
462 | * (IPIs must obey or appear to obey normal Linux cache | |
463 | * coherency rules -- see comment in generic_exec_single). | |
464 | */ | |
465 | smp_mb(); | |
466 | ||
467 | /* Send a message to all CPUs in the map */ | |
468 | arch_send_call_function_ipi_mask(data->cpumask); | |
469 | ||
470 | /* Optionally wait for the CPUs to complete */ | |
471 | if (wait) | |
472 | csd_lock_wait(&data->csd); | |
473 | } | |
474 | EXPORT_SYMBOL(smp_call_function_many); | |
475 | ||
476 | /** | |
477 | * smp_call_function(): Run a function on all other CPUs. | |
478 | * @func: The function to run. This must be fast and non-blocking. | |
479 | * @info: An arbitrary pointer to pass to the function. | |
480 | * @wait: If true, wait (atomically) until function has completed | |
481 | * on other CPUs. | |
482 | * | |
483 | * Returns 0. | |
484 | * | |
485 | * If @wait is true, then returns once @func has returned; otherwise | |
486 | * it returns just before the target cpu calls @func. | |
487 | * | |
488 | * You must not call this function with disabled interrupts or from a | |
489 | * hardware interrupt handler or from a bottom half handler. | |
490 | */ | |
491 | int smp_call_function(void (*func)(void *), void *info, int wait) | |
492 | { | |
493 | preempt_disable(); | |
494 | smp_call_function_many(cpu_online_mask, func, info, wait); | |
495 | preempt_enable(); | |
496 | ||
497 | return 0; | |
498 | } | |
499 | EXPORT_SYMBOL(smp_call_function); | |
500 | ||
501 | void ipi_call_lock(void) | |
502 | { | |
503 | raw_spin_lock(&call_function.lock); | |
504 | } | |
505 | ||
506 | void ipi_call_unlock(void) | |
507 | { | |
508 | raw_spin_unlock(&call_function.lock); | |
509 | } | |
510 | ||
511 | void ipi_call_lock_irq(void) | |
512 | { | |
513 | raw_spin_lock_irq(&call_function.lock); | |
514 | } | |
515 | ||
516 | void ipi_call_unlock_irq(void) | |
517 | { | |
518 | raw_spin_unlock_irq(&call_function.lock); | |
519 | } |