]> bbs.cooldavid.org Git - net-next-2.6.git/blame - arch/x86/kernel/io_apic.c
git://bbs.cooldavid.org / net-next-2.6.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
sparseirq: fix intr-remap with dyn_array/nr_irqs changes]
[net-next-2.6.git] / arch / x86 / kernel / io_apic.c
CommitLineData
1da177e4
LT		 1/*
2 * Intel IO-APIC support for multi-Pentium hosts.
3 *
4 * Copyright (C) 1997, 1998, 1999, 2000 Ingo Molnar, Hajnalka Szabo
5 *
6 * Many thanks to Stig Venaas for trying out countless experimental
7 * patches and reporting/debugging problems patiently!
8 *
9 * (c) 1999, Multiple IO-APIC support, developed by
10 * Ken-ichi Yaku <yaku@css1.kbnes.nec.co.jp> and
11 * Hidemi Kishimoto <kisimoto@css1.kbnes.nec.co.jp>,
12 * further tested and cleaned up by Zach Brown <zab@redhat.com>
13 * and Ingo Molnar <mingo@redhat.com>
14 *
15 * Fixes
16 * Maciej W. Rozycki : Bits for genuine 82489DX APICs;
17 * thanks to Eric Gilmore
18 * and Rolf G. Tews
19 * for testing these extensively
20 * Paul Diefenbaugh : Added full ACPI support
21 */
22
23#include <linux/mm.h>
1da177e4
LT		 24#include <linux/interrupt.h>
25#include <linux/init.h>
26#include <linux/delay.h>
27#include <linux/sched.h>
d4057bdb 28#include <linux/pci.h>
1da177e4
LT		 29#include <linux/mc146818rtc.h>
30#include <linux/compiler.h>
31#include <linux/acpi.h>
129f6946 32#include <linux/module.h>
1da177e4 33#include <linux/sysdev.h>
3b7d1921 34#include <linux/msi.h>
95d77884 35#include <linux/htirq.h>
7dfb7103 36#include <linux/freezer.h>
f26d6a2b 37#include <linux/kthread.h>
54168ed7 38#include <linux/jiffies.h> /* time_after() */
d4057bdb
YL		 39#ifdef CONFIG_ACPI
40#include <acpi/acpi_bus.h>
41#endif
42#include <linux/bootmem.h>
43#include <linux/dmar.h>
54d5d424 44
d4057bdb 45#include <asm/idle.h>
1da177e4
LT		 46#include <asm/io.h>
47#include <asm/smp.h>
48#include <asm/desc.h>
d4057bdb
YL		 49#include <asm/proto.h>
50#include <asm/acpi.h>
51#include <asm/dma.h>
1da177e4 52#include <asm/timer.h>
306e440d 53#include <asm/i8259.h>
3e4ff115 54#include <asm/nmi.h>
2d3fcc1c 55#include <asm/msidef.h>
8b955b0d 56#include <asm/hypertransport.h>
a4dbc34d 57#include <asm/setup.h>
d4057bdb 58#include <asm/irq_remapping.h>
1da177e4 59
497c9a19 60#include <mach_ipi.h>
1da177e4 61#include <mach_apic.h>
874c4fe3 62#include <mach_apicdef.h>
1da177e4 63
32f71aff
MR		 64#define __apicdebuginit(type) static type __init
65
1da177e4 66/*
54168ed7
IM		 67 * Is the SiS APIC rmw bug present ?
68 * -1 = don't know, 0 = no, 1 = yes
1da177e4
LT		 69 */
70int sis_apic_bug = -1;
71
efa2559f
YL		 72static DEFINE_SPINLOCK(ioapic_lock);
73static DEFINE_SPINLOCK(vector_lock);
74
1da177e4
LT		 75/*
76 * # of IRQ routing registers
77 */
78int nr_ioapic_registers[MAX_IO_APICS];
79
9f640ccb 80/* I/O APIC entries */
ec2cd0a2 81struct mp_config_ioapic mp_ioapics[MAX_IO_APICS];
9f640ccb
AS		 82int nr_ioapics;
83
584f734d 84/* MP IRQ source entries */
2fddb6e2 85struct mp_config_intsrc mp_irqs[MAX_IRQ_SOURCES];
584f734d
AS		 86
87/* # of MP IRQ source entries */
88int mp_irq_entries;
89
8732fc4b
AS		 90#if defined (CONFIG_MCA) || defined (CONFIG_EISA)
91int mp_bus_id_to_type[MAX_MP_BUSSES];
92#endif
93
94DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);
95
efa2559f
YL		 96int skip_ioapic_setup;
97
54168ed7 98static int __init parse_noapic(char *str)
efa2559f
YL		 99{
100 /* disable IO-APIC */
101 disable_ioapic_setup();
102 return 0;
103}
104early_param("noapic", parse_noapic);
66759a01 105
da51a821 106struct irq_cfg;
0f978f45 107struct irq_pin_list;
a1420f39 108struct irq_cfg {
da51a821 109 unsigned int irq;
8f09cd20 110#ifdef CONFIG_HAVE_SPARSE_IRQ
da51a821 111 struct irq_cfg *next;
8f09cd20 112#endif
0f978f45 113 struct irq_pin_list *irq_2_pin;
497c9a19
YL		 114 cpumask_t domain;
115 cpumask_t old_domain;
116 unsigned move_cleanup_count;
a1420f39 117 u8 vector;
497c9a19 118 u8 move_in_progress : 1;
a1420f39
YL		 119};
120
a1420f39
YL		 121/* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */
122static struct irq_cfg irq_cfg_legacy[] __initdata = {
497c9a19
YL		 123 [0] = { .irq = 0, .domain = CPU_MASK_ALL, .vector = IRQ0_VECTOR, },
124 [1] = { .irq = 1, .domain = CPU_MASK_ALL, .vector = IRQ1_VECTOR, },
125 [2] = { .irq = 2, .domain = CPU_MASK_ALL, .vector = IRQ2_VECTOR, },
126 [3] = { .irq = 3, .domain = CPU_MASK_ALL, .vector = IRQ3_VECTOR, },
127 [4] = { .irq = 4, .domain = CPU_MASK_ALL, .vector = IRQ4_VECTOR, },
128 [5] = { .irq = 5, .domain = CPU_MASK_ALL, .vector = IRQ5_VECTOR, },
129 [6] = { .irq = 6, .domain = CPU_MASK_ALL, .vector = IRQ6_VECTOR, },
130 [7] = { .irq = 7, .domain = CPU_MASK_ALL, .vector = IRQ7_VECTOR, },
131 [8] = { .irq = 8, .domain = CPU_MASK_ALL, .vector = IRQ8_VECTOR, },
132 [9] = { .irq = 9, .domain = CPU_MASK_ALL, .vector = IRQ9_VECTOR, },
133 [10] = { .irq = 10, .domain = CPU_MASK_ALL, .vector = IRQ10_VECTOR, },
134 [11] = { .irq = 11, .domain = CPU_MASK_ALL, .vector = IRQ11_VECTOR, },
135 [12] = { .irq = 12, .domain = CPU_MASK_ALL, .vector = IRQ12_VECTOR, },
136 [13] = { .irq = 13, .domain = CPU_MASK_ALL, .vector = IRQ13_VECTOR, },
137 [14] = { .irq = 14, .domain = CPU_MASK_ALL, .vector = IRQ14_VECTOR, },
138 [15] = { .irq = 15, .domain = CPU_MASK_ALL, .vector = IRQ15_VECTOR, },
a1420f39
YL		 139};
140
da51a821 141static struct irq_cfg irq_cfg_init = { .irq = -1U, };
da51a821
YL		 142
143static void init_one_irq_cfg(struct irq_cfg *cfg)
144{
145 memcpy(cfg, &irq_cfg_init, sizeof(struct irq_cfg));
146}
147
148static struct irq_cfg *irq_cfgx;
8f09cd20 149#ifdef CONFIG_HAVE_SPARSE_IRQ
da51a821 150static struct irq_cfg *irq_cfgx_free;
8f09cd20 151#endif
a1420f39
YL		 152static void __init init_work(void *data)
153{
da51a821
YL		 154 struct dyn_array *da = data;
155 struct irq_cfg *cfg;
156 int legacy_count;
157 int i;
158
159 cfg = *da->name;
a1420f39 160
da51a821 161 memcpy(cfg, irq_cfg_legacy, sizeof(irq_cfg_legacy));
a1420f39 162
da51a821
YL		 163 legacy_count = sizeof(irq_cfg_legacy)/sizeof(irq_cfg_legacy[0]);
164 for (i = legacy_count; i < *da->nr; i++)
165 init_one_irq_cfg(&cfg[i]);
a1420f39 166
8f09cd20 167#ifdef CONFIG_HAVE_SPARSE_IRQ
da51a821
YL		 168 for (i = 1; i < *da->nr; i++)
169 cfg[i-1].next = &cfg[i];
a1420f39 170
da51a821
YL		 171 irq_cfgx_free = &irq_cfgx[legacy_count];
172 irq_cfgx[legacy_count - 1].next = NULL;
8f09cd20
YL		 173#endif
174}
175
176#ifdef CONFIG_HAVE_SPARSE_IRQ
177/* need to be biger than size of irq_cfg_legacy */
178static int nr_irq_cfg = 32;
179
180static int __init parse_nr_irq_cfg(char *arg)
181{
182 if (arg) {
183 nr_irq_cfg = simple_strtoul(arg, NULL, 0);
184 if (nr_irq_cfg < 32)
185 nr_irq_cfg = 32;
186 }
187 return 0;
a1420f39
YL		 188}
189
8f09cd20
YL		 190early_param("nr_irq_cfg", parse_nr_irq_cfg);
191
192#define for_each_irq_cfg(irqX, cfg) \
193 for (cfg = irq_cfgx, irqX = cfg->irq; cfg; cfg = cfg->next, irqX = cfg ? cfg->irq : -1U)
194
da51a821
YL		 195
196DEFINE_DYN_ARRAY(irq_cfgx, sizeof(struct irq_cfg), nr_irq_cfg, PAGE_SIZE, init_work);
a1420f39
YL		 197
198static struct irq_cfg *irq_cfg(unsigned int irq)
199{
da51a821
YL		 200 struct irq_cfg *cfg;
201
202 cfg = irq_cfgx;
203 while (cfg) {
204 if (cfg->irq == irq)
205 return cfg;
206
207 cfg = cfg->next;
208 }
209
210 return NULL;
211}
212
213static struct irq_cfg *irq_cfg_alloc(unsigned int irq)
214{
215 struct irq_cfg *cfg, *cfg_pri;
216 int i;
217 int count = 0;
218
219 cfg_pri = cfg = irq_cfgx;
220 while (cfg) {
221 if (cfg->irq == irq)
222 return cfg;
223
224 cfg_pri = cfg;
225 cfg = cfg->next;
226 count++;
227 }
228
229 if (!irq_cfgx_free) {
230 unsigned long phys;
231 unsigned long total_bytes;
232 /*
233 * we run out of pre-allocate ones, allocate more
234 */
235 printk(KERN_DEBUG "try to get more irq_cfg %d\n", nr_irq_cfg);
236
237 total_bytes = sizeof(struct irq_cfg) * nr_irq_cfg;
238 if (after_bootmem)
239 cfg = kzalloc(total_bytes, GFP_ATOMIC);
240 else
241 cfg = __alloc_bootmem_nopanic(total_bytes, PAGE_SIZE, 0);
a1420f39 242
da51a821
YL		 243 if (!cfg)
244 panic("please boot with nr_irq_cfg= %d\n", count * 2);
245
246 phys = __pa(cfg);
247 printk(KERN_DEBUG "irq_irq ==> [%#lx - %#lx]\n", phys, phys + total_bytes);
248
249 for (i = 0; i < nr_irq_cfg; i++)
250 init_one_irq_cfg(&cfg[i]);
251
252 for (i = 1; i < nr_irq_cfg; i++)
253 cfg[i-1].next = &cfg[i];
254
255 irq_cfgx_free = cfg;
256 }
257
258 cfg = irq_cfgx_free;
259 irq_cfgx_free = irq_cfgx_free->next;
260 cfg->next = NULL;
261 if (cfg_pri)
262 cfg_pri->next = cfg;
263 else
264 irq_cfgx = cfg;
265 cfg->irq = irq;
266 printk(KERN_DEBUG "found new irq_cfg for irq %d\n", cfg->irq);
da51a821
YL		 267#ifdef CONFIG_HAVE_SPARSE_IRQ_DEBUG
268 {
269 /* dump the results */
270 struct irq_cfg *cfg;
271 unsigned long phys;
272 unsigned long bytes = sizeof(struct irq_cfg);
273
274 printk(KERN_DEBUG "=========================== %d\n", irq);
275 printk(KERN_DEBUG "irq_cfg dump after get that for %d\n", irq);
276 for_each_irq_cfg(cfg) {
277 phys = __pa(cfg);
278 printk(KERN_DEBUG "irq_cfg %d ==> [%#lx - %#lx]\n", cfg->irq, phys, phys + bytes);
279 }
280 printk(KERN_DEBUG "===========================\n");
281 }
282#endif
283 return cfg;
a1420f39 284}
8f09cd20
YL		 285#else
286
287#define for_each_irq_cfg(irq, cfg) \
288 for (irq = 0, cfg = &irq_cfgx[irq]; irq < nr_irqs; irq++, cfg = &irq_cfgx[irq])
289
290DEFINE_DYN_ARRAY(irq_cfgx, sizeof(struct irq_cfg), nr_irqs, PAGE_SIZE, init_work);
a1420f39 291
8f09cd20
YL		 292struct irq_cfg *irq_cfg(unsigned int irq)
293{
294 if (irq < nr_irqs)
295 return &irq_cfgx[irq];
296
297 return NULL;
298}
299struct irq_cfg *irq_cfg_alloc(unsigned int irq)
300{
301 return irq_cfg(irq);
302}
303
304#endif
1da177e4
LT		 305/*
306 * This is performance-critical, we want to do it O(1)
307 *
308 * the indexing order of this array favors 1:1 mappings
309 * between pins and IRQs.
310 */
311
0f978f45
YL		 312struct irq_pin_list {
313 int apic, pin;
314 struct irq_pin_list *next;
315};
316
317static struct irq_pin_list *irq_2_pin_head;
318/* fill one page ? */
319static int nr_irq_2_pin = 0x100;
320static struct irq_pin_list *irq_2_pin_ptr;
321static void __init irq_2_pin_init_work(void *data)
322{
323 struct dyn_array *da = data;
324 struct irq_pin_list *pin;
325 int i;
326
327 pin = *da->name;
328
329 for (i = 1; i < *da->nr; i++)
330 pin[i-1].next = &pin[i];
331
332 irq_2_pin_ptr = &pin[0];
333}
334DEFINE_DYN_ARRAY(irq_2_pin_head, sizeof(struct irq_pin_list), nr_irq_2_pin, PAGE_SIZE, irq_2_pin_init_work);
335
336static struct irq_pin_list *get_one_free_irq_2_pin(void)
337{
338 struct irq_pin_list *pin;
339 int i;
340
341 pin = irq_2_pin_ptr;
342
343 if (pin) {
344 irq_2_pin_ptr = pin->next;
345 pin->next = NULL;
346 return pin;
347 }
348
349 /*
350 * we run out of pre-allocate ones, allocate more
351 */
352 printk(KERN_DEBUG "try to get more irq_2_pin %d\n", nr_irq_2_pin);
353
354 if (after_bootmem)
355 pin = kzalloc(sizeof(struct irq_pin_list)*nr_irq_2_pin,
356 GFP_ATOMIC);
357 else
358 pin = __alloc_bootmem_nopanic(sizeof(struct irq_pin_list) *
359 nr_irq_2_pin, PAGE_SIZE, 0);
360
361 if (!pin)
362 panic("can not get more irq_2_pin\n");
301e6190 363
0f978f45
YL		 364 for (i = 1; i < nr_irq_2_pin; i++)
365 pin[i-1].next = &pin[i];
366
367 irq_2_pin_ptr = pin->next;
368 pin->next = NULL;
369
370 return pin;
371}
1da177e4 372
130fe05d
LT		 373struct io_apic {
374 unsigned int index;
375 unsigned int unused[3];
376 unsigned int data;
377};
378
379static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
380{
381 return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
ec2cd0a2 382 + (mp_ioapics[idx].mp_apicaddr & ~PAGE_MASK);
130fe05d
LT		 383}
384
385static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg)
386{
387 struct io_apic __iomem *io_apic = io_apic_base(apic);
388 writel(reg, &io_apic->index);
389 return readl(&io_apic->data);
390}
391
392static inline void io_apic_write(unsigned int apic, unsigned int reg, unsigned int value)
393{
394 struct io_apic __iomem *io_apic = io_apic_base(apic);
395 writel(reg, &io_apic->index);
396 writel(value, &io_apic->data);
397}
398
399/*
400 * Re-write a value: to be used for read-modify-write
401 * cycles where the read already set up the index register.
402 *
403 * Older SiS APIC requires we rewrite the index register
404 */
405static inline void io_apic_modify(unsigned int apic, unsigned int reg, unsigned int value)
406{
54168ed7
IM		 407 struct io_apic __iomem *io_apic = io_apic_base(apic);
408 if (sis_apic_bug)
409 writel(reg, &io_apic->index);
130fe05d
LT		 410 writel(value, &io_apic->data);
411}
412
047c8fdb
YL		 413static bool io_apic_level_ack_pending(unsigned int irq)
414{
415 struct irq_pin_list *entry;
416 unsigned long flags;
417 struct irq_cfg *cfg = irq_cfg(irq);
418
419 spin_lock_irqsave(&ioapic_lock, flags);
420 entry = cfg->irq_2_pin;
421 for (;;) {
422 unsigned int reg;
423 int pin;
424
425 if (!entry)
426 break;
427 pin = entry->pin;
428 reg = io_apic_read(entry->apic, 0x10 + pin*2);
429 /* Is the remote IRR bit set? */
430 if (reg & IO_APIC_REDIR_REMOTE_IRR) {
431 spin_unlock_irqrestore(&ioapic_lock, flags);
432 return true;
433 }
434 if (!entry->next)
435 break;
436 entry = entry->next;
437 }
438 spin_unlock_irqrestore(&ioapic_lock, flags);
439
440 return false;
441}
047c8fdb 442
cf4c6a2f
AK		 443union entry_union {
444 struct { u32 w1, w2; };
445 struct IO_APIC_route_entry entry;
446};
447
448static struct IO_APIC_route_entry ioapic_read_entry(int apic, int pin)
449{
450 union entry_union eu;
451 unsigned long flags;
452 spin_lock_irqsave(&ioapic_lock, flags);
453 eu.w1 = io_apic_read(apic, 0x10 + 2 * pin);
454 eu.w2 = io_apic_read(apic, 0x11 + 2 * pin);
455 spin_unlock_irqrestore(&ioapic_lock, flags);
456 return eu.entry;
457}
458
f9dadfa7
LT		 459/*
460 * When we write a new IO APIC routing entry, we need to write the high
461 * word first! If the mask bit in the low word is clear, we will enable
462 * the interrupt, and we need to make sure the entry is fully populated
463 * before that happens.
464 */
d15512f4
AK		 465static void
466__ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
cf4c6a2f 467{
cf4c6a2f
AK		 468 union entry_union eu;
469 eu.entry = e;
f9dadfa7
LT		 470 io_apic_write(apic, 0x11 + 2*pin, eu.w2);
471 io_apic_write(apic, 0x10 + 2*pin, eu.w1);
d15512f4
AK		 472}
473
474static void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
475{
476 unsigned long flags;
477 spin_lock_irqsave(&ioapic_lock, flags);
478 __ioapic_write_entry(apic, pin, e);
f9dadfa7
LT		 479 spin_unlock_irqrestore(&ioapic_lock, flags);
480}
481
482/*
483 * When we mask an IO APIC routing entry, we need to write the low
484 * word first, in order to set the mask bit before we change the
485 * high bits!
486 */
487static void ioapic_mask_entry(int apic, int pin)
488{
489 unsigned long flags;
490 union entry_union eu = { .entry.mask = 1 };
491
cf4c6a2f
AK		 492 spin_lock_irqsave(&ioapic_lock, flags);
493 io_apic_write(apic, 0x10 + 2*pin, eu.w1);
494 io_apic_write(apic, 0x11 + 2*pin, eu.w2);
495 spin_unlock_irqrestore(&ioapic_lock, flags);
496}
497
497c9a19
YL		 498#ifdef CONFIG_SMP
499static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, u8 vector)
500{
501 int apic, pin;
502 struct irq_cfg *cfg;
503 struct irq_pin_list *entry;
504
505 cfg = irq_cfg(irq);
506 entry = cfg->irq_2_pin;
507 for (;;) {
508 unsigned int reg;
509
510 if (!entry)
511 break;
512
513 apic = entry->apic;
514 pin = entry->pin;
54168ed7
IM		 515#ifdef CONFIG_INTR_REMAP
516 /*
517 * With interrupt-remapping, destination information comes
518 * from interrupt-remapping table entry.
519 */
520 if (!irq_remapped(irq))
521 io_apic_write(apic, 0x11 + pin*2, dest);
522#else
497c9a19 523 io_apic_write(apic, 0x11 + pin*2, dest);
54168ed7 524#endif
497c9a19
YL		 525 reg = io_apic_read(apic, 0x10 + pin*2);
526 reg &= ~IO_APIC_REDIR_VECTOR_MASK;
527 reg |= vector;
54168ed7 528 io_apic_modify(apic, 0x10 + pin*2, reg);
497c9a19
YL		 529 if (!entry->next)
530 break;
531 entry = entry->next;
532 }
533}
efa2559f
YL		 534
535static int assign_irq_vector(int irq, cpumask_t mask);
536
497c9a19
YL		 537static void set_ioapic_affinity_irq(unsigned int irq, cpumask_t mask)
538{
539 struct irq_cfg *cfg;
540 unsigned long flags;
541 unsigned int dest;
542 cpumask_t tmp;
54168ed7 543 struct irq_desc *desc;
497c9a19 544
497c9a19
YL		 545 cpus_and(tmp, mask, cpu_online_map);
546 if (cpus_empty(tmp))
547 return;
548
047c8fdb 549 cfg = irq_cfg(irq);
497c9a19
YL		 550 if (assign_irq_vector(irq, mask))
551 return;
552
553 cpus_and(tmp, cfg->domain, mask);
497c9a19
YL		 554 dest = cpu_mask_to_apicid(tmp);
555 /*
556 * Only the high 8 bits are valid.
557 */
558 dest = SET_APIC_LOGICAL_ID(dest);
559
54168ed7 560 desc = irq_to_desc(irq);
497c9a19
YL		 561 spin_lock_irqsave(&ioapic_lock, flags);
562 __target_IO_APIC_irq(irq, dest, cfg->vector);
54168ed7 563 desc->affinity = mask;
497c9a19
YL		 564 spin_unlock_irqrestore(&ioapic_lock, flags);
565}
497c9a19
YL		 566#endif /* CONFIG_SMP */
567
1da177e4
LT		 568/*
569 * The common case is 1:1 IRQ<->pin mappings. Sometimes there are
570 * shared ISA-space IRQs, so we have to support them. We are super
571 * fast in the common case, and fast for shared ISA-space IRQs.
572 */
573static void add_pin_to_irq(unsigned int irq, int apic, int pin)
574{
0f978f45
YL		 575 struct irq_cfg *cfg;
576 struct irq_pin_list *entry;
577
578 /* first time to refer irq_cfg, so with new */
579 cfg = irq_cfg_alloc(irq);
580 entry = cfg->irq_2_pin;
581 if (!entry) {
582 entry = get_one_free_irq_2_pin();
583 cfg->irq_2_pin = entry;
584 entry->apic = apic;
585 entry->pin = pin;
586 printk(KERN_DEBUG " 0 add_pin_to_irq: irq %d --> apic %d pin %d\n", irq, apic, pin);
587 return;
588 }
1da177e4 589
0f978f45
YL		 590 while (entry->next) {
591 /* not again, please */
592 if (entry->apic == apic && entry->pin == pin)
593 return;
1da177e4 594
0f978f45 595 entry = entry->next;
1da177e4 596 }
0f978f45
YL		 597
598 entry->next = get_one_free_irq_2_pin();
599 entry = entry->next;
1da177e4
LT		 600 entry->apic = apic;
601 entry->pin = pin;
0f978f45 602 printk(KERN_DEBUG " x add_pin_to_irq: irq %d --> apic %d pin %d\n", irq, apic, pin);
1da177e4
LT		 603}
604
605/*
606 * Reroute an IRQ to a different pin.
607 */
608static void __init replace_pin_at_irq(unsigned int irq,
609 int oldapic, int oldpin,
610 int newapic, int newpin)
611{
0f978f45
YL		 612 struct irq_cfg *cfg = irq_cfg(irq);
613 struct irq_pin_list *entry = cfg->irq_2_pin;
614 int replaced = 0;
1da177e4 615
0f978f45 616 while (entry) {
1da177e4
LT		 617 if (entry->apic == oldapic && entry->pin == oldpin) {
618 entry->apic = newapic;
619 entry->pin = newpin;
0f978f45
YL		 620 replaced = 1;
621 /* every one is different, right? */
1da177e4 622 break;
0f978f45
YL		 623 }
624 entry = entry->next;
1da177e4 625 }
0f978f45
YL		 626
627 /* why? call replace before add? */
628 if (!replaced)
629 add_pin_to_irq(irq, newapic, newpin);
1da177e4
LT		 630}
631
4e738e2f 632#define __DO_ACTION(R, ACTION_ENABLE, ACTION_DISABLE, FINAL) \
047c8fdb
YL		 633 \
634{ \
635 int pin; \
636 struct irq_cfg *cfg; \
637 struct irq_pin_list *entry; \
638 \
639 cfg = irq_cfg(irq); \
640 entry = cfg->irq_2_pin; \
641 for (;;) { \
642 unsigned int reg; \
643 if (!entry) \
644 break; \
645 pin = entry->pin; \
646 reg = io_apic_read(entry->apic, 0x10 + R + pin*2); \
4e738e2f
YL		 647 reg ACTION_DISABLE; \
648 reg ACTION_ENABLE; \
047c8fdb
YL		 649 io_apic_modify(entry->apic, 0x10 + R + pin*2, reg); \
650 FINAL; \
651 if (!entry->next) \
652 break; \
653 entry = entry->next; \
654 } \
655}
656
4e738e2f 657#define DO_ACTION(name,R, ACTION_ENABLE, ACTION_DISABLE, FINAL) \
047c8fdb
YL		 658 \
659 static void name##_IO_APIC_irq (unsigned int irq) \
4e738e2f 660 __DO_ACTION(R, ACTION_ENABLE, ACTION_DISABLE, FINAL)
047c8fdb
YL		 661
662/* mask = 0 */
4e738e2f 663DO_ACTION(__unmask, 0, |= 0, &= ~IO_APIC_REDIR_MASKED, )
047c8fdb 664
4e738e2f
YL		 665#ifdef CONFIG_X86_64
666/*
667 * Synchronize the IO-APIC and the CPU by doing
668 * a dummy read from the IO-APIC
669 */
670static inline void io_apic_sync(unsigned int apic)
1da177e4 671{
4e738e2f
YL		 672 struct io_apic __iomem *io_apic = io_apic_base(apic);
673 readl(&io_apic->data);
1da177e4
LT		 674}
675
676/* mask = 1 */
4e738e2f 677DO_ACTION(__mask, 0, |= IO_APIC_REDIR_MASKED, &= ~0, io_apic_sync(entry->apic))
1da177e4 678
4e738e2f
YL		 679#else
680
681/* mask = 1 */
682DO_ACTION(__mask, 0, |= IO_APIC_REDIR_MASKED, &= ~0, )
1da177e4
LT		 683
684/* mask = 1, trigger = 0 */
4e738e2f 685DO_ACTION(__mask_and_edge, 0, |= IO_APIC_REDIR_MASKED, &= ~IO_APIC_REDIR_LEVEL_TRIGGER, )
1da177e4
LT		 686
687/* mask = 0, trigger = 1 */
4e738e2f 688DO_ACTION(__unmask_and_level, 0, |= IO_APIC_REDIR_LEVEL_TRIGGER, &= ~IO_APIC_REDIR_MASKED, )
1da177e4 689
047c8fdb
YL		 690#endif
691
54168ed7 692static void mask_IO_APIC_irq (unsigned int irq)
1da177e4
LT		 693{
694 unsigned long flags;
695
696 spin_lock_irqsave(&ioapic_lock, flags);
697 __mask_IO_APIC_irq(irq);
698 spin_unlock_irqrestore(&ioapic_lock, flags);
699}
700
54168ed7 701static void unmask_IO_APIC_irq (unsigned int irq)
1da177e4
LT		 702{
703 unsigned long flags;
704
705 spin_lock_irqsave(&ioapic_lock, flags);
706 __unmask_IO_APIC_irq(irq);
707 spin_unlock_irqrestore(&ioapic_lock, flags);
708}
709
710static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
711{
712 struct IO_APIC_route_entry entry;
36062448 713
1da177e4 714 /* Check delivery_mode to be sure we're not clearing an SMI pin */
cf4c6a2f 715 entry = ioapic_read_entry(apic, pin);
1da177e4
LT		 716 if (entry.delivery_mode == dest_SMI)
717 return;
1da177e4
LT		 718 /*
719 * Disable it in the IO-APIC irq-routing table:
720 */
f9dadfa7 721 ioapic_mask_entry(apic, pin);
1da177e4
LT		 722}
723
54168ed7 724static void clear_IO_APIC (void)
1da177e4
LT		 725{
726 int apic, pin;
727
728 for (apic = 0; apic < nr_ioapics; apic++)
729 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
730 clear_IO_APIC_pin(apic, pin);
731}
732
54168ed7 733#if !defined(CONFIG_SMP) && defined(CONFIG_X86_32)
75604d7f 734void send_IPI_self(int vector)
1da177e4
LT		 735{
736 unsigned int cfg;
737
738 /*
739 * Wait for idle.
740 */
741 apic_wait_icr_idle();
742 cfg = APIC_DM_FIXED | APIC_DEST_SELF | vector | APIC_DEST_LOGICAL;
743 /*
744 * Send the IPI. The write to APIC_ICR fires this off.
745 */
593f4a78 746 apic_write(APIC_ICR, cfg);
1da177e4 747}
54168ed7 748#endif /* !CONFIG_SMP && CONFIG_X86_32*/
1da177e4 749
54168ed7 750#ifdef CONFIG_X86_32
1da177e4
LT		 751/*
752 * support for broken MP BIOSs, enables hand-redirection of PIRQ0-7 to
753 * specific CPU-side IRQs.
754 */
755
756#define MAX_PIRQS 8
757static int pirq_entries [MAX_PIRQS];
758static int pirqs_enabled;
1da177e4 759
1da177e4
LT		 760static int __init ioapic_pirq_setup(char *str)
761{
762 int i, max;
763 int ints[MAX_PIRQS+1];
764
765 get_options(str, ARRAY_SIZE(ints), ints);
766
767 for (i = 0; i < MAX_PIRQS; i++)
768 pirq_entries[i] = -1;
769
770 pirqs_enabled = 1;
771 apic_printk(APIC_VERBOSE, KERN_INFO
772 "PIRQ redirection, working around broken MP-BIOS.\n");
773 max = MAX_PIRQS;
774 if (ints[0] < MAX_PIRQS)
775 max = ints[0];
776
777 for (i = 0; i < max; i++) {
778 apic_printk(APIC_VERBOSE, KERN_DEBUG
779 "... PIRQ%d -> IRQ %d\n", i, ints[i+1]);
780 /*
781 * PIRQs are mapped upside down, usually.
782 */
783 pirq_entries[MAX_PIRQS-i-1] = ints[i+1];
784 }
785 return 1;
786}
787
788__setup("pirq=", ioapic_pirq_setup);
54168ed7
IM		 789#endif /* CONFIG_X86_32 */
790
791#ifdef CONFIG_INTR_REMAP
792/* I/O APIC RTE contents at the OS boot up */
793static struct IO_APIC_route_entry *early_ioapic_entries[MAX_IO_APICS];
794
795/*
796 * Saves and masks all the unmasked IO-APIC RTE's
797 */
798int save_mask_IO_APIC_setup(void)
799{
800 union IO_APIC_reg_01 reg_01;
801 unsigned long flags;
802 int apic, pin;
803
804 /*
805 * The number of IO-APIC IRQ registers (== #pins):
806 */
807 for (apic = 0; apic < nr_ioapics; apic++) {
808 spin_lock_irqsave(&ioapic_lock, flags);
809 reg_01.raw = io_apic_read(apic, 1);
810 spin_unlock_irqrestore(&ioapic_lock, flags);
811 nr_ioapic_registers[apic] = reg_01.bits.entries+1;
812 }
813
814 for (apic = 0; apic < nr_ioapics; apic++) {
815 early_ioapic_entries[apic] =
816 kzalloc(sizeof(struct IO_APIC_route_entry) *
817 nr_ioapic_registers[apic], GFP_KERNEL);
818 if (!early_ioapic_entries[apic])
819 return -ENOMEM;
820 }
821
822 for (apic = 0; apic < nr_ioapics; apic++)
823 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
824 struct IO_APIC_route_entry entry;
825
826 entry = early_ioapic_entries[apic][pin] =
827 ioapic_read_entry(apic, pin);
828 if (!entry.mask) {
829 entry.mask = 1;
830 ioapic_write_entry(apic, pin, entry);
831 }
832 }
833 return 0;
834}
835
836void restore_IO_APIC_setup(void)
837{
838 int apic, pin;
839
840 for (apic = 0; apic < nr_ioapics; apic++)
841 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
842 ioapic_write_entry(apic, pin,
843 early_ioapic_entries[apic][pin]);
844}
845
846void reinit_intr_remapped_IO_APIC(int intr_remapping)
847{
848 /*
849 * for now plain restore of previous settings.
850 * TBD: In the case of OS enabling interrupt-remapping,
851 * IO-APIC RTE's need to be setup to point to interrupt-remapping
852 * table entries. for now, do a plain restore, and wait for
853 * the setup_IO_APIC_irqs() to do proper initialization.
854 */
855 restore_IO_APIC_setup();
856}
857#endif
1da177e4
LT		 858
859/*
860 * Find the IRQ entry number of a certain pin.
861 */
862static int find_irq_entry(int apic, int pin, int type)
863{
864 int i;
865
866 for (i = 0; i < mp_irq_entries; i++)
2fddb6e2
AS		 867 if (mp_irqs[i].mp_irqtype == type &&
868 (mp_irqs[i].mp_dstapic == mp_ioapics[apic].mp_apicid ||
869 mp_irqs[i].mp_dstapic == MP_APIC_ALL) &&
870 mp_irqs[i].mp_dstirq == pin)
1da177e4
LT		 871 return i;
872
873 return -1;
874}
875
876/*
877 * Find the pin to which IRQ[irq] (ISA) is connected
878 */
fcfd636a 879static int __init find_isa_irq_pin(int irq, int type)
1da177e4
LT		 880{
881 int i;
882
883 for (i = 0; i < mp_irq_entries; i++) {
2fddb6e2 884 int lbus = mp_irqs[i].mp_srcbus;
1da177e4 885
d27e2b8e 886 if (test_bit(lbus, mp_bus_not_pci) &&
2fddb6e2
AS		 887 (mp_irqs[i].mp_irqtype == type) &&
888 (mp_irqs[i].mp_srcbusirq == irq))
1da177e4 889
2fddb6e2 890 return mp_irqs[i].mp_dstirq;
1da177e4
LT		 891 }
892 return -1;
893}
894
fcfd636a
EB		 895static int __init find_isa_irq_apic(int irq, int type)
896{
897 int i;
898
899 for (i = 0; i < mp_irq_entries; i++) {
2fddb6e2 900 int lbus = mp_irqs[i].mp_srcbus;
fcfd636a 901
73b2961b 902 if (test_bit(lbus, mp_bus_not_pci) &&
2fddb6e2
AS		 903 (mp_irqs[i].mp_irqtype == type) &&
904 (mp_irqs[i].mp_srcbusirq == irq))
fcfd636a
EB		 905 break;
906 }
907 if (i < mp_irq_entries) {
908 int apic;
54168ed7 909 for(apic = 0; apic < nr_ioapics; apic++) {
2fddb6e2 910 if (mp_ioapics[apic].mp_apicid == mp_irqs[i].mp_dstapic)
fcfd636a
EB		 911 return apic;
912 }
913 }
914
915 return -1;
916}
917
1da177e4
LT		 918/*
919 * Find a specific PCI IRQ entry.
920 * Not an __init, possibly needed by modules
921 */
922static int pin_2_irq(int idx, int apic, int pin);
923
924int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin)
925{
926 int apic, i, best_guess = -1;
927
54168ed7
IM		 928 apic_printk(APIC_DEBUG, "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
929 bus, slot, pin);
ce6444d3 930 if (test_bit(bus, mp_bus_not_pci)) {
54168ed7 931 apic_printk(APIC_VERBOSE, "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
1da177e4
LT		 932 return -1;
933 }
934 for (i = 0; i < mp_irq_entries; i++) {
2fddb6e2 935 int lbus = mp_irqs[i].mp_srcbus;
1da177e4
LT		 936
937 for (apic = 0; apic < nr_ioapics; apic++)
2fddb6e2
AS		 938 if (mp_ioapics[apic].mp_apicid == mp_irqs[i].mp_dstapic ||
939 mp_irqs[i].mp_dstapic == MP_APIC_ALL)
1da177e4
LT		 940 break;
941
47cab822 942 if (!test_bit(lbus, mp_bus_not_pci) &&
2fddb6e2 943 !mp_irqs[i].mp_irqtype &&
1da177e4 944 (bus == lbus) &&
2fddb6e2 945 (slot == ((mp_irqs[i].mp_srcbusirq >> 2) & 0x1f))) {
54168ed7 946 int irq = pin_2_irq(i,apic,mp_irqs[i].mp_dstirq);
1da177e4
LT		 947
948 if (!(apic || IO_APIC_IRQ(irq)))
949 continue;
950
2fddb6e2 951 if (pin == (mp_irqs[i].mp_srcbusirq & 3))
1da177e4
LT		 952 return irq;
953 /*
954 * Use the first all-but-pin matching entry as a
955 * best-guess fuzzy result for broken mptables.
956 */
957 if (best_guess < 0)
958 best_guess = irq;
959 }
960 }
961 return best_guess;
962}
54168ed7 963
129f6946 964EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);
1da177e4 965
c0a282c2 966#if defined(CONFIG_EISA) || defined(CONFIG_MCA)
1da177e4
LT		 967/*
968 * EISA Edge/Level control register, ELCR
969 */
970static int EISA_ELCR(unsigned int irq)
971{
972 if (irq < 16) {
973 unsigned int port = 0x4d0 + (irq >> 3);
974 return (inb(port) >> (irq & 7)) & 1;
975 }
976 apic_printk(APIC_VERBOSE, KERN_INFO
977 "Broken MPtable reports ISA irq %d\n", irq);
978 return 0;
979}
54168ed7 980
c0a282c2 981#endif
1da177e4 982
6728801d
AS		 983/* ISA interrupts are always polarity zero edge triggered,
984 * when listed as conforming in the MP table. */
985
986#define default_ISA_trigger(idx) (0)
987#define default_ISA_polarity(idx) (0)
988
1da177e4
LT		 989/* EISA interrupts are always polarity zero and can be edge or level
990 * trigger depending on the ELCR value. If an interrupt is listed as
991 * EISA conforming in the MP table, that means its trigger type must
992 * be read in from the ELCR */
993
2fddb6e2 994#define default_EISA_trigger(idx) (EISA_ELCR(mp_irqs[idx].mp_srcbusirq))
6728801d 995#define default_EISA_polarity(idx) default_ISA_polarity(idx)
1da177e4
LT		 996
997/* PCI interrupts are always polarity one level triggered,
998 * when listed as conforming in the MP table. */
999
1000#define default_PCI_trigger(idx) (1)
1001#define default_PCI_polarity(idx) (1)
1002
1003/* MCA interrupts are always polarity zero level triggered,
1004 * when listed as conforming in the MP table. */
1005
1006#define default_MCA_trigger(idx) (1)
6728801d 1007#define default_MCA_polarity(idx) default_ISA_polarity(idx)
1da177e4 1008
61fd47e0 1009static int MPBIOS_polarity(int idx)
1da177e4 1010{
2fddb6e2 1011 int bus = mp_irqs[idx].mp_srcbus;
1da177e4
LT		 1012 int polarity;
1013
1014 /*
1015 * Determine IRQ line polarity (high active or low active):
1016 */
54168ed7 1017 switch (mp_irqs[idx].mp_irqflag & 3)
36062448 1018 {
54168ed7
IM		 1019 case 0: /* conforms, ie. bus-type dependent polarity */
1020 if (test_bit(bus, mp_bus_not_pci))
1021 polarity = default_ISA_polarity(idx);
1022 else
1023 polarity = default_PCI_polarity(idx);
1024 break;
1025 case 1: /* high active */
1026 {
1027 polarity = 0;
1028 break;
1029 }
1030 case 2: /* reserved */
1031 {
1032 printk(KERN_WARNING "broken BIOS!!\n");
1033 polarity = 1;
1034 break;
1035 }
1036 case 3: /* low active */
1037 {
1038 polarity = 1;
1039 break;
1040 }
1041 default: /* invalid */
1042 {
1043 printk(KERN_WARNING "broken BIOS!!\n");
1044 polarity = 1;
1045 break;
1046 }
1da177e4
LT		 1047 }
1048 return polarity;
1049}
1050
1051static int MPBIOS_trigger(int idx)
1052{
2fddb6e2 1053 int bus = mp_irqs[idx].mp_srcbus;
1da177e4
LT		 1054 int trigger;
1055
1056 /*
1057 * Determine IRQ trigger mode (edge or level sensitive):
1058 */
54168ed7 1059 switch ((mp_irqs[idx].mp_irqflag>>2) & 3)
1da177e4 1060 {
54168ed7
IM		 1061 case 0: /* conforms, ie. bus-type dependent */
1062 if (test_bit(bus, mp_bus_not_pci))
1063 trigger = default_ISA_trigger(idx);
1064 else
1065 trigger = default_PCI_trigger(idx);
c0a282c2 1066#if defined(CONFIG_EISA) || defined(CONFIG_MCA)
54168ed7
IM		 1067 switch (mp_bus_id_to_type[bus]) {
1068 case MP_BUS_ISA: /* ISA pin */
1069 {
1070 /* set before the switch */
1071 break;
1072 }
1073 case MP_BUS_EISA: /* EISA pin */
1074 {
1075 trigger = default_EISA_trigger(idx);
1076 break;
1077 }
1078 case MP_BUS_PCI: /* PCI pin */
1079 {
1080 /* set before the switch */
1081 break;
1082 }
1083 case MP_BUS_MCA: /* MCA pin */
1084 {
1085 trigger = default_MCA_trigger(idx);
1086 break;
1087 }
1088 default:
1089 {
1090 printk(KERN_WARNING "broken BIOS!!\n");
1091 trigger = 1;
1092 break;
1093 }
1094 }
1095#endif
1da177e4 1096 break;
54168ed7 1097 case 1: /* edge */
1da177e4 1098 {
54168ed7 1099 trigger = 0;
1da177e4
LT		 1100 break;
1101 }
54168ed7 1102 case 2: /* reserved */
1da177e4 1103 {
54168ed7
IM		 1104 printk(KERN_WARNING "broken BIOS!!\n");
1105 trigger = 1;
1da177e4
LT		 1106 break;
1107 }
54168ed7 1108 case 3: /* level */
1da177e4 1109 {
54168ed7 1110 trigger = 1;
1da177e4
LT		 1111 break;
1112 }
54168ed7 1113 default: /* invalid */
1da177e4
LT		 1114 {
1115 printk(KERN_WARNING "broken BIOS!!\n");
54168ed7 1116 trigger = 0;
1da177e4
LT		 1117 break;
1118 }
1119 }
1120 return trigger;
1121}
1122
1123static inline int irq_polarity(int idx)
1124{
1125 return MPBIOS_polarity(idx);
1126}
1127
1128static inline int irq_trigger(int idx)
1129{
1130 return MPBIOS_trigger(idx);
1131}
1132
efa2559f 1133int (*ioapic_renumber_irq)(int ioapic, int irq);
1da177e4
LT		 1134static int pin_2_irq(int idx, int apic, int pin)
1135{
1136 int irq, i;
2fddb6e2 1137 int bus = mp_irqs[idx].mp_srcbus;
1da177e4
LT		 1138
1139 /*
1140 * Debugging check, we are in big trouble if this message pops up!
1141 */
2fddb6e2 1142 if (mp_irqs[idx].mp_dstirq != pin)
1da177e4
LT		 1143 printk(KERN_ERR "broken BIOS or MPTABLE parser, ayiee!!\n");
1144
54168ed7 1145 if (test_bit(bus, mp_bus_not_pci)) {
2fddb6e2 1146 irq = mp_irqs[idx].mp_srcbusirq;
54168ed7 1147 } else {
643befed
AS		 1148 /*
1149 * PCI IRQs are mapped in order
1150 */
1151 i = irq = 0;
1152 while (i < apic)
1153 irq += nr_ioapic_registers[i++];
1154 irq += pin;
54168ed7
IM		 1155 /*
1156 * For MPS mode, so far only needed by ES7000 platform
1157 */
1158 if (ioapic_renumber_irq)
1159 irq = ioapic_renumber_irq(apic, irq);
1da177e4
LT		 1160 }
1161
54168ed7 1162#ifdef CONFIG_X86_32
1da177e4
LT		 1163 /*
1164 * PCI IRQ command line redirection. Yes, limits are hardcoded.
1165 */
1166 if ((pin >= 16) && (pin <= 23)) {
1167 if (pirq_entries[pin-16] != -1) {
1168 if (!pirq_entries[pin-16]) {
1169 apic_printk(APIC_VERBOSE, KERN_DEBUG
1170 "disabling PIRQ%d\n", pin-16);
1171 } else {
1172 irq = pirq_entries[pin-16];
1173 apic_printk(APIC_VERBOSE, KERN_DEBUG
1174 "using PIRQ%d -> IRQ %d\n",
1175 pin-16, irq);
1176 }
1177 }
1178 }
54168ed7
IM		 1179#endif
1180
1da177e4
LT		 1181 return irq;
1182}
1183
497c9a19
YL		 1184void lock_vector_lock(void)
1185{
1186 /* Used to the online set of cpus does not change
1187 * during assign_irq_vector.
1188 */
1189 spin_lock(&vector_lock);
1190}
1da177e4 1191
497c9a19 1192void unlock_vector_lock(void)
1da177e4 1193{
497c9a19
YL		 1194 spin_unlock(&vector_lock);
1195}
1da177e4 1196
497c9a19
YL		 1197static int __assign_irq_vector(int irq, cpumask_t mask)
1198{
047c8fdb
YL		 1199 /*
1200 * NOTE! The local APIC isn't very good at handling
1201 * multiple interrupts at the same interrupt level.
1202 * As the interrupt level is determined by taking the
1203 * vector number and shifting that right by 4, we
1204 * want to spread these out a bit so that they don't
1205 * all fall in the same interrupt level.
1206 *
1207 * Also, we've got to be careful not to trash gate
1208 * 0x80, because int 0x80 is hm, kind of importantish. ;)
1209 */
54168ed7
IM		 1210 static int current_vector = FIRST_DEVICE_VECTOR, current_offset = 0;
1211 unsigned int old_vector;
1212 int cpu;
1213 struct irq_cfg *cfg;
ace80ab7 1214
54168ed7 1215 cfg = irq_cfg(irq);
8339f000 1216
54168ed7
IM		 1217 /* Only try and allocate irqs on cpus that are present */
1218 cpus_and(mask, mask, cpu_online_map);
ace80ab7 1219
54168ed7
IM		 1220 if ((cfg->move_in_progress) || cfg->move_cleanup_count)
1221 return -EBUSY;
0a1ad60d 1222
54168ed7
IM		 1223 old_vector = cfg->vector;
1224 if (old_vector) {
1225 cpumask_t tmp;
1226 cpus_and(tmp, cfg->domain, mask);
1227 if (!cpus_empty(tmp))
1228 return 0;
1229 }
497c9a19 1230
54168ed7
IM		 1231 for_each_cpu_mask_nr(cpu, mask) {
1232 cpumask_t domain, new_mask;
1233 int new_cpu;
1234 int vector, offset;
497c9a19 1235
54168ed7
IM		 1236 domain = vector_allocation_domain(cpu);
1237 cpus_and(new_mask, domain, cpu_online_map);
497c9a19 1238
54168ed7
IM		 1239 vector = current_vector;
1240 offset = current_offset;
497c9a19 1241next:
54168ed7
IM		 1242 vector += 8;
1243 if (vector >= first_system_vector) {
1244 /* If we run out of vectors on large boxen, must share them. */
1245 offset = (offset + 1) % 8;
1246 vector = FIRST_DEVICE_VECTOR + offset;
1247 }
1248 if (unlikely(current_vector == vector))
1249 continue;
047c8fdb 1250#ifdef CONFIG_X86_64
54168ed7
IM		 1251 if (vector == IA32_SYSCALL_VECTOR)
1252 goto next;
047c8fdb 1253#else
54168ed7
IM		 1254 if (vector == SYSCALL_VECTOR)
1255 goto next;
047c8fdb 1256#endif
54168ed7
IM		 1257 for_each_cpu_mask_nr(new_cpu, new_mask)
1258 if (per_cpu(vector_irq, new_cpu)[vector] != -1)
1259 goto next;
1260 /* Found one! */
1261 current_vector = vector;
1262 current_offset = offset;
1263 if (old_vector) {
1264 cfg->move_in_progress = 1;
1265 cfg->old_domain = cfg->domain;
7a959cff 1266 }
54168ed7
IM		 1267 for_each_cpu_mask_nr(new_cpu, new_mask)
1268 per_cpu(vector_irq, new_cpu)[vector] = irq;
1269 cfg->vector = vector;
1270 cfg->domain = domain;
1271 return 0;
1272 }
1273 return -ENOSPC;
497c9a19
YL		 1274}
1275
1276static int assign_irq_vector(int irq, cpumask_t mask)
1277{
1278 int err;
ace80ab7 1279 unsigned long flags;
ace80ab7
EB		 1280
1281 spin_lock_irqsave(&vector_lock, flags);
497c9a19 1282 err = __assign_irq_vector(irq, mask);
26a3c49c 1283 spin_unlock_irqrestore(&vector_lock, flags);
497c9a19
YL		 1284 return err;
1285}
1286
1287static void __clear_irq_vector(int irq)
1288{
1289 struct irq_cfg *cfg;
1290 cpumask_t mask;
1291 int cpu, vector;
1292
1293 cfg = irq_cfg(irq);
1294 BUG_ON(!cfg->vector);
1295
1296 vector = cfg->vector;
1297 cpus_and(mask, cfg->domain, cpu_online_map);
1298 for_each_cpu_mask_nr(cpu, mask)
1299 per_cpu(vector_irq, cpu)[vector] = -1;
1300
1301 cfg->vector = 0;
1302 cpus_clear(cfg->domain);
1303}
1304
1305void __setup_vector_irq(int cpu)
1306{
1307 /* Initialize vector_irq on a new cpu */
1308 /* This function must be called with vector_lock held */
1309 int irq, vector;
1310 struct irq_cfg *cfg;
1311
1312 /* Mark the inuse vectors */
8f09cd20 1313 for_each_irq_cfg(irq, cfg) {
497c9a19
YL		 1314 if (!cpu_isset(cpu, cfg->domain))
1315 continue;
1316 vector = cfg->vector;
497c9a19
YL		 1317 per_cpu(vector_irq, cpu)[vector] = irq;
1318 }
1319 /* Mark the free vectors */
1320 for (vector = 0; vector < NR_VECTORS; ++vector) {
1321 irq = per_cpu(vector_irq, cpu)[vector];
1322 if (irq < 0)
1323 continue;
1324
1325 cfg = irq_cfg(irq);
1326 if (!cpu_isset(cpu, cfg->domain))
1327 per_cpu(vector_irq, cpu)[vector] = -1;
54168ed7 1328 }
1da177e4 1329}
3fde6900 1330
f5b9ed7a 1331static struct irq_chip ioapic_chip;
54168ed7
IM		 1332#ifdef CONFIG_INTR_REMAP
1333static struct irq_chip ir_ioapic_chip;
1334#endif
1da177e4 1335
54168ed7
IM		 1336#define IOAPIC_AUTO -1
1337#define IOAPIC_EDGE 0
1338#define IOAPIC_LEVEL 1
1da177e4 1339
047c8fdb 1340#ifdef CONFIG_X86_32
1d025192
YL		 1341static inline int IO_APIC_irq_trigger(int irq)
1342{
54168ed7 1343 int apic, idx, pin;
1d025192 1344
54168ed7
IM		 1345 for (apic = 0; apic < nr_ioapics; apic++) {
1346 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
1347 idx = find_irq_entry(apic, pin, mp_INT);
1348 if ((idx != -1) && (irq == pin_2_irq(idx, apic, pin)))
1349 return irq_trigger(idx);
1350 }
1351 }
1352 /*
1353 * nonexistent IRQs are edge default
1354 */
1355 return 0;
1d025192 1356}
047c8fdb
YL		 1357#else
1358static inline int IO_APIC_irq_trigger(int irq)
1359{
54168ed7 1360 return 1;
047c8fdb
YL		 1361}
1362#endif
1d025192 1363
497c9a19 1364static void ioapic_register_intr(int irq, unsigned long trigger)
1da177e4 1365{
08678b08
YL		 1366 struct irq_desc *desc;
1367
199751d7
YL		 1368 /* first time to use this irq_desc */
1369 if (irq < 16)
1370 desc = irq_to_desc(irq);
1371 else
1372 desc = irq_to_desc_alloc(irq);
1373
6ebcc00e 1374 if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
047c8fdb 1375 trigger == IOAPIC_LEVEL)
08678b08 1376 desc->status |= IRQ_LEVEL;
047c8fdb
YL		 1377 else
1378 desc->status &= ~IRQ_LEVEL;
1379
54168ed7
IM		 1380#ifdef CONFIG_INTR_REMAP
1381 if (irq_remapped(irq)) {
1382 desc->status |= IRQ_MOVE_PCNTXT;
1383 if (trigger)
1384 set_irq_chip_and_handler_name(irq, &ir_ioapic_chip,
1385 handle_fasteoi_irq,
1386 "fasteoi");
1387 else
1388 set_irq_chip_and_handler_name(irq, &ir_ioapic_chip,
1389 handle_edge_irq, "edge");
1390 return;
1391 }
1392#endif
047c8fdb
YL		 1393 if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
1394 trigger == IOAPIC_LEVEL)
a460e745 1395 set_irq_chip_and_handler_name(irq, &ioapic_chip,
54168ed7
IM		 1396 handle_fasteoi_irq,
1397 "fasteoi");
047c8fdb 1398 else
a460e745 1399 set_irq_chip_and_handler_name(irq, &ioapic_chip,
54168ed7 1400 handle_edge_irq, "edge");
1da177e4
LT		 1401}
1402
497c9a19
YL		 1403static int setup_ioapic_entry(int apic, int irq,
1404 struct IO_APIC_route_entry *entry,
1405 unsigned int destination, int trigger,
1406 int polarity, int vector)
1da177e4 1407{
497c9a19
YL		 1408 /*
1409 * add it to the IO-APIC irq-routing table:
1410 */
1411 memset(entry,0,sizeof(*entry));
1412
54168ed7
IM		 1413#ifdef CONFIG_INTR_REMAP
1414 if (intr_remapping_enabled) {
1415 struct intel_iommu *iommu = map_ioapic_to_ir(apic);
1416 struct irte irte;
1417 struct IR_IO_APIC_route_entry *ir_entry =
1418 (struct IR_IO_APIC_route_entry *) entry;
1419 int index;
1420
1421 if (!iommu)
1422 panic("No mapping iommu for ioapic %d\n", apic);
1423
1424 index = alloc_irte(iommu, irq, 1);
1425 if (index < 0)
1426 panic("Failed to allocate IRTE for ioapic %d\n", apic);
1427
1428 memset(&irte, 0, sizeof(irte));
1429
1430 irte.present = 1;
1431 irte.dst_mode = INT_DEST_MODE;
1432 irte.trigger_mode = trigger;
1433 irte.dlvry_mode = INT_DELIVERY_MODE;
1434 irte.vector = vector;
1435 irte.dest_id = IRTE_DEST(destination);
1436
1437 modify_irte(irq, &irte);
1438
1439 ir_entry->index2 = (index >> 15) & 0x1;
1440 ir_entry->zero = 0;
1441 ir_entry->format = 1;
1442 ir_entry->index = (index & 0x7fff);
1443 } else
1444#endif
1445 {
1446 entry->delivery_mode = INT_DELIVERY_MODE;
1447 entry->dest_mode = INT_DEST_MODE;
1448 entry->dest = destination;
1449 }
497c9a19 1450
54168ed7 1451 entry->mask = 0; /* enable IRQ */
497c9a19
YL		 1452 entry->trigger = trigger;
1453 entry->polarity = polarity;
1454 entry->vector = vector;
1455
1456 /* Mask level triggered irqs.
1457 * Use IRQ_DELAYED_DISABLE for edge triggered irqs.
1458 */
1459 if (trigger)
1460 entry->mask = 1;
497c9a19
YL		 1461 return 0;
1462}
1463
1464static void setup_IO_APIC_irq(int apic, int pin, unsigned int irq,
54168ed7 1465 int trigger, int polarity)
497c9a19
YL		 1466{
1467 struct irq_cfg *cfg;
1da177e4 1468 struct IO_APIC_route_entry entry;
497c9a19
YL		 1469 cpumask_t mask;
1470
1471 if (!IO_APIC_IRQ(irq))
1472 return;
1473
1474 cfg = irq_cfg(irq);
1475
1476 mask = TARGET_CPUS;
1477 if (assign_irq_vector(irq, mask))
1478 return;
1479
1480 cpus_and(mask, cfg->domain, mask);
1481
1482 apic_printk(APIC_VERBOSE,KERN_DEBUG
1483 "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> "
1484 "IRQ %d Mode:%i Active:%i)\n",
1485 apic, mp_ioapics[apic].mp_apicid, pin, cfg->vector,
1486 irq, trigger, polarity);
1487
1488
1489 if (setup_ioapic_entry(mp_ioapics[apic].mp_apicid, irq, &entry,
1490 cpu_mask_to_apicid(mask), trigger, polarity,
1491 cfg->vector)) {
1492 printk("Failed to setup ioapic entry for ioapic %d, pin %d\n",
1493 mp_ioapics[apic].mp_apicid, pin);
1494 __clear_irq_vector(irq);
1495 return;
1496 }
1497
1498 ioapic_register_intr(irq, trigger);
1499 if (irq < 16)
1500 disable_8259A_irq(irq);
1501
1502 ioapic_write_entry(apic, pin, entry);
1503}
1504
1505static void __init setup_IO_APIC_irqs(void)
1506{
1507 int apic, pin, idx, irq, first_notcon = 1;
1da177e4
LT		 1508
1509 apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");
1510
1511 for (apic = 0; apic < nr_ioapics; apic++) {
1512 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
1513
497c9a19 1514 idx = find_irq_entry(apic,pin,mp_INT);
1da177e4
LT		 1515 if (idx == -1) {
1516 if (first_notcon) {
497c9a19 1517 apic_printk(APIC_VERBOSE, KERN_DEBUG " IO-APIC (apicid-pin) %d-%d", mp_ioapics[apic].mp_apicid, pin);
1da177e4
LT		 1518 first_notcon = 0;
1519 } else
497c9a19 1520 apic_printk(APIC_VERBOSE, ", %d-%d", mp_ioapics[apic].mp_apicid, pin);
1da177e4
LT		 1521 continue;
1522 }
20d225b9
YL		 1523 if (!first_notcon) {
1524 apic_printk(APIC_VERBOSE, " not connected.\n");
1525 first_notcon = 1;
1526 }
1527
1da177e4 1528 irq = pin_2_irq(idx, apic, pin);
54168ed7 1529#ifdef CONFIG_X86_32
497c9a19
YL		 1530 if (multi_timer_check(apic, irq))
1531 continue;
54168ed7 1532#endif
497c9a19 1533 add_pin_to_irq(irq, apic, pin);
36062448 1534
497c9a19
YL		 1535 setup_IO_APIC_irq(apic, pin, irq,
1536 irq_trigger(idx), irq_polarity(idx));
1da177e4
LT		 1537 }
1538 }
1539
1540 if (!first_notcon)
1541 apic_printk(APIC_VERBOSE, " not connected.\n");
1542}
1543
1544/*
f7633ce5 1545 * Set up the timer pin, possibly with the 8259A-master behind.
1da177e4 1546 */
f7633ce5
MR		 1547static void __init setup_timer_IRQ0_pin(unsigned int apic, unsigned int pin,
1548 int vector)
1da177e4
LT		 1549{
1550 struct IO_APIC_route_entry entry;
1da177e4 1551
54168ed7
IM		 1552#ifdef CONFIG_INTR_REMAP
1553 if (intr_remapping_enabled)
1554 return;
1555#endif
1556
36062448 1557 memset(&entry, 0, sizeof(entry));
1da177e4
LT		 1558
1559 /*
1560 * We use logical delivery to get the timer IRQ
1561 * to the first CPU.
1562 */
1563 entry.dest_mode = INT_DEST_MODE;
03be7505 1564 entry.mask = 1; /* mask IRQ now */
d83e94ac 1565 entry.dest = cpu_mask_to_apicid(TARGET_CPUS);
1da177e4
LT		 1566 entry.delivery_mode = INT_DELIVERY_MODE;
1567 entry.polarity = 0;
1568 entry.trigger = 0;
1569 entry.vector = vector;
1570
1571 /*
1572 * The timer IRQ doesn't have to know that behind the
f7633ce5 1573 * scene we may have a 8259A-master in AEOI mode ...
1da177e4 1574 */
54168ed7 1575 set_irq_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq, "edge");
1da177e4
LT		 1576
1577 /*
1578 * Add it to the IO-APIC irq-routing table:
1579 */
cf4c6a2f 1580 ioapic_write_entry(apic, pin, entry);
1da177e4
LT		 1581}
1582
32f71aff
MR		 1583
1584__apicdebuginit(void) print_IO_APIC(void)
1da177e4
LT		 1585{
1586 int apic, i;
1587 union IO_APIC_reg_00 reg_00;
1588 union IO_APIC_reg_01 reg_01;
1589 union IO_APIC_reg_02 reg_02;
1590 union IO_APIC_reg_03 reg_03;
1591 unsigned long flags;
0f978f45 1592 struct irq_cfg *cfg;
8f09cd20 1593 unsigned int irq;
1da177e4
LT		 1594
1595 if (apic_verbosity == APIC_QUIET)
1596 return;
1597
36062448 1598 printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
1da177e4
LT		 1599 for (i = 0; i < nr_ioapics; i++)
1600 printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
ec2cd0a2 1601 mp_ioapics[i].mp_apicid, nr_ioapic_registers[i]);
1da177e4
LT		 1602
1603 /*
1604 * We are a bit conservative about what we expect. We have to
1605 * know about every hardware change ASAP.
1606 */
1607 printk(KERN_INFO "testing the IO APIC.......................\n");
1608
1609 for (apic = 0; apic < nr_ioapics; apic++) {
1610
1611 spin_lock_irqsave(&ioapic_lock, flags);
1612 reg_00.raw = io_apic_read(apic, 0);
1613 reg_01.raw = io_apic_read(apic, 1);
1614 if (reg_01.bits.version >= 0x10)
1615 reg_02.raw = io_apic_read(apic, 2);
54168ed7
IM		 1616 if (reg_01.bits.version >= 0x20)
1617 reg_03.raw = io_apic_read(apic, 3);
1da177e4
LT		 1618 spin_unlock_irqrestore(&ioapic_lock, flags);
1619
54168ed7 1620 printk("\n");
ec2cd0a2 1621 printk(KERN_DEBUG "IO APIC #%d......\n", mp_ioapics[apic].mp_apicid);
1da177e4
LT		 1622 printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw);
1623 printk(KERN_DEBUG "....... : physical APIC id: %02X\n", reg_00.bits.ID);
1624 printk(KERN_DEBUG "....... : Delivery Type: %X\n", reg_00.bits.delivery_type);
1625 printk(KERN_DEBUG "....... : LTS : %X\n", reg_00.bits.LTS);
1da177e4 1626
54168ed7 1627 printk(KERN_DEBUG ".... register #01: %08X\n", *(int *)&reg_01);
1da177e4 1628 printk(KERN_DEBUG "....... : max redirection entries: %04X\n", reg_01.bits.entries);
1da177e4
LT		 1629
1630 printk(KERN_DEBUG "....... : PRQ implemented: %X\n", reg_01.bits.PRQ);
1631 printk(KERN_DEBUG "....... : IO APIC version: %04X\n", reg_01.bits.version);
1da177e4
LT		 1632
1633 /*
1634 * Some Intel chipsets with IO APIC VERSION of 0x1? don't have reg_02,
1635 * but the value of reg_02 is read as the previous read register
1636 * value, so ignore it if reg_02 == reg_01.
1637 */
1638 if (reg_01.bits.version >= 0x10 && reg_02.raw != reg_01.raw) {
1639 printk(KERN_DEBUG ".... register #02: %08X\n", reg_02.raw);
1640 printk(KERN_DEBUG "....... : arbitration: %02X\n", reg_02.bits.arbitration);
1da177e4
LT		 1641 }
1642
1643 /*
1644 * Some Intel chipsets with IO APIC VERSION of 0x2? don't have reg_02
1645 * or reg_03, but the value of reg_0[23] is read as the previous read
1646 * register value, so ignore it if reg_03 == reg_0[12].
1647 */
1648 if (reg_01.bits.version >= 0x20 && reg_03.raw != reg_02.raw &&
1649 reg_03.raw != reg_01.raw) {
1650 printk(KERN_DEBUG ".... register #03: %08X\n", reg_03.raw);
1651 printk(KERN_DEBUG "....... : Boot DT : %X\n", reg_03.bits.boot_DT);
1da177e4
LT		 1652 }
1653
1654 printk(KERN_DEBUG ".... IRQ redirection table:\n");
1655
d83e94ac
YL		 1656 printk(KERN_DEBUG " NR Dst Mask Trig IRR Pol"
1657 " Stat Dmod Deli Vect: \n");
1da177e4
LT		 1658
1659 for (i = 0; i <= reg_01.bits.entries; i++) {
1660 struct IO_APIC_route_entry entry;
1661
cf4c6a2f 1662 entry = ioapic_read_entry(apic, i);
1da177e4 1663
54168ed7
IM		 1664 printk(KERN_DEBUG " %02x %03X ",
1665 i,
1666 entry.dest
1667 );
1da177e4
LT		 1668
1669 printk("%1d %1d %1d %1d %1d %1d %1d %02X\n",
1670 entry.mask,
1671 entry.trigger,
1672 entry.irr,
1673 entry.polarity,
1674 entry.delivery_status,
1675 entry.dest_mode,
1676 entry.delivery_mode,
1677 entry.vector
1678 );
1679 }
1680 }
1da177e4 1681 printk(KERN_DEBUG "IRQ to pin mappings:\n");
8f09cd20 1682 for_each_irq_cfg(irq, cfg) {
0f978f45
YL		 1683 struct irq_pin_list *entry = cfg->irq_2_pin;
1684 if (!entry)
1da177e4 1685 continue;
8f09cd20 1686 printk(KERN_DEBUG "IRQ%d ", irq);
1da177e4
LT		 1687 for (;;) {
1688 printk("-> %d:%d", entry->apic, entry->pin);
1689 if (!entry->next)
1690 break;
0f978f45 1691 entry = entry->next;
1da177e4
LT		 1692 }
1693 printk("\n");
1694 }
1695
1696 printk(KERN_INFO ".................................... done.\n");
1697
1698 return;
1699}
1700
32f71aff 1701__apicdebuginit(void) print_APIC_bitfield(int base)
1da177e4
LT		 1702{
1703 unsigned int v;
1704 int i, j;
1705
1706 if (apic_verbosity == APIC_QUIET)
1707 return;
1708
1709 printk(KERN_DEBUG "0123456789abcdef0123456789abcdef\n" KERN_DEBUG);
1710 for (i = 0; i < 8; i++) {
1711 v = apic_read(base + i*0x10);
1712 for (j = 0; j < 32; j++) {
1713 if (v & (1<<j))
1714 printk("1");
1715 else
1716 printk("0");
1717 }
1718 printk("\n");
1719 }
1720}
1721
32f71aff 1722__apicdebuginit(void) print_local_APIC(void *dummy)
1da177e4
LT		 1723{
1724 unsigned int v, ver, maxlvt;
7ab6af7a 1725 u64 icr;
1da177e4
LT		 1726
1727 if (apic_verbosity == APIC_QUIET)
1728 return;
1729
1730 printk("\n" KERN_DEBUG "printing local APIC contents on CPU#%d/%d:\n",
1731 smp_processor_id(), hard_smp_processor_id());
66823114 1732 v = apic_read(APIC_ID);
54168ed7 1733 printk(KERN_INFO "... APIC ID: %08x (%01x)\n", v, read_apic_id());
1da177e4
LT		 1734 v = apic_read(APIC_LVR);
1735 printk(KERN_INFO "... APIC VERSION: %08x\n", v);
1736 ver = GET_APIC_VERSION(v);
e05d723f 1737 maxlvt = lapic_get_maxlvt();
1da177e4
LT		 1738
1739 v = apic_read(APIC_TASKPRI);
1740 printk(KERN_DEBUG "... APIC TASKPRI: %08x (%02x)\n", v, v & APIC_TPRI_MASK);
1741
54168ed7 1742 if (APIC_INTEGRATED(ver)) { /* !82489DX */
1da177e4
LT		 1743 v = apic_read(APIC_ARBPRI);
1744 printk(KERN_DEBUG "... APIC ARBPRI: %08x (%02x)\n", v,
1745 v & APIC_ARBPRI_MASK);
1746 v = apic_read(APIC_PROCPRI);
1747 printk(KERN_DEBUG "... APIC PROCPRI: %08x\n", v);
1748 }
1749
1750 v = apic_read(APIC_EOI);
1751 printk(KERN_DEBUG "... APIC EOI: %08x\n", v);
1752 v = apic_read(APIC_RRR);
1753 printk(KERN_DEBUG "... APIC RRR: %08x\n", v);
1754 v = apic_read(APIC_LDR);
1755 printk(KERN_DEBUG "... APIC LDR: %08x\n", v);
1756 v = apic_read(APIC_DFR);
1757 printk(KERN_DEBUG "... APIC DFR: %08x\n", v);
1758 v = apic_read(APIC_SPIV);
1759 printk(KERN_DEBUG "... APIC SPIV: %08x\n", v);
1760
1761 printk(KERN_DEBUG "... APIC ISR field:\n");
1762 print_APIC_bitfield(APIC_ISR);
1763 printk(KERN_DEBUG "... APIC TMR field:\n");
1764 print_APIC_bitfield(APIC_TMR);
1765 printk(KERN_DEBUG "... APIC IRR field:\n");
1766 print_APIC_bitfield(APIC_IRR);
1767
54168ed7
IM		 1768 if (APIC_INTEGRATED(ver)) { /* !82489DX */
1769 if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
1da177e4 1770 apic_write(APIC_ESR, 0);
54168ed7 1771
1da177e4
LT		 1772 v = apic_read(APIC_ESR);
1773 printk(KERN_DEBUG "... APIC ESR: %08x\n", v);
1774 }
1775
7ab6af7a 1776 icr = apic_icr_read();
0c425cec
IM		 1777 printk(KERN_DEBUG "... APIC ICR: %08x\n", (u32)icr);
1778 printk(KERN_DEBUG "... APIC ICR2: %08x\n", (u32)(icr >> 32));
1da177e4
LT		 1779
1780 v = apic_read(APIC_LVTT);
1781 printk(KERN_DEBUG "... APIC LVTT: %08x\n", v);
1782
1783 if (maxlvt > 3) { /* PC is LVT#4. */
1784 v = apic_read(APIC_LVTPC);
1785 printk(KERN_DEBUG "... APIC LVTPC: %08x\n", v);
1786 }
1787 v = apic_read(APIC_LVT0);
1788 printk(KERN_DEBUG "... APIC LVT0: %08x\n", v);
1789 v = apic_read(APIC_LVT1);
1790 printk(KERN_DEBUG "... APIC LVT1: %08x\n", v);
1791
1792 if (maxlvt > 2) { /* ERR is LVT#3. */
1793 v = apic_read(APIC_LVTERR);
1794 printk(KERN_DEBUG "... APIC LVTERR: %08x\n", v);
1795 }
1796
1797 v = apic_read(APIC_TMICT);
1798 printk(KERN_DEBUG "... APIC TMICT: %08x\n", v);
1799 v = apic_read(APIC_TMCCT);
1800 printk(KERN_DEBUG "... APIC TMCCT: %08x\n", v);
1801 v = apic_read(APIC_TDCR);
1802 printk(KERN_DEBUG "... APIC TDCR: %08x\n", v);
1803 printk("\n");
1804}
1805
32f71aff 1806__apicdebuginit(void) print_all_local_APICs(void)
1da177e4 1807{
ffd5aae7
YL		 1808 int cpu;
1809
1810 preempt_disable();
1811 for_each_online_cpu(cpu)
1812 smp_call_function_single(cpu, print_local_APIC, NULL, 1);
1813 preempt_enable();
1da177e4
LT		 1814}
1815
32f71aff 1816__apicdebuginit(void) print_PIC(void)
1da177e4 1817{
1da177e4
LT		 1818 unsigned int v;
1819 unsigned long flags;
1820
1821 if (apic_verbosity == APIC_QUIET)
1822 return;
1823
1824 printk(KERN_DEBUG "\nprinting PIC contents\n");
1825
1826 spin_lock_irqsave(&i8259A_lock, flags);
1827
1828 v = inb(0xa1) << 8 | inb(0x21);
1829 printk(KERN_DEBUG "... PIC IMR: %04x\n", v);
1830
1831 v = inb(0xa0) << 8 | inb(0x20);
1832 printk(KERN_DEBUG "... PIC IRR: %04x\n", v);
1833
54168ed7
IM		 1834 outb(0x0b,0xa0);
1835 outb(0x0b,0x20);
1da177e4 1836 v = inb(0xa0) << 8 | inb(0x20);
54168ed7
IM		 1837 outb(0x0a,0xa0);
1838 outb(0x0a,0x20);
1da177e4
LT		 1839
1840 spin_unlock_irqrestore(&i8259A_lock, flags);
1841
1842 printk(KERN_DEBUG "... PIC ISR: %04x\n", v);
1843
1844 v = inb(0x4d1) << 8 | inb(0x4d0);
1845 printk(KERN_DEBUG "... PIC ELCR: %04x\n", v);
1846}
1847
32f71aff
MR		 1848__apicdebuginit(int) print_all_ICs(void)
1849{
1850 print_PIC();
1851 print_all_local_APICs();
1852 print_IO_APIC();
1853
1854 return 0;
1855}
1856
1857fs_initcall(print_all_ICs);
1858
1da177e4 1859
efa2559f
YL		 1860/* Where if anywhere is the i8259 connect in external int mode */
1861static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };
1862
54168ed7 1863void __init enable_IO_APIC(void)
1da177e4
LT		 1864{
1865 union IO_APIC_reg_01 reg_01;
fcfd636a 1866 int i8259_apic, i8259_pin;
54168ed7 1867 int apic;
1da177e4
LT		 1868 unsigned long flags;
1869
54168ed7
IM		 1870#ifdef CONFIG_X86_32
1871 int i;
1da177e4
LT		 1872 if (!pirqs_enabled)
1873 for (i = 0; i < MAX_PIRQS; i++)
1874 pirq_entries[i] = -1;
54168ed7 1875#endif
1da177e4
LT		 1876
1877 /*
1878 * The number of IO-APIC IRQ registers (== #pins):
1879 */
fcfd636a 1880 for (apic = 0; apic < nr_ioapics; apic++) {
1da177e4 1881 spin_lock_irqsave(&ioapic_lock, flags);
fcfd636a 1882 reg_01.raw = io_apic_read(apic, 1);
1da177e4 1883 spin_unlock_irqrestore(&ioapic_lock, flags);
fcfd636a
EB		 1884 nr_ioapic_registers[apic] = reg_01.bits.entries+1;
1885 }
54168ed7 1886 for(apic = 0; apic < nr_ioapics; apic++) {
fcfd636a
EB		 1887 int pin;
1888 /* See if any of the pins is in ExtINT mode */
1008fddc 1889 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
fcfd636a 1890 struct IO_APIC_route_entry entry;
cf4c6a2f 1891 entry = ioapic_read_entry(apic, pin);
fcfd636a 1892
fcfd636a
EB		 1893 /* If the interrupt line is enabled and in ExtInt mode
1894 * I have found the pin where the i8259 is connected.
1895 */
1896 if ((entry.mask == 0) && (entry.delivery_mode == dest_ExtINT)) {
1897 ioapic_i8259.apic = apic;
1898 ioapic_i8259.pin = pin;
1899 goto found_i8259;
1900 }
1901 }
1902 }
1903 found_i8259:
1904 /* Look to see what if the MP table has reported the ExtINT */
1905 /* If we could not find the appropriate pin by looking at the ioapic
1906 * the i8259 probably is not connected the ioapic but give the
1907 * mptable a chance anyway.
1908 */
1909 i8259_pin = find_isa_irq_pin(0, mp_ExtINT);
1910 i8259_apic = find_isa_irq_apic(0, mp_ExtINT);
1911 /* Trust the MP table if nothing is setup in the hardware */
1912 if ((ioapic_i8259.pin == -1) && (i8259_pin >= 0)) {
1913 printk(KERN_WARNING "ExtINT not setup in hardware but reported by MP table\n");
1914 ioapic_i8259.pin = i8259_pin;
1915 ioapic_i8259.apic = i8259_apic;
1916 }
1917 /* Complain if the MP table and the hardware disagree */
1918 if (((ioapic_i8259.apic != i8259_apic) || (ioapic_i8259.pin != i8259_pin)) &&
1919 (i8259_pin >= 0) && (ioapic_i8259.pin >= 0))
1920 {
1921 printk(KERN_WARNING "ExtINT in hardware and MP table differ\n");
1da177e4
LT		 1922 }
1923
1924 /*
1925 * Do not trust the IO-APIC being empty at bootup
1926 */
1927 clear_IO_APIC();
1928}
1929
1930/*
1931 * Not an __init, needed by the reboot code
1932 */
1933void disable_IO_APIC(void)
1934{
1935 /*
1936 * Clear the IO-APIC before rebooting:
1937 */
1938 clear_IO_APIC();
1939
650927ef 1940 /*
0b968d23 1941 * If the i8259 is routed through an IOAPIC
650927ef 1942 * Put that IOAPIC in virtual wire mode
0b968d23 1943 * so legacy interrupts can be delivered.
650927ef 1944 */
fcfd636a 1945 if (ioapic_i8259.pin != -1) {
650927ef 1946 struct IO_APIC_route_entry entry;
650927ef
EB		 1947
1948 memset(&entry, 0, sizeof(entry));
1949 entry.mask = 0; /* Enabled */
1950 entry.trigger = 0; /* Edge */
1951 entry.irr = 0;
1952 entry.polarity = 0; /* High */
1953 entry.delivery_status = 0;
1954 entry.dest_mode = 0; /* Physical */
fcfd636a 1955 entry.delivery_mode = dest_ExtINT; /* ExtInt */
650927ef 1956 entry.vector = 0;
54168ed7 1957 entry.dest = read_apic_id();
650927ef
EB		 1958
1959 /*
1960 * Add it to the IO-APIC irq-routing table:
1961 */
cf4c6a2f 1962 ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry);
650927ef 1963 }
54168ed7 1964
fcfd636a 1965 disconnect_bsp_APIC(ioapic_i8259.pin != -1);
1da177e4
LT		 1966}
1967
54168ed7 1968#ifdef CONFIG_X86_32
1da177e4
LT		 1969/*
1970 * function to set the IO-APIC physical IDs based on the
1971 * values stored in the MPC table.
1972 *
1973 * by Matt Domsch <Matt_Domsch@dell.com> Tue Dec 21 12:25:05 CST 1999
1974 */
1975
1da177e4
LT		 1976static void __init setup_ioapic_ids_from_mpc(void)
1977{
1978 union IO_APIC_reg_00 reg_00;
1979 physid_mask_t phys_id_present_map;
1980 int apic;
1981 int i;
1982 unsigned char old_id;
1983 unsigned long flags;
1984
a4dbc34d 1985 if (x86_quirks->setup_ioapic_ids && x86_quirks->setup_ioapic_ids())
d49c4288 1986 return;
d49c4288 1987
ca05fea6
NP		 1988 /*
1989 * Don't check I/O APIC IDs for xAPIC systems. They have
1990 * no meaning without the serial APIC bus.
1991 */
7c5c1e42
SL		 1992 if (!(boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
1993 || APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
ca05fea6 1994 return;
1da177e4
LT		 1995 /*
1996 * This is broken; anything with a real cpu count has to
1997 * circumvent this idiocy regardless.
1998 */
1999 phys_id_present_map = ioapic_phys_id_map(phys_cpu_present_map);
2000
2001 /*
2002 * Set the IOAPIC ID to the value stored in the MPC table.
2003 */
2004 for (apic = 0; apic < nr_ioapics; apic++) {
2005
2006 /* Read the register 0 value */
2007 spin_lock_irqsave(&ioapic_lock, flags);
2008 reg_00.raw = io_apic_read(apic, 0);
2009 spin_unlock_irqrestore(&ioapic_lock, flags);
36062448 2010
ec2cd0a2 2011 old_id = mp_ioapics[apic].mp_apicid;
1da177e4 2012
ec2cd0a2 2013 if (mp_ioapics[apic].mp_apicid >= get_physical_broadcast()) {
1da177e4 2014 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n",
ec2cd0a2 2015 apic, mp_ioapics[apic].mp_apicid);
1da177e4
LT		 2016 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
2017 reg_00.bits.ID);
ec2cd0a2 2018 mp_ioapics[apic].mp_apicid = reg_00.bits.ID;
1da177e4
LT		 2019 }
2020
1da177e4
LT		 2021 /*
2022 * Sanity check, is the ID really free? Every APIC in a
2023 * system must have a unique ID or we get lots of nice
2024 * 'stuck on smp_invalidate_needed IPI wait' messages.
2025 */
2026 if (check_apicid_used(phys_id_present_map,
ec2cd0a2 2027 mp_ioapics[apic].mp_apicid)) {
1da177e4 2028 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n",
ec2cd0a2 2029 apic, mp_ioapics[apic].mp_apicid);
1da177e4
LT		 2030 for (i = 0; i < get_physical_broadcast(); i++)
2031 if (!physid_isset(i, phys_id_present_map))
2032 break;
2033 if (i >= get_physical_broadcast())
2034 panic("Max APIC ID exceeded!\n");
2035 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
2036 i);
2037 physid_set(i, phys_id_present_map);
ec2cd0a2 2038 mp_ioapics[apic].mp_apicid = i;
1da177e4
LT		 2039 } else {
2040 physid_mask_t tmp;
ec2cd0a2 2041 tmp = apicid_to_cpu_present(mp_ioapics[apic].mp_apicid);
1da177e4
LT		 2042 apic_printk(APIC_VERBOSE, "Setting %d in the "
2043 "phys_id_present_map\n",
ec2cd0a2 2044 mp_ioapics[apic].mp_apicid);
1da177e4
LT		 2045 physids_or(phys_id_present_map, phys_id_present_map, tmp);
2046 }
2047
2048
2049 /*
2050 * We need to adjust the IRQ routing table
2051 * if the ID changed.
2052 */
ec2cd0a2 2053 if (old_id != mp_ioapics[apic].mp_apicid)
1da177e4 2054 for (i = 0; i < mp_irq_entries; i++)
2fddb6e2
AS		 2055 if (mp_irqs[i].mp_dstapic == old_id)
2056 mp_irqs[i].mp_dstapic
ec2cd0a2 2057 = mp_ioapics[apic].mp_apicid;
1da177e4
LT		 2058
2059 /*
2060 * Read the right value from the MPC table and
2061 * write it into the ID register.
36062448 2062 */
1da177e4
LT		 2063 apic_printk(APIC_VERBOSE, KERN_INFO
2064 "...changing IO-APIC physical APIC ID to %d ...",
ec2cd0a2 2065 mp_ioapics[apic].mp_apicid);
1da177e4 2066
ec2cd0a2 2067 reg_00.bits.ID = mp_ioapics[apic].mp_apicid;
1da177e4 2068 spin_lock_irqsave(&ioapic_lock, flags);
1da177e4
LT		 2069
2070 /*
2071 * Sanity check
2072 */
2073 spin_lock_irqsave(&ioapic_lock, flags);
2074 reg_00.raw = io_apic_read(apic, 0);
2075 spin_unlock_irqrestore(&ioapic_lock, flags);
ec2cd0a2 2076 if (reg_00.bits.ID != mp_ioapics[apic].mp_apicid)
1da177e4
LT		 2077 printk("could not set ID!\n");
2078 else
2079 apic_printk(APIC_VERBOSE, " ok.\n");
2080 }
2081}
54168ed7 2082#endif
1da177e4 2083
7ce0bcfd 2084int no_timer_check __initdata;
8542b200
ZA		 2085
2086static int __init notimercheck(char *s)
2087{
2088 no_timer_check = 1;
2089 return 1;
2090}
2091__setup("no_timer_check", notimercheck);
2092
1da177e4
LT		 2093/*
2094 * There is a nasty bug in some older SMP boards, their mptable lies
2095 * about the timer IRQ. We do the following to work around the situation:
2096 *
2097 * - timer IRQ defaults to IO-APIC IRQ
2098 * - if this function detects that timer IRQs are defunct, then we fall
2099 * back to ISA timer IRQs
2100 */
f0a7a5c9 2101static int __init timer_irq_works(void)
1da177e4
LT		 2102{
2103 unsigned long t1 = jiffies;
4aae0702 2104 unsigned long flags;
1da177e4 2105
8542b200
ZA		 2106 if (no_timer_check)
2107 return 1;
2108
4aae0702 2109 local_save_flags(flags);
1da177e4
LT		 2110 local_irq_enable();
2111 /* Let ten ticks pass... */
2112 mdelay((10 * 1000) / HZ);
4aae0702 2113 local_irq_restore(flags);
1da177e4
LT		 2114
2115 /*
2116 * Expect a few ticks at least, to be sure some possible
2117 * glue logic does not lock up after one or two first
2118 * ticks in a non-ExtINT mode. Also the local APIC
2119 * might have cached one ExtINT interrupt. Finally, at
2120 * least one tick may be lost due to delays.
2121 */
54168ed7
IM		 2122
2123 /* jiffies wrap? */
1d16b53e 2124 if (time_after(jiffies, t1 + 4))
1da177e4 2125 return 1;
1da177e4
LT		 2126 return 0;
2127}
2128
2129/*
2130 * In the SMP+IOAPIC case it might happen that there are an unspecified
2131 * number of pending IRQ events unhandled. These cases are very rare,
2132 * so we 'resend' these IRQs via IPIs, to the same CPU. It's much
2133 * better to do it this way as thus we do not have to be aware of
2134 * 'pending' interrupts in the IRQ path, except at this point.
2135 */
2136/*
2137 * Edge triggered needs to resend any interrupt
2138 * that was delayed but this is now handled in the device
2139 * independent code.
2140 */
2141
2142/*
2143 * Starting up a edge-triggered IO-APIC interrupt is
2144 * nasty - we need to make sure that we get the edge.
2145 * If it is already asserted for some reason, we need
2146 * return 1 to indicate that is was pending.
2147 *
2148 * This is not complete - we should be able to fake
2149 * an edge even if it isn't on the 8259A...
2150 */
54168ed7 2151
f5b9ed7a 2152static unsigned int startup_ioapic_irq(unsigned int irq)
1da177e4
LT		 2153{
2154 int was_pending = 0;
2155 unsigned long flags;
2156
2157 spin_lock_irqsave(&ioapic_lock, flags);
2158 if (irq < 16) {
2159 disable_8259A_irq(irq);
2160 if (i8259A_irq_pending(irq))
2161 was_pending = 1;
2162 }
2163 __unmask_IO_APIC_irq(irq);
2164 spin_unlock_irqrestore(&ioapic_lock, flags);
2165
2166 return was_pending;
2167}
2168
54168ed7 2169#ifdef CONFIG_X86_64
ace80ab7 2170static int ioapic_retrigger_irq(unsigned int irq)
1da177e4 2171{
54168ed7
IM		 2172
2173 struct irq_cfg *cfg = irq_cfg(irq);
2174 unsigned long flags;
2175
2176 spin_lock_irqsave(&vector_lock, flags);
2177 send_IPI_mask(cpumask_of_cpu(first_cpu(cfg->domain)), cfg->vector);
2178 spin_unlock_irqrestore(&vector_lock, flags);
c0ad90a3
IM		 2179
2180 return 1;
2181}
54168ed7
IM		 2182#else
2183static int ioapic_retrigger_irq(unsigned int irq)
497c9a19 2184{
54168ed7 2185 send_IPI_self(irq_cfg(irq)->vector);
497c9a19 2186
54168ed7
IM		 2187 return 1;
2188}
2189#endif
497c9a19 2190
54168ed7
IM		 2191/*
2192 * Level and edge triggered IO-APIC interrupts need different handling,
2193 * so we use two separate IRQ descriptors. Edge triggered IRQs can be
2194 * handled with the level-triggered descriptor, but that one has slightly
2195 * more overhead. Level-triggered interrupts cannot be handled with the
2196 * edge-triggered handler, without risking IRQ storms and other ugly
2197 * races.
2198 */
497c9a19 2199
54168ed7 2200#ifdef CONFIG_SMP
497c9a19 2201
54168ed7
IM		 2202#ifdef CONFIG_INTR_REMAP
2203static void ir_irq_migration(struct work_struct *work);
497c9a19 2204
54168ed7 2205static DECLARE_DELAYED_WORK(ir_migration_work, ir_irq_migration);
497c9a19 2206
54168ed7
IM		 2207/*
2208 * Migrate the IO-APIC irq in the presence of intr-remapping.
2209 *
2210 * For edge triggered, irq migration is a simple atomic update(of vector
2211 * and cpu destination) of IRTE and flush the hardware cache.
2212 *
2213 * For level triggered, we need to modify the io-apic RTE aswell with the update
2214 * vector information, along with modifying IRTE with vector and destination.
2215 * So irq migration for level triggered is little bit more complex compared to
2216 * edge triggered migration. But the good news is, we use the same algorithm
2217 * for level triggered migration as we have today, only difference being,
2218 * we now initiate the irq migration from process context instead of the
2219 * interrupt context.
2220 *
2221 * In future, when we do a directed EOI (combined with cpu EOI broadcast
2222 * suppression) to the IO-APIC, level triggered irq migration will also be
2223 * as simple as edge triggered migration and we can do the irq migration
2224 * with a simple atomic update to IO-APIC RTE.
2225 */
2226static void migrate_ioapic_irq(int irq, cpumask_t mask)
497c9a19 2227{
54168ed7
IM		 2228 struct irq_cfg *cfg;
2229 struct irq_desc *desc;
2230 cpumask_t tmp, cleanup_mask;
2231 struct irte irte;
2232 int modify_ioapic_rte;
2233 unsigned int dest;
2234 unsigned long flags;
497c9a19 2235
54168ed7
IM		 2236 cpus_and(tmp, mask, cpu_online_map);
2237 if (cpus_empty(tmp))
497c9a19
YL		 2238 return;
2239
54168ed7
IM		 2240 if (get_irte(irq, &irte))
2241 return;
497c9a19 2242
54168ed7
IM		 2243 if (assign_irq_vector(irq, mask))
2244 return;
2245
2246 cfg = irq_cfg(irq);
2247 cpus_and(tmp, cfg->domain, mask);
2248 dest = cpu_mask_to_apicid(tmp);
2249
2250 desc = irq_to_desc(irq);
2251 modify_ioapic_rte = desc->status & IRQ_LEVEL;
2252 if (modify_ioapic_rte) {
2253 spin_lock_irqsave(&ioapic_lock, flags);
2254 __target_IO_APIC_irq(irq, dest, cfg->vector);
2255 spin_unlock_irqrestore(&ioapic_lock, flags);
2256 }
2257
2258 irte.vector = cfg->vector;
2259 irte.dest_id = IRTE_DEST(dest);
2260
2261 /*
2262 * Modified the IRTE and flushes the Interrupt entry cache.
2263 */
2264 modify_irte(irq, &irte);
2265
2266 if (cfg->move_in_progress) {
2267 cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map);
2268 cfg->move_cleanup_count = cpus_weight(cleanup_mask);
2269 send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
2270 cfg->move_in_progress = 0;
2271 }
2272
2273 desc->affinity = mask;
2274}
2275
2276static int migrate_irq_remapped_level(int irq)
2277{
2278 int ret = -1;
2279 struct irq_desc *desc = irq_to_desc(irq);
2280
2281 mask_IO_APIC_irq(irq);
2282
2283 if (io_apic_level_ack_pending(irq)) {
2284 /*
2285 * Interrupt in progress. Migrating irq now will change the
2286 * vector information in the IO-APIC RTE and that will confuse
2287 * the EOI broadcast performed by cpu.
2288 * So, delay the irq migration to the next instance.
2289 */
2290 schedule_delayed_work(&ir_migration_work, 1);
2291 goto unmask;
2292 }
2293
2294 /* everthing is clear. we have right of way */
2295 migrate_ioapic_irq(irq, desc->pending_mask);
2296
2297 ret = 0;
2298 desc->status &= ~IRQ_MOVE_PENDING;
2299 cpus_clear(desc->pending_mask);
2300
2301unmask:
2302 unmask_IO_APIC_irq(irq);
2303 return ret;
2304}
2305
2306static void ir_irq_migration(struct work_struct *work)
2307{
2308 unsigned int irq;
2309 struct irq_desc *desc;
2310
2311 for_each_irq_desc(irq, desc) {
2312 if (desc->status & IRQ_MOVE_PENDING) {
2313 unsigned long flags;
2314
2315 spin_lock_irqsave(&desc->lock, flags);
2316 if (!desc->chip->set_affinity ||
2317 !(desc->status & IRQ_MOVE_PENDING)) {
2318 desc->status &= ~IRQ_MOVE_PENDING;
2319 spin_unlock_irqrestore(&desc->lock, flags);
2320 continue;
2321 }
2322
2323 desc->chip->set_affinity(irq, desc->pending_mask);
2324 spin_unlock_irqrestore(&desc->lock, flags);
2325 }
2326 }
2327}
2328
2329/*
2330 * Migrates the IRQ destination in the process context.
2331 */
2332static void set_ir_ioapic_affinity_irq(unsigned int irq, cpumask_t mask)
2333{
2334 struct irq_desc *desc = irq_to_desc(irq);
2335
2336 if (desc->status & IRQ_LEVEL) {
2337 desc->status |= IRQ_MOVE_PENDING;
2338 desc->pending_mask = mask;
2339 migrate_irq_remapped_level(irq);
2340 return;
2341 }
2342
2343 migrate_ioapic_irq(irq, mask);
2344}
2345#endif
2346
2347asmlinkage void smp_irq_move_cleanup_interrupt(void)
2348{
2349 unsigned vector, me;
2350 ack_APIC_irq();
2351#ifdef CONFIG_X86_64
2352 exit_idle();
2353#endif
2354 irq_enter();
2355
2356 me = smp_processor_id();
2357 for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
2358 unsigned int irq;
2359 struct irq_desc *desc;
2360 struct irq_cfg *cfg;
2361 irq = __get_cpu_var(vector_irq)[vector];
2362
2363 desc = irq_to_desc(irq);
2364 if (!desc)
2365 continue;
2366
2367 cfg = irq_cfg(irq);
2368 spin_lock(&desc->lock);
2369 if (!cfg->move_cleanup_count)
2370 goto unlock;
2371
2372 if ((vector == cfg->vector) && cpu_isset(me, cfg->domain))
2373 goto unlock;
2374
2375 __get_cpu_var(vector_irq)[vector] = -1;
2376 cfg->move_cleanup_count--;
2377unlock:
2378 spin_unlock(&desc->lock);
2379 }
2380
2381 irq_exit();
2382}
2383
2384static void irq_complete_move(unsigned int irq)
2385{
2386 struct irq_cfg *cfg = irq_cfg(irq);
2387 unsigned vector, me;
2388
2389 if (likely(!cfg->move_in_progress))
2390 return;
2391
2392 vector = ~get_irq_regs()->orig_ax;
2393 me = smp_processor_id();
2394 if ((vector == cfg->vector) && cpu_isset(me, cfg->domain)) {
2395 cpumask_t cleanup_mask;
2396
2397 cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map);
2398 cfg->move_cleanup_count = cpus_weight(cleanup_mask);
2399 send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
497c9a19
YL		 2400 cfg->move_in_progress = 0;
2401 }
2402}
2403#else
2404static inline void irq_complete_move(unsigned int irq) {}
2405#endif
54168ed7
IM		 2406#ifdef CONFIG_INTR_REMAP
2407static void ack_x2apic_level(unsigned int irq)
2408{
2409 ack_x2APIC_irq();
2410}
2411
2412static void ack_x2apic_edge(unsigned int irq)
2413{
2414 ack_x2APIC_irq();
2415}
2416#endif
497c9a19 2417
1d025192
YL		 2418static void ack_apic_edge(unsigned int irq)
2419{
2420 irq_complete_move(irq);
2421 move_native_irq(irq);
2422 ack_APIC_irq();
2423}
2424
3eb2cce8
YL		 2425#ifdef CONFIG_X86_32
2426atomic_t irq_mis_count;
2427#endif
2428
047c8fdb
YL		 2429static void ack_apic_level(unsigned int irq)
2430{
3eb2cce8
YL		 2431#ifdef CONFIG_X86_32
2432 unsigned long v;
2433 int i;
2434#endif
54168ed7 2435 int do_unmask_irq = 0;
047c8fdb 2436
54168ed7 2437 irq_complete_move(irq);
047c8fdb 2438#ifdef CONFIG_GENERIC_PENDING_IRQ
54168ed7
IM		 2439 /* If we are moving the irq we need to mask it */
2440 if (unlikely(irq_to_desc(irq)->status & IRQ_MOVE_PENDING)) {
2441 do_unmask_irq = 1;
2442 mask_IO_APIC_irq(irq);
2443 }
047c8fdb
YL		 2444#endif
2445
3eb2cce8
YL		 2446#ifdef CONFIG_X86_32
2447 /*
2448 * It appears there is an erratum which affects at least version 0x11
2449 * of I/O APIC (that's the 82093AA and cores integrated into various
2450 * chipsets). Under certain conditions a level-triggered interrupt is
2451 * erroneously delivered as edge-triggered one but the respective IRR
2452 * bit gets set nevertheless. As a result the I/O unit expects an EOI
2453 * message but it will never arrive and further interrupts are blocked
2454 * from the source. The exact reason is so far unknown, but the
2455 * phenomenon was observed when two consecutive interrupt requests
2456 * from a given source get delivered to the same CPU and the source is
2457 * temporarily disabled in between.
2458 *
2459 * A workaround is to simulate an EOI message manually. We achieve it
2460 * by setting the trigger mode to edge and then to level when the edge
2461 * trigger mode gets detected in the TMR of a local APIC for a
2462 * level-triggered interrupt. We mask the source for the time of the
2463 * operation to prevent an edge-triggered interrupt escaping meanwhile.
2464 * The idea is from Manfred Spraul. --macro
2465 */
2466 i = irq_cfg(irq)->vector;
2467
2468 v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1));
2469#endif
2470
54168ed7
IM		 2471 /*
2472 * We must acknowledge the irq before we move it or the acknowledge will
2473 * not propagate properly.
2474 */
2475 ack_APIC_irq();
2476
2477 /* Now we can move and renable the irq */
2478 if (unlikely(do_unmask_irq)) {
2479 /* Only migrate the irq if the ack has been received.
2480 *
2481 * On rare occasions the broadcast level triggered ack gets
2482 * delayed going to ioapics, and if we reprogram the
2483 * vector while Remote IRR is still set the irq will never
2484 * fire again.
2485 *
2486 * To prevent this scenario we read the Remote IRR bit
2487 * of the ioapic. This has two effects.
2488 * - On any sane system the read of the ioapic will
2489 * flush writes (and acks) going to the ioapic from
2490 * this cpu.
2491 * - We get to see if the ACK has actually been delivered.
2492 *
2493 * Based on failed experiments of reprogramming the
2494 * ioapic entry from outside of irq context starting
2495 * with masking the ioapic entry and then polling until
2496 * Remote IRR was clear before reprogramming the
2497 * ioapic I don't trust the Remote IRR bit to be
2498 * completey accurate.
2499 *
2500 * However there appears to be no other way to plug
2501 * this race, so if the Remote IRR bit is not
2502 * accurate and is causing problems then it is a hardware bug
2503 * and you can go talk to the chipset vendor about it.
2504 */
2505 if (!io_apic_level_ack_pending(irq))
2506 move_masked_irq(irq);
2507 unmask_IO_APIC_irq(irq);
2508 }
1d025192 2509
3eb2cce8 2510#ifdef CONFIG_X86_32
1d025192
YL		 2511 if (!(v & (1 << (i & 0x1f)))) {
2512 atomic_inc(&irq_mis_count);
2513 spin_lock(&ioapic_lock);
2514 __mask_and_edge_IO_APIC_irq(irq);
2515 __unmask_and_level_IO_APIC_irq(irq);
2516 spin_unlock(&ioapic_lock);
2517 }
047c8fdb 2518#endif
3eb2cce8 2519}
1d025192 2520
f5b9ed7a
IM		 2521static struct irq_chip ioapic_chip __read_mostly = {
2522 .name = "IO-APIC",
ace80ab7
EB		 2523 .startup = startup_ioapic_irq,
2524 .mask = mask_IO_APIC_irq,
2525 .unmask = unmask_IO_APIC_irq,
1d025192
YL		 2526 .ack = ack_apic_edge,
2527 .eoi = ack_apic_level,
54d5d424 2528#ifdef CONFIG_SMP
ace80ab7 2529 .set_affinity = set_ioapic_affinity_irq,
54d5d424 2530#endif
ace80ab7 2531 .retrigger = ioapic_retrigger_irq,
1da177e4
LT		 2532};
2533
54168ed7
IM		 2534#ifdef CONFIG_INTR_REMAP
2535static struct irq_chip ir_ioapic_chip __read_mostly = {
2536 .name = "IR-IO-APIC",
2537 .startup = startup_ioapic_irq,
2538 .mask = mask_IO_APIC_irq,
2539 .unmask = unmask_IO_APIC_irq,
2540 .ack = ack_x2apic_edge,
2541 .eoi = ack_x2apic_level,
2542#ifdef CONFIG_SMP
2543 .set_affinity = set_ir_ioapic_affinity_irq,
2544#endif
2545 .retrigger = ioapic_retrigger_irq,
2546};
2547#endif
1da177e4
LT		 2548
2549static inline void init_IO_APIC_traps(void)
2550{
2551 int irq;
08678b08 2552 struct irq_desc *desc;
da51a821 2553 struct irq_cfg *cfg;
1da177e4
LT		 2554
2555 /*
2556 * NOTE! The local APIC isn't very good at handling
2557 * multiple interrupts at the same interrupt level.
2558 * As the interrupt level is determined by taking the
2559 * vector number and shifting that right by 4, we
2560 * want to spread these out a bit so that they don't
2561 * all fall in the same interrupt level.
2562 *
2563 * Also, we've got to be careful not to trash gate
2564 * 0x80, because int 0x80 is hm, kind of importantish. ;)
2565 */
8f09cd20 2566 for_each_irq_cfg(irq, cfg) {
da51a821 2567 if (IO_APIC_IRQ(irq) && !cfg->vector) {
1da177e4
LT		 2568 /*
2569 * Hmm.. We don't have an entry for this,
2570 * so default to an old-fashioned 8259
2571 * interrupt if we can..
2572 */
2573 if (irq < 16)
2574 make_8259A_irq(irq);
08678b08
YL		 2575 else {
2576 desc = irq_to_desc(irq);
1da177e4 2577 /* Strange. Oh, well.. */
08678b08
YL		 2578 desc->chip = &no_irq_chip;
2579 }
1da177e4
LT		 2580 }
2581 }
2582}
2583
f5b9ed7a
IM		 2584/*
2585 * The local APIC irq-chip implementation:
2586 */
1da177e4 2587
36062448 2588static void mask_lapic_irq(unsigned int irq)
1da177e4
LT		 2589{
2590 unsigned long v;
2591
2592 v = apic_read(APIC_LVT0);
593f4a78 2593 apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
1da177e4
LT		 2594}
2595
36062448 2596static void unmask_lapic_irq(unsigned int irq)
1da177e4 2597{
f5b9ed7a 2598 unsigned long v;
1da177e4 2599
f5b9ed7a 2600 v = apic_read(APIC_LVT0);
593f4a78 2601 apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED);
f5b9ed7a 2602}
1da177e4 2603
54168ed7 2604static void ack_lapic_irq (unsigned int irq)
1d025192
YL		 2605{
2606 ack_APIC_irq();
2607}
2608
f5b9ed7a 2609static struct irq_chip lapic_chip __read_mostly = {
9a1c6192 2610 .name = "local-APIC",
f5b9ed7a
IM		 2611 .mask = mask_lapic_irq,
2612 .unmask = unmask_lapic_irq,
c88ac1df 2613 .ack = ack_lapic_irq,
1da177e4
LT		 2614};
2615
497c9a19 2616static void lapic_register_intr(int irq)
c88ac1df 2617{
08678b08
YL		 2618 struct irq_desc *desc;
2619
2620 desc = irq_to_desc(irq);
2621 desc->status &= ~IRQ_LEVEL;
c88ac1df
MR		 2622 set_irq_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq,
2623 "edge");
c88ac1df
MR		 2624}
2625
e9427101 2626static void __init setup_nmi(void)
1da177e4
LT		 2627{
2628 /*
36062448 2629 * Dirty trick to enable the NMI watchdog ...
1da177e4
LT		 2630 * We put the 8259A master into AEOI mode and
2631 * unmask on all local APICs LVT0 as NMI.
2632 *
2633 * The idea to use the 8259A in AEOI mode ('8259A Virtual Wire')
2634 * is from Maciej W. Rozycki - so we do not have to EOI from
2635 * the NMI handler or the timer interrupt.
36062448 2636 */
1da177e4
LT		 2637 apic_printk(APIC_VERBOSE, KERN_INFO "activating NMI Watchdog ...");
2638
e9427101 2639 enable_NMI_through_LVT0();
1da177e4
LT		 2640
2641 apic_printk(APIC_VERBOSE, " done.\n");
2642}
2643
2644/*
2645 * This looks a bit hackish but it's about the only one way of sending
2646 * a few INTA cycles to 8259As and any associated glue logic. ICR does
2647 * not support the ExtINT mode, unfortunately. We need to send these
2648 * cycles as some i82489DX-based boards have glue logic that keeps the
2649 * 8259A interrupt line asserted until INTA. --macro
2650 */
28acf285 2651static inline void __init unlock_ExtINT_logic(void)
1da177e4 2652{
fcfd636a 2653 int apic, pin, i;
1da177e4
LT		 2654 struct IO_APIC_route_entry entry0, entry1;
2655 unsigned char save_control, save_freq_select;
1da177e4 2656
fcfd636a 2657 pin = find_isa_irq_pin(8, mp_INT);
956fb531
AB		 2658 if (pin == -1) {
2659 WARN_ON_ONCE(1);
2660 return;
2661 }
fcfd636a 2662 apic = find_isa_irq_apic(8, mp_INT);
956fb531
AB		 2663 if (apic == -1) {
2664 WARN_ON_ONCE(1);
1da177e4 2665 return;
956fb531 2666 }
1da177e4 2667
cf4c6a2f 2668 entry0 = ioapic_read_entry(apic, pin);
fcfd636a 2669 clear_IO_APIC_pin(apic, pin);
1da177e4
LT		 2670
2671 memset(&entry1, 0, sizeof(entry1));
2672
2673 entry1.dest_mode = 0; /* physical delivery */
2674 entry1.mask = 0; /* unmask IRQ now */
d83e94ac 2675 entry1.dest = hard_smp_processor_id();
1da177e4
LT		 2676 entry1.delivery_mode = dest_ExtINT;
2677 entry1.polarity = entry0.polarity;
2678 entry1.trigger = 0;
2679 entry1.vector = 0;
2680
cf4c6a2f 2681 ioapic_write_entry(apic, pin, entry1);
1da177e4
LT		 2682
2683 save_control = CMOS_READ(RTC_CONTROL);
2684 save_freq_select = CMOS_READ(RTC_FREQ_SELECT);
2685 CMOS_WRITE((save_freq_select & ~RTC_RATE_SELECT) | 0x6,
2686 RTC_FREQ_SELECT);
2687 CMOS_WRITE(save_control | RTC_PIE, RTC_CONTROL);
2688
2689 i = 100;
2690 while (i-- > 0) {
2691 mdelay(10);
2692 if ((CMOS_READ(RTC_INTR_FLAGS) & RTC_PF) == RTC_PF)
2693 i -= 10;
2694 }
2695
2696 CMOS_WRITE(save_control, RTC_CONTROL);
2697 CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT);
fcfd636a 2698 clear_IO_APIC_pin(apic, pin);
1da177e4 2699
cf4c6a2f 2700 ioapic_write_entry(apic, pin, entry0);
1da177e4
LT		 2701}
2702
efa2559f 2703static int disable_timer_pin_1 __initdata;
047c8fdb 2704/* Actually the next is obsolete, but keep it for paranoid reasons -AK */
54168ed7 2705static int __init disable_timer_pin_setup(char *arg)
efa2559f
YL		 2706{
2707 disable_timer_pin_1 = 1;
2708 return 0;
2709}
54168ed7 2710early_param("disable_timer_pin_1", disable_timer_pin_setup);
efa2559f
YL		 2711
2712int timer_through_8259 __initdata;
2713
1da177e4
LT		 2714/*
2715 * This code may look a bit paranoid, but it's supposed to cooperate with
2716 * a wide range of boards and BIOS bugs. Fortunately only the timer IRQ
2717 * is so screwy. Thanks to Brian Perkins for testing/hacking this beast
2718 * fanatically on his truly buggy board.
54168ed7
IM		 2719 *
2720 * FIXME: really need to revamp this for all platforms.
1da177e4 2721 */
8542b200 2722static inline void __init check_timer(void)
1da177e4 2723{
497c9a19 2724 struct irq_cfg *cfg = irq_cfg(0);
fcfd636a 2725 int apic1, pin1, apic2, pin2;
4aae0702 2726 unsigned long flags;
047c8fdb
YL		 2727 unsigned int ver;
2728 int no_pin1 = 0;
4aae0702
IM		 2729
2730 local_irq_save(flags);
d4d25dec 2731
54168ed7
IM		 2732 ver = apic_read(APIC_LVR);
2733 ver = GET_APIC_VERSION(ver);
6e908947 2734
1da177e4
LT		 2735 /*
2736 * get/set the timer IRQ vector:
2737 */
2738 disable_8259A_irq(0);
497c9a19 2739 assign_irq_vector(0, TARGET_CPUS);
1da177e4
LT		 2740
2741 /*
d11d5794
MR		 2742 * As IRQ0 is to be enabled in the 8259A, the virtual
2743 * wire has to be disabled in the local APIC. Also
2744 * timer interrupts need to be acknowledged manually in
2745 * the 8259A for the i82489DX when using the NMI
2746 * watchdog as that APIC treats NMIs as level-triggered.
2747 * The AEOI mode will finish them in the 8259A
2748 * automatically.
1da177e4 2749 */
593f4a78 2750 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
1da177e4 2751 init_8259A(1);
54168ed7 2752#ifdef CONFIG_X86_32
d11d5794 2753 timer_ack = (nmi_watchdog == NMI_IO_APIC && !APIC_INTEGRATED(ver));
54168ed7 2754#endif
1da177e4 2755
fcfd636a
EB		 2756 pin1 = find_isa_irq_pin(0, mp_INT);
2757 apic1 = find_isa_irq_apic(0, mp_INT);
2758 pin2 = ioapic_i8259.pin;
2759 apic2 = ioapic_i8259.apic;
1da177e4 2760
49a66a0b
MR		 2761 apic_printk(APIC_QUIET, KERN_INFO "..TIMER: vector=0x%02X "
2762 "apic1=%d pin1=%d apic2=%d pin2=%d\n",
497c9a19 2763 cfg->vector, apic1, pin1, apic2, pin2);
1da177e4 2764
691874fa
MR		 2765 /*
2766 * Some BIOS writers are clueless and report the ExtINTA
2767 * I/O APIC input from the cascaded 8259A as the timer
2768 * interrupt input. So just in case, if only one pin
2769 * was found above, try it both directly and through the
2770 * 8259A.
2771 */
2772 if (pin1 == -1) {
54168ed7
IM		 2773#ifdef CONFIG_INTR_REMAP
2774 if (intr_remapping_enabled)
2775 panic("BIOS bug: timer not connected to IO-APIC");
2776#endif
691874fa
MR		 2777 pin1 = pin2;
2778 apic1 = apic2;
2779 no_pin1 = 1;
2780 } else if (pin2 == -1) {
2781 pin2 = pin1;
2782 apic2 = apic1;
2783 }
2784
1da177e4
LT		 2785 if (pin1 != -1) {
2786 /*
2787 * Ok, does IRQ0 through the IOAPIC work?
2788 */
691874fa
MR		 2789 if (no_pin1) {
2790 add_pin_to_irq(0, apic1, pin1);
497c9a19 2791 setup_timer_IRQ0_pin(apic1, pin1, cfg->vector);
691874fa 2792 }
1da177e4
LT		 2793 unmask_IO_APIC_irq(0);
2794 if (timer_irq_works()) {
2795 if (nmi_watchdog == NMI_IO_APIC) {
1da177e4
LT		 2796 setup_nmi();
2797 enable_8259A_irq(0);
1da177e4 2798 }
66759a01
CE		 2799 if (disable_timer_pin_1 > 0)
2800 clear_IO_APIC_pin(0, pin1);
4aae0702 2801 goto out;
1da177e4 2802 }
54168ed7
IM		 2803#ifdef CONFIG_INTR_REMAP
2804 if (intr_remapping_enabled)
2805 panic("timer doesn't work through Interrupt-remapped IO-APIC");
2806#endif
fcfd636a 2807 clear_IO_APIC_pin(apic1, pin1);
691874fa 2808 if (!no_pin1)
49a66a0b
MR		 2809 apic_printk(APIC_QUIET, KERN_ERR "..MP-BIOS bug: "
2810 "8254 timer not connected to IO-APIC\n");
1da177e4 2811
49a66a0b
MR		 2812 apic_printk(APIC_QUIET, KERN_INFO "...trying to set up timer "
2813 "(IRQ0) through the 8259A ...\n");
2814 apic_printk(APIC_QUIET, KERN_INFO
2815 "..... (found apic %d pin %d) ...\n", apic2, pin2);
1da177e4
LT		 2816 /*
2817 * legacy devices should be connected to IO APIC #0
2818 */
691874fa 2819 replace_pin_at_irq(0, apic1, pin1, apic2, pin2);
497c9a19 2820 setup_timer_IRQ0_pin(apic2, pin2, cfg->vector);
24742ece 2821 unmask_IO_APIC_irq(0);
ecd29476 2822 enable_8259A_irq(0);
1da177e4 2823 if (timer_irq_works()) {
49a66a0b 2824 apic_printk(APIC_QUIET, KERN_INFO "....... works.\n");
35542c5e 2825 timer_through_8259 = 1;
1da177e4 2826 if (nmi_watchdog == NMI_IO_APIC) {
60134ebe 2827 disable_8259A_irq(0);
1da177e4 2828 setup_nmi();
60134ebe 2829 enable_8259A_irq(0);
1da177e4 2830 }
4aae0702 2831 goto out;
1da177e4
LT		 2832 }
2833 /*
2834 * Cleanup, just in case ...
2835 */
ecd29476 2836 disable_8259A_irq(0);
fcfd636a 2837 clear_IO_APIC_pin(apic2, pin2);
49a66a0b 2838 apic_printk(APIC_QUIET, KERN_INFO "....... failed.\n");
1da177e4 2839 }
1da177e4
LT		 2840
2841 if (nmi_watchdog == NMI_IO_APIC) {
49a66a0b
MR		 2842 apic_printk(APIC_QUIET, KERN_WARNING "timer doesn't work "
2843 "through the IO-APIC - disabling NMI Watchdog!\n");
067fa0ff 2844 nmi_watchdog = NMI_NONE;
1da177e4 2845 }
54168ed7 2846#ifdef CONFIG_X86_32
d11d5794 2847 timer_ack = 0;
54168ed7 2848#endif
1da177e4 2849
49a66a0b
MR		 2850 apic_printk(APIC_QUIET, KERN_INFO
2851 "...trying to set up timer as Virtual Wire IRQ...\n");
1da177e4 2852
497c9a19
YL		 2853 lapic_register_intr(0);
2854 apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector); /* Fixed mode */
1da177e4
LT		 2855 enable_8259A_irq(0);
2856
2857 if (timer_irq_works()) {
49a66a0b 2858 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
4aae0702 2859 goto out;
1da177e4 2860 }
e67465f1 2861 disable_8259A_irq(0);
497c9a19 2862 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector);
49a66a0b 2863 apic_printk(APIC_QUIET, KERN_INFO "..... failed.\n");
1da177e4 2864
49a66a0b
MR		 2865 apic_printk(APIC_QUIET, KERN_INFO
2866 "...trying to set up timer as ExtINT IRQ...\n");
1da177e4 2867
1da177e4
LT		 2868 init_8259A(0);
2869 make_8259A_irq(0);
593f4a78 2870 apic_write(APIC_LVT0, APIC_DM_EXTINT);
1da177e4
LT		 2871
2872 unlock_ExtINT_logic();
2873
2874 if (timer_irq_works()) {
49a66a0b 2875 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
4aae0702 2876 goto out;
1da177e4 2877 }
49a66a0b 2878 apic_printk(APIC_QUIET, KERN_INFO "..... failed :(.\n");
1da177e4 2879 panic("IO-APIC + timer doesn't work! Boot with apic=debug and send a "
49a66a0b 2880 "report. Then try booting with the 'noapic' option.\n");
4aae0702
IM		 2881out:
2882 local_irq_restore(flags);
1da177e4
LT		 2883}
2884
2885/*
af174783
MR		 2886 * Traditionally ISA IRQ2 is the cascade IRQ, and is not available
2887 * to devices. However there may be an I/O APIC pin available for
2888 * this interrupt regardless. The pin may be left unconnected, but
2889 * typically it will be reused as an ExtINT cascade interrupt for
2890 * the master 8259A. In the MPS case such a pin will normally be
2891 * reported as an ExtINT interrupt in the MP table. With ACPI
2892 * there is no provision for ExtINT interrupts, and in the absence
2893 * of an override it would be treated as an ordinary ISA I/O APIC
2894 * interrupt, that is edge-triggered and unmasked by default. We
2895 * used to do this, but it caused problems on some systems because
2896 * of the NMI watchdog and sometimes IRQ0 of the 8254 timer using
2897 * the same ExtINT cascade interrupt to drive the local APIC of the
2898 * bootstrap processor. Therefore we refrain from routing IRQ2 to
2899 * the I/O APIC in all cases now. No actual device should request
2900 * it anyway. --macro
1da177e4
LT		 2901 */
2902#define PIC_IRQS (1 << PIC_CASCADE_IR)
2903
2904void __init setup_IO_APIC(void)
2905{
54168ed7
IM		 2906
2907#ifdef CONFIG_X86_32
1da177e4 2908 enable_IO_APIC();
54168ed7
IM		 2909#else
2910 /*
2911 * calling enable_IO_APIC() is moved to setup_local_APIC for BP
2912 */
2913#endif
1da177e4 2914
af174783 2915 io_apic_irqs = ~PIC_IRQS;
1da177e4 2916
54168ed7
IM		 2917 apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n");
2918 /*
2919 * Set up IO-APIC IRQ routing.
2920 */
2921#ifdef CONFIG_X86_32
2922 if (!acpi_ioapic)
2923 setup_ioapic_ids_from_mpc();
2924#endif
1da177e4
LT		 2925 sync_Arb_IDs();
2926 setup_IO_APIC_irqs();
2927 init_IO_APIC_traps();
1e4c85f9 2928 check_timer();
1da177e4
LT		 2929}
2930
2931/*
54168ed7
IM		 2932 * Called after all the initialization is done. If we didnt find any
2933 * APIC bugs then we can allow the modify fast path
1da177e4 2934 */
36062448 2935
1da177e4
LT		 2936static int __init io_apic_bug_finalize(void)
2937{
54168ed7
IM		 2938 if (sis_apic_bug == -1)
2939 sis_apic_bug = 0;
2940 return 0;
1da177e4
LT		 2941}
2942
2943late_initcall(io_apic_bug_finalize);
2944
2945struct sysfs_ioapic_data {
2946 struct sys_device dev;
2947 struct IO_APIC_route_entry entry[0];
2948};
54168ed7 2949static struct sysfs_ioapic_data * mp_ioapic_data[MAX_IO_APICS];
1da177e4 2950
438510f6 2951static int ioapic_suspend(struct sys_device *dev, pm_message_t state)
1da177e4
LT		 2952{
2953 struct IO_APIC_route_entry *entry;
2954 struct sysfs_ioapic_data *data;
1da177e4 2955 int i;
36062448 2956
1da177e4
LT		 2957 data = container_of(dev, struct sysfs_ioapic_data, dev);
2958 entry = data->entry;
54168ed7
IM		 2959 for (i = 0; i < nr_ioapic_registers[dev->id]; i ++, entry ++ )
2960 *entry = ioapic_read_entry(dev->id, i);
1da177e4
LT		 2961
2962 return 0;
2963}
2964
2965static int ioapic_resume(struct sys_device *dev)
2966{
2967 struct IO_APIC_route_entry *entry;
2968 struct sysfs_ioapic_data *data;
2969 unsigned long flags;
2970 union IO_APIC_reg_00 reg_00;
2971 int i;
36062448 2972
1da177e4
LT		 2973 data = container_of(dev, struct sysfs_ioapic_data, dev);
2974 entry = data->entry;
2975
2976 spin_lock_irqsave(&ioapic_lock, flags);
2977 reg_00.raw = io_apic_read(dev->id, 0);
ec2cd0a2
AS		 2978 if (reg_00.bits.ID != mp_ioapics[dev->id].mp_apicid) {
2979 reg_00.bits.ID = mp_ioapics[dev->id].mp_apicid;
1da177e4
LT		 2980 io_apic_write(dev->id, 0, reg_00.raw);
2981 }
1da177e4 2982 spin_unlock_irqrestore(&ioapic_lock, flags);
36062448 2983 for (i = 0; i < nr_ioapic_registers[dev->id]; i++)
cf4c6a2f 2984 ioapic_write_entry(dev->id, i, entry[i]);
1da177e4
LT		 2985
2986 return 0;
2987}
2988
2989static struct sysdev_class ioapic_sysdev_class = {
af5ca3f4 2990 .name = "ioapic",
1da177e4
LT		 2991 .suspend = ioapic_suspend,
2992 .resume = ioapic_resume,
2993};
2994
2995static int __init ioapic_init_sysfs(void)
2996{
54168ed7
IM		 2997 struct sys_device * dev;
2998 int i, size, error;
1da177e4
LT		 2999
3000 error = sysdev_class_register(&ioapic_sysdev_class);
3001 if (error)
3002 return error;
3003
54168ed7 3004 for (i = 0; i < nr_ioapics; i++ ) {
36062448 3005 size = sizeof(struct sys_device) + nr_ioapic_registers[i]
1da177e4 3006 * sizeof(struct IO_APIC_route_entry);
25556c16 3007 mp_ioapic_data[i] = kzalloc(size, GFP_KERNEL);
1da177e4
LT		 3008 if (!mp_ioapic_data[i]) {
3009 printk(KERN_ERR "Can't suspend/resume IOAPIC %d\n", i);
3010 continue;
3011 }
1da177e4 3012 dev = &mp_ioapic_data[i]->dev;
36062448 3013 dev->id = i;
1da177e4
LT		 3014 dev->cls = &ioapic_sysdev_class;
3015 error = sysdev_register(dev);
3016 if (error) {
3017 kfree(mp_ioapic_data[i]);
3018 mp_ioapic_data[i] = NULL;
3019 printk(KERN_ERR "Can't suspend/resume IOAPIC %d\n", i);
3020 continue;
3021 }
3022 }
3023
3024 return 0;
3025}
3026
3027device_initcall(ioapic_init_sysfs);
3028
3fc471ed 3029/*
95d77884 3030 * Dynamic irq allocate and deallocation
3fc471ed 3031 */
199751d7 3032unsigned int create_irq_nr(unsigned int irq_want)
3fc471ed 3033{
ace80ab7 3034 /* Allocate an unused irq */
54168ed7
IM		 3035 unsigned int irq;
3036 unsigned int new;
3fc471ed 3037 unsigned long flags;
da51a821 3038 struct irq_cfg *cfg_new;
3fc471ed 3039
497c9a19 3040#ifndef CONFIG_HAVE_SPARSE_IRQ
199751d7 3041 irq_want = nr_irqs - 1;
497c9a19 3042#endif
199751d7
YL		 3043
3044 irq = 0;
ace80ab7 3045 spin_lock_irqsave(&vector_lock, flags);
54168ed7 3046 for (new = irq_want; new > 0; new--) {
ace80ab7
EB		 3047 if (platform_legacy_irq(new))
3048 continue;
da51a821
YL		 3049 cfg_new = irq_cfg(new);
3050 if (cfg_new && cfg_new->vector != 0)
ace80ab7 3051 continue;
047c8fdb 3052 /* check if need to create one */
da51a821
YL		 3053 if (!cfg_new)
3054 cfg_new = irq_cfg_alloc(new);
497c9a19 3055 if (__assign_irq_vector(new, TARGET_CPUS) == 0)
ace80ab7
EB		 3056 irq = new;
3057 break;
3058 }
3059 spin_unlock_irqrestore(&vector_lock, flags);
3fc471ed 3060
199751d7 3061 if (irq > 0) {
3fc471ed
EB		 3062 dynamic_irq_init(irq);
3063 }
3064 return irq;
3065}
3066
199751d7
YL		 3067int create_irq(void)
3068{
54168ed7
IM		 3069 int irq;
3070
3071 irq = create_irq_nr(nr_irqs - 1);
3072
3073 if (irq == 0)
3074 irq = -1;
3075
3076 return irq;
199751d7
YL		 3077}
3078
3fc471ed
EB		 3079void destroy_irq(unsigned int irq)
3080{
3081 unsigned long flags;
3fc471ed
EB		 3082
3083 dynamic_irq_cleanup(irq);
3084
54168ed7
IM		 3085#ifdef CONFIG_INTR_REMAP
3086 free_irte(irq);
3087#endif
3fc471ed 3088 spin_lock_irqsave(&vector_lock, flags);
497c9a19 3089 __clear_irq_vector(irq);
3fc471ed
EB		 3090 spin_unlock_irqrestore(&vector_lock, flags);
3091}
3fc471ed 3092
2d3fcc1c 3093/*
27b46d76 3094 * MSI message composition
2d3fcc1c
EB		 3095 */
3096#ifdef CONFIG_PCI_MSI
3b7d1921 3097static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_msg *msg)
2d3fcc1c 3098{
497c9a19
YL		 3099 struct irq_cfg *cfg;
3100 int err;
2d3fcc1c 3101 unsigned dest;
497c9a19 3102 cpumask_t tmp;
2d3fcc1c 3103
497c9a19
YL		 3104 tmp = TARGET_CPUS;
3105 err = assign_irq_vector(irq, tmp);
3106 if (err)
3107 return err;
2d3fcc1c 3108
497c9a19
YL		 3109 cfg = irq_cfg(irq);
3110 cpus_and(tmp, cfg->domain, tmp);
3111 dest = cpu_mask_to_apicid(tmp);
3112
54168ed7
IM		 3113#ifdef CONFIG_INTR_REMAP
3114 if (irq_remapped(irq)) {
3115 struct irte irte;
3116 int ir_index;
3117 u16 sub_handle;
3118
3119 ir_index = map_irq_to_irte_handle(irq, &sub_handle);
3120 BUG_ON(ir_index == -1);
3121
3122 memset (&irte, 0, sizeof(irte));
3123
3124 irte.present = 1;
3125 irte.dst_mode = INT_DEST_MODE;
3126 irte.trigger_mode = 0; /* edge */
3127 irte.dlvry_mode = INT_DELIVERY_MODE;
3128 irte.vector = cfg->vector;
3129 irte.dest_id = IRTE_DEST(dest);
3130
3131 modify_irte(irq, &irte);
3132
3133 msg->address_hi = MSI_ADDR_BASE_HI;
3134 msg->data = sub_handle;
3135 msg->address_lo = MSI_ADDR_BASE_LO | MSI_ADDR_IR_EXT_INT |
3136 MSI_ADDR_IR_SHV |
3137 MSI_ADDR_IR_INDEX1(ir_index) |
3138 MSI_ADDR_IR_INDEX2(ir_index);
3139 } else
3140#endif
3141 {
3142 msg->address_hi = MSI_ADDR_BASE_HI;
3143 msg->address_lo =
3144 MSI_ADDR_BASE_LO |
3145 ((INT_DEST_MODE == 0) ?
3146 MSI_ADDR_DEST_MODE_PHYSICAL:
3147 MSI_ADDR_DEST_MODE_LOGICAL) |
3148 ((INT_DELIVERY_MODE != dest_LowestPrio) ?
3149 MSI_ADDR_REDIRECTION_CPU:
3150 MSI_ADDR_REDIRECTION_LOWPRI) |
3151 MSI_ADDR_DEST_ID(dest);
497c9a19 3152
54168ed7
IM		 3153 msg->data =
3154 MSI_DATA_TRIGGER_EDGE |
3155 MSI_DATA_LEVEL_ASSERT |
3156 ((INT_DELIVERY_MODE != dest_LowestPrio) ?
3157 MSI_DATA_DELIVERY_FIXED:
3158 MSI_DATA_DELIVERY_LOWPRI) |
3159 MSI_DATA_VECTOR(cfg->vector);
3160 }
497c9a19 3161 return err;
2d3fcc1c
EB		 3162}
3163
3b7d1921
EB		 3164#ifdef CONFIG_SMP
3165static void set_msi_irq_affinity(unsigned int irq, cpumask_t mask)
2d3fcc1c 3166{
497c9a19 3167 struct irq_cfg *cfg;
3b7d1921
EB		 3168 struct msi_msg msg;
3169 unsigned int dest;
3170 cpumask_t tmp;
54168ed7 3171 struct irq_desc *desc;
3b7d1921
EB		 3172
3173 cpus_and(tmp, mask, cpu_online_map);
3174 if (cpus_empty(tmp))
497c9a19 3175 return;
2d3fcc1c 3176
497c9a19 3177 if (assign_irq_vector(irq, mask))
3b7d1921 3178 return;
2d3fcc1c 3179
497c9a19
YL		 3180 cfg = irq_cfg(irq);
3181 cpus_and(tmp, cfg->domain, mask);
3182 dest = cpu_mask_to_apicid(tmp);
3b7d1921
EB		 3183
3184 read_msi_msg(irq, &msg);
3185
3186 msg.data &= ~MSI_DATA_VECTOR_MASK;
497c9a19 3187 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3b7d1921
EB		 3188 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3189 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3190
3191 write_msi_msg(irq, &msg);
54168ed7
IM		 3192 desc = irq_to_desc(irq);
3193 desc->affinity = mask;
2d3fcc1c 3194}
54168ed7
IM		 3195
3196#ifdef CONFIG_INTR_REMAP
3197/*
3198 * Migrate the MSI irq to another cpumask. This migration is
3199 * done in the process context using interrupt-remapping hardware.
3200 */
3201static void ir_set_msi_irq_affinity(unsigned int irq, cpumask_t mask)
3202{
3203 struct irq_cfg *cfg;
3204 unsigned int dest;
3205 cpumask_t tmp, cleanup_mask;
3206 struct irte irte;
3207 struct irq_desc *desc;
3208
3209 cpus_and(tmp, mask, cpu_online_map);
3210 if (cpus_empty(tmp))
3211 return;
3212
3213 if (get_irte(irq, &irte))
3214 return;
3215
3216 if (assign_irq_vector(irq, mask))
3217 return;
3218
3219 cfg = irq_cfg(irq);
3220 cpus_and(tmp, cfg->domain, mask);
3221 dest = cpu_mask_to_apicid(tmp);
3222
3223 irte.vector = cfg->vector;
3224 irte.dest_id = IRTE_DEST(dest);
3225
3226 /*
3227 * atomically update the IRTE with the new destination and vector.
3228 */
3229 modify_irte(irq, &irte);
3230
3231 /*
3232 * After this point, all the interrupts will start arriving
3233 * at the new destination. So, time to cleanup the previous
3234 * vector allocation.
3235 */
3236 if (cfg->move_in_progress) {
3237 cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map);
3238 cfg->move_cleanup_count = cpus_weight(cleanup_mask);
3239 send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
3240 cfg->move_in_progress = 0;
3241 }
3242
3243 desc = irq_to_desc(irq);
3244 desc->affinity = mask;
3245}
3246#endif
3b7d1921 3247#endif /* CONFIG_SMP */
2d3fcc1c 3248
3b7d1921
EB		 3249/*
3250 * IRQ Chip for MSI PCI/PCI-X/PCI-Express Devices,
3251 * which implement the MSI or MSI-X Capability Structure.
3252 */
3253static struct irq_chip msi_chip = {
3254 .name = "PCI-MSI",
3255 .unmask = unmask_msi_irq,
3256 .mask = mask_msi_irq,
1d025192 3257 .ack = ack_apic_edge,
3b7d1921
EB		 3258#ifdef CONFIG_SMP
3259 .set_affinity = set_msi_irq_affinity,
3260#endif
3261 .retrigger = ioapic_retrigger_irq,
2d3fcc1c
EB		 3262};
3263
54168ed7
IM		 3264#ifdef CONFIG_INTR_REMAP
3265static struct irq_chip msi_ir_chip = {
3266 .name = "IR-PCI-MSI",
3267 .unmask = unmask_msi_irq,
3268 .mask = mask_msi_irq,
3269 .ack = ack_x2apic_edge,
3270#ifdef CONFIG_SMP
3271 .set_affinity = ir_set_msi_irq_affinity,
3272#endif
3273 .retrigger = ioapic_retrigger_irq,
3274};
3275
3276/*
3277 * Map the PCI dev to the corresponding remapping hardware unit
3278 * and allocate 'nvec' consecutive interrupt-remapping table entries
3279 * in it.
3280 */
3281static int msi_alloc_irte(struct pci_dev *dev, int irq, int nvec)
3282{
3283 struct intel_iommu *iommu;
3284 int index;
3285
3286 iommu = map_dev_to_ir(dev);
3287 if (!iommu) {
3288 printk(KERN_ERR
3289 "Unable to map PCI %s to iommu\n", pci_name(dev));
3290 return -ENOENT;
3291 }
3292
3293 index = alloc_irte(iommu, irq, nvec);
3294 if (index < 0) {
3295 printk(KERN_ERR
3296 "Unable to allocate %d IRTE for PCI %s\n", nvec,
3297 pci_name(dev));
3298 return -ENOSPC;
3299 }
3300 return index;
3301}
3302#endif
1d025192
YL		 3303
3304static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc, int irq)
3305{
3306 int ret;
3307 struct msi_msg msg;
3308
3309 ret = msi_compose_msg(dev, irq, &msg);
3310 if (ret < 0)
3311 return ret;
3312
3313 set_irq_msi(irq, desc);
3314 write_msi_msg(irq, &msg);
3315
54168ed7
IM		 3316#ifdef CONFIG_INTR_REMAP
3317 if (irq_remapped(irq)) {
3318 struct irq_desc *desc = irq_to_desc(irq);
3319 /*
3320 * irq migration in process context
3321 */
3322 desc->status |= IRQ_MOVE_PCNTXT;
3323 set_irq_chip_and_handler_name(irq, &msi_ir_chip, handle_edge_irq, "edge");
3324 } else
3325#endif
3326 set_irq_chip_and_handler_name(irq, &msi_chip, handle_edge_irq, "edge");
1d025192
YL		 3327
3328 return 0;
3329}
3330
199751d7
YL		 3331static unsigned int build_irq_for_pci_dev(struct pci_dev *dev)
3332{
3333 unsigned int irq;
3334
3335 irq = dev->bus->number;
3336 irq <<= 8;
3337 irq |= dev->devfn;
3338 irq <<= 12;
3339
3340 return irq;
3341}
3342
f7feaca7 3343int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc)
3b7d1921 3344{
54168ed7
IM		 3345 unsigned int irq;
3346 int ret;
199751d7
YL		 3347 unsigned int irq_want;
3348
3349 irq_want = build_irq_for_pci_dev(dev) + 0x100;
3350
3351 irq = create_irq_nr(irq_want);
199751d7
YL		 3352 if (irq == 0)
3353 return -1;
f7feaca7 3354
54168ed7
IM		 3355#ifdef CONFIG_INTR_REMAP
3356 if (!intr_remapping_enabled)
3357 goto no_ir;
3358
3359 ret = msi_alloc_irte(dev, irq, 1);
3360 if (ret < 0)
3361 goto error;
3362no_ir:
3363#endif
1d025192 3364 ret = setup_msi_irq(dev, desc, irq);
f7feaca7
EB		 3365 if (ret < 0) {
3366 destroy_irq(irq);
3b7d1921 3367 return ret;
54168ed7 3368 }
7fe3730d 3369 return 0;
54168ed7
IM		 3370
3371#ifdef CONFIG_INTR_REMAP
3372error:
3373 destroy_irq(irq);
3374 return ret;
3375#endif
3b7d1921
EB		 3376}
3377
047c8fdb
YL		 3378int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
3379{
54168ed7
IM		 3380 unsigned int irq;
3381 int ret, sub_handle;
3382 struct msi_desc *desc;
3383 unsigned int irq_want;
3384
3385#ifdef CONFIG_INTR_REMAP
3386 struct intel_iommu *iommu = 0;
3387 int index = 0;
3388#endif
3389
3390 irq_want = build_irq_for_pci_dev(dev) + 0x100;
3391 sub_handle = 0;
3392 list_for_each_entry(desc, &dev->msi_list, list) {
3393 irq = create_irq_nr(irq_want--);
3394 if (irq == 0)
3395 return -1;
3396#ifdef CONFIG_INTR_REMAP
3397 if (!intr_remapping_enabled)
3398 goto no_ir;
3399
3400 if (!sub_handle) {
3401 /*
3402 * allocate the consecutive block of IRTE's
3403 * for 'nvec'
3404 */
3405 index = msi_alloc_irte(dev, irq, nvec);
3406 if (index < 0) {
3407 ret = index;
3408 goto error;
3409 }
3410 } else {
3411 iommu = map_dev_to_ir(dev);
3412 if (!iommu) {
3413 ret = -ENOENT;
3414 goto error;
3415 }
3416 /*
3417 * setup the mapping between the irq and the IRTE
3418 * base index, the sub_handle pointing to the
3419 * appropriate interrupt remap table entry.
3420 */
3421 set_irte_irq(irq, iommu, index, sub_handle);
3422 }
3423no_ir:
3424#endif
3425 ret = setup_msi_irq(dev, desc, irq);
3426 if (ret < 0)
3427 goto error;
3428 sub_handle++;
3429 }
3430 return 0;
047c8fdb
YL		 3431
3432error:
54168ed7
IM		 3433 destroy_irq(irq);
3434 return ret;
047c8fdb
YL		 3435}
3436
3b7d1921
EB		 3437void arch_teardown_msi_irq(unsigned int irq)
3438{
f7feaca7 3439 destroy_irq(irq);
3b7d1921
EB		 3440}
3441
54168ed7
IM		 3442#ifdef CONFIG_DMAR
3443#ifdef CONFIG_SMP
3444static void dmar_msi_set_affinity(unsigned int irq, cpumask_t mask)
3445{
3446 struct irq_cfg *cfg;
3447 struct msi_msg msg;
3448 unsigned int dest;
3449 cpumask_t tmp;
3450 struct irq_desc *desc;
3451
3452 cpus_and(tmp, mask, cpu_online_map);
3453 if (cpus_empty(tmp))
3454 return;
3455
3456 if (assign_irq_vector(irq, mask))
3457 return;
3458
3459 cfg = irq_cfg(irq);
3460 cpus_and(tmp, cfg->domain, mask);
3461 dest = cpu_mask_to_apicid(tmp);
3462
3463 dmar_msi_read(irq, &msg);
3464
3465 msg.data &= ~MSI_DATA_VECTOR_MASK;
3466 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3467 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3468 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3469
3470 dmar_msi_write(irq, &msg);
3471 desc = irq_to_desc(irq);
3472 desc->affinity = mask;
3473}
3474#endif /* CONFIG_SMP */
3475
3476struct irq_chip dmar_msi_type = {
3477 .name = "DMAR_MSI",
3478 .unmask = dmar_msi_unmask,
3479 .mask = dmar_msi_mask,
3480 .ack = ack_apic_edge,
3481#ifdef CONFIG_SMP
3482 .set_affinity = dmar_msi_set_affinity,
3483#endif
3484 .retrigger = ioapic_retrigger_irq,
3485};
3486
3487int arch_setup_dmar_msi(unsigned int irq)
3488{
3489 int ret;
3490 struct msi_msg msg;
2d3fcc1c 3491
54168ed7
IM		 3492 ret = msi_compose_msg(NULL, irq, &msg);
3493 if (ret < 0)
3494 return ret;
3495 dmar_msi_write(irq, &msg);
3496 set_irq_chip_and_handler_name(irq, &dmar_msi_type, handle_edge_irq,
3497 "edge");
3498 return 0;
3499}
3500#endif
3501
3502#endif /* CONFIG_PCI_MSI */
8b955b0d
EB		 3503/*
3504 * Hypertransport interrupt support
3505 */
3506#ifdef CONFIG_HT_IRQ
3507
3508#ifdef CONFIG_SMP
3509
497c9a19 3510static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector)
8b955b0d 3511{
ec68307c
EB		 3512 struct ht_irq_msg msg;
3513 fetch_ht_irq_msg(irq, &msg);
8b955b0d 3514
497c9a19 3515 msg.address_lo &= ~(HT_IRQ_LOW_VECTOR_MASK | HT_IRQ_LOW_DEST_ID_MASK);
ec68307c 3516 msg.address_hi &= ~(HT_IRQ_HIGH_DEST_ID_MASK);
8b955b0d 3517
497c9a19 3518 msg.address_lo |= HT_IRQ_LOW_VECTOR(vector) | HT_IRQ_LOW_DEST_ID(dest);
ec68307c 3519 msg.address_hi |= HT_IRQ_HIGH_DEST_ID(dest);
8b955b0d 3520
ec68307c 3521 write_ht_irq_msg(irq, &msg);
8b955b0d
EB		 3522}
3523
3524static void set_ht_irq_affinity(unsigned int irq, cpumask_t mask)
3525{
497c9a19 3526 struct irq_cfg *cfg;
8b955b0d
EB		 3527 unsigned int dest;
3528 cpumask_t tmp;
54168ed7 3529 struct irq_desc *desc;
8b955b0d
EB		 3530
3531 cpus_and(tmp, mask, cpu_online_map);
3532 if (cpus_empty(tmp))
497c9a19 3533 return;
8b955b0d 3534
497c9a19
YL		 3535 if (assign_irq_vector(irq, mask))
3536 return;
8b955b0d 3537
497c9a19
YL		 3538 cfg = irq_cfg(irq);
3539 cpus_and(tmp, cfg->domain, mask);
3540 dest = cpu_mask_to_apicid(tmp);
8b955b0d 3541
497c9a19 3542 target_ht_irq(irq, dest, cfg->vector);
54168ed7
IM		 3543 desc = irq_to_desc(irq);
3544 desc->affinity = mask;
8b955b0d
EB		 3545}
3546#endif
3547
c37e108d 3548static struct irq_chip ht_irq_chip = {
8b955b0d
EB		 3549 .name = "PCI-HT",
3550 .mask = mask_ht_irq,
3551 .unmask = unmask_ht_irq,
1d025192 3552 .ack = ack_apic_edge,
8b955b0d
EB		 3553#ifdef CONFIG_SMP
3554 .set_affinity = set_ht_irq_affinity,
3555#endif
3556 .retrigger = ioapic_retrigger_irq,
3557};
3558
3559int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
3560{
497c9a19
YL		 3561 struct irq_cfg *cfg;
3562 int err;
3563 cpumask_t tmp;
8b955b0d 3564
497c9a19
YL		 3565 tmp = TARGET_CPUS;
3566 err = assign_irq_vector(irq, tmp);
54168ed7 3567 if (!err) {
ec68307c 3568 struct ht_irq_msg msg;
8b955b0d 3569 unsigned dest;
8b955b0d 3570
497c9a19
YL		 3571 cfg = irq_cfg(irq);
3572 cpus_and(tmp, cfg->domain, tmp);
8b955b0d
EB		 3573 dest = cpu_mask_to_apicid(tmp);
3574
ec68307c 3575 msg.address_hi = HT_IRQ_HIGH_DEST_ID(dest);
8b955b0d 3576
ec68307c
EB		 3577 msg.address_lo =
3578 HT_IRQ_LOW_BASE |
8b955b0d 3579 HT_IRQ_LOW_DEST_ID(dest) |
497c9a19 3580 HT_IRQ_LOW_VECTOR(cfg->vector) |
8b955b0d
EB		 3581 ((INT_DEST_MODE == 0) ?
3582 HT_IRQ_LOW_DM_PHYSICAL :
3583 HT_IRQ_LOW_DM_LOGICAL) |
3584 HT_IRQ_LOW_RQEOI_EDGE |
3585 ((INT_DELIVERY_MODE != dest_LowestPrio) ?
3586 HT_IRQ_LOW_MT_FIXED :
3587 HT_IRQ_LOW_MT_ARBITRATED) |
3588 HT_IRQ_LOW_IRQ_MASKED;
3589
ec68307c 3590 write_ht_irq_msg(irq, &msg);
8b955b0d 3591
a460e745
IM		 3592 set_irq_chip_and_handler_name(irq, &ht_irq_chip,
3593 handle_edge_irq, "edge");
8b955b0d 3594 }
497c9a19 3595 return err;
8b955b0d
EB		 3596}
3597#endif /* CONFIG_HT_IRQ */
3598
9d6a4d08
YL		 3599int __init io_apic_get_redir_entries (int ioapic)
3600{
3601 union IO_APIC_reg_01 reg_01;
3602 unsigned long flags;
3603
3604 spin_lock_irqsave(&ioapic_lock, flags);
3605 reg_01.raw = io_apic_read(ioapic, 1);
3606 spin_unlock_irqrestore(&ioapic_lock, flags);
3607
3608 return reg_01.bits.entries;
3609}
3610
3611int __init probe_nr_irqs(void)
3612{
3613 int idx;
3614 int nr = 0;
3615
3616 for (idx = 0; idx < nr_ioapics; idx++)
3617 nr += io_apic_get_redir_entries(idx);
3618
3619 /* double it for hotplug and msi and nmi */
3620 nr <<= 1;
3621
3622 /* something wrong ? */
3623 if (nr < 32)
3624 nr = 32;
3625
3626 return nr;
3627}
3628
1da177e4 3629/* --------------------------------------------------------------------------
54168ed7 3630 ACPI-based IOAPIC Configuration
1da177e4
LT		 3631 -------------------------------------------------------------------------- */
3632
888ba6c6 3633#ifdef CONFIG_ACPI
1da177e4 3634
54168ed7 3635#ifdef CONFIG_X86_32
36062448 3636int __init io_apic_get_unique_id(int ioapic, int apic_id)
1da177e4
LT		 3637{
3638 union IO_APIC_reg_00 reg_00;
3639 static physid_mask_t apic_id_map = PHYSID_MASK_NONE;
3640 physid_mask_t tmp;
3641 unsigned long flags;
3642 int i = 0;
3643
3644 /*
36062448
PC		 3645 * The P4 platform supports up to 256 APIC IDs on two separate APIC
3646 * buses (one for LAPICs, one for IOAPICs), where predecessors only
1da177e4 3647 * supports up to 16 on one shared APIC bus.
36062448 3648 *
1da177e4
LT		 3649 * TBD: Expand LAPIC/IOAPIC support on P4-class systems to take full
3650 * advantage of new APIC bus architecture.
3651 */
3652
3653 if (physids_empty(apic_id_map))
3654 apic_id_map = ioapic_phys_id_map(phys_cpu_present_map);
3655
3656 spin_lock_irqsave(&ioapic_lock, flags);
3657 reg_00.raw = io_apic_read(ioapic, 0);
3658 spin_unlock_irqrestore(&ioapic_lock, flags);
3659
3660 if (apic_id >= get_physical_broadcast()) {
3661 printk(KERN_WARNING "IOAPIC[%d]: Invalid apic_id %d, trying "
3662 "%d\n", ioapic, apic_id, reg_00.bits.ID);
3663 apic_id = reg_00.bits.ID;
3664 }
3665
3666 /*
36062448 3667 * Every APIC in a system must have a unique ID or we get lots of nice
1da177e4
LT		 3668 * 'stuck on smp_invalidate_needed IPI wait' messages.
3669 */
3670 if (check_apicid_used(apic_id_map, apic_id)) {
3671
3672 for (i = 0; i < get_physical_broadcast(); i++) {
3673 if (!check_apicid_used(apic_id_map, i))
3674 break;
3675 }
3676
3677 if (i == get_physical_broadcast())
3678 panic("Max apic_id exceeded!\n");
3679
3680 printk(KERN_WARNING "IOAPIC[%d]: apic_id %d already used, "
3681 "trying %d\n", ioapic, apic_id, i);
3682
3683 apic_id = i;
36062448 3684 }
1da177e4
LT		 3685
3686 tmp = apicid_to_cpu_present(apic_id);
3687 physids_or(apic_id_map, apic_id_map, tmp);
3688
3689 if (reg_00.bits.ID != apic_id) {
3690 reg_00.bits.ID = apic_id;
3691
3692 spin_lock_irqsave(&ioapic_lock, flags);
3693 io_apic_write(ioapic, 0, reg_00.raw);
3694 reg_00.raw = io_apic_read(ioapic, 0);
3695 spin_unlock_irqrestore(&ioapic_lock, flags);
3696
3697 /* Sanity check */
6070f9ec
AD		 3698 if (reg_00.bits.ID != apic_id) {
3699 printk("IOAPIC[%d]: Unable to change apic_id!\n", ioapic);
3700 return -1;
3701 }
1da177e4
LT		 3702 }
3703
3704 apic_printk(APIC_VERBOSE, KERN_INFO
3705 "IOAPIC[%d]: Assigned apic_id %d\n", ioapic, apic_id);
3706
3707 return apic_id;
3708}
3709
36062448 3710int __init io_apic_get_version(int ioapic)
1da177e4
LT		 3711{
3712 union IO_APIC_reg_01 reg_01;
3713 unsigned long flags;
3714
3715 spin_lock_irqsave(&ioapic_lock, flags);
3716 reg_01.raw = io_apic_read(ioapic, 1);
3717 spin_unlock_irqrestore(&ioapic_lock, flags);
3718
3719 return reg_01.bits.version;
3720}
54168ed7 3721#endif
1da177e4 3722
54168ed7 3723int io_apic_set_pci_routing (int ioapic, int pin, int irq, int triggering, int polarity)
1da177e4 3724{
1da177e4 3725 if (!IO_APIC_IRQ(irq)) {
54168ed7 3726 apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n",
1da177e4
LT		 3727 ioapic);
3728 return -EINVAL;
3729 }
3730
1da177e4
LT		 3731 /*
3732 * IRQs < 16 are already in the irq_2_pin[] map
3733 */
3734 if (irq >= 16)
3735 add_pin_to_irq(irq, ioapic, pin);
3736
497c9a19 3737 setup_IO_APIC_irq(ioapic, pin, irq, triggering, polarity);
1da177e4
LT		 3738
3739 return 0;
3740}
3741
54168ed7 3742
61fd47e0
SL		 3743int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity)
3744{
3745 int i;
3746
3747 if (skip_ioapic_setup)
3748 return -1;
3749
3750 for (i = 0; i < mp_irq_entries; i++)
2fddb6e2
AS		 3751 if (mp_irqs[i].mp_irqtype == mp_INT &&
3752 mp_irqs[i].mp_srcbusirq == bus_irq)
61fd47e0
SL		 3753 break;
3754 if (i >= mp_irq_entries)
3755 return -1;
3756
3757 *trigger = irq_trigger(i);
3758 *polarity = irq_polarity(i);
3759 return 0;
3760}
3761
888ba6c6 3762#endif /* CONFIG_ACPI */
1a3f239d 3763
497c9a19
YL		 3764/*
3765 * This function currently is only a helper for the i386 smp boot process where
3766 * we need to reprogram the ioredtbls to cater for the cpus which have come online
3767 * so mask in all cases should simply be TARGET_CPUS
3768 */
3769#ifdef CONFIG_SMP
3770void __init setup_ioapic_dest(void)
3771{
3772 int pin, ioapic, irq, irq_entry;
3773 struct irq_cfg *cfg;
497c9a19
YL		 3774
3775 if (skip_ioapic_setup == 1)
3776 return;
3777
3778 for (ioapic = 0; ioapic < nr_ioapics; ioapic++) {
3779 for (pin = 0; pin < nr_ioapic_registers[ioapic]; pin++) {
3780 irq_entry = find_irq_entry(ioapic, pin, mp_INT);
3781 if (irq_entry == -1)
3782 continue;
3783 irq = pin_2_irq(irq_entry, ioapic, pin);
3784
3785 /* setup_IO_APIC_irqs could fail to get vector for some device
3786 * when you have too many devices, because at that time only boot
3787 * cpu is online.
3788 */
3789 cfg = irq_cfg(irq);
3790 if (!cfg->vector)
3791 setup_IO_APIC_irq(ioapic, pin, irq,
3792 irq_trigger(irq_entry),
3793 irq_polarity(irq_entry));
54168ed7
IM		 3794#ifdef CONFIG_INTR_REMAP
3795 else if (intr_remapping_enabled)
3796 set_ir_ioapic_affinity_irq(irq, TARGET_CPUS);
3797#endif
3798 else
497c9a19 3799 set_ioapic_affinity_irq(irq, TARGET_CPUS);
497c9a19
YL		 3800 }
3801
3802 }
3803}
3804#endif
3805
54168ed7
IM		 3806#define IOAPIC_RESOURCE_NAME_SIZE 11
3807
3808static struct resource *ioapic_resources;
3809
3810static struct resource * __init ioapic_setup_resources(void)
3811{
3812 unsigned long n;
3813 struct resource *res;
3814 char *mem;
3815 int i;
3816
3817 if (nr_ioapics <= 0)
3818 return NULL;
3819
3820 n = IOAPIC_RESOURCE_NAME_SIZE + sizeof(struct resource);
3821 n *= nr_ioapics;
3822
3823 mem = alloc_bootmem(n);
3824 res = (void *)mem;
3825
3826 if (mem != NULL) {
3827 mem += sizeof(struct resource) * nr_ioapics;
3828
3829 for (i = 0; i < nr_ioapics; i++) {
3830 res[i].name = mem;
3831 res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY;
3832 sprintf(mem, "IOAPIC %u", i);
3833 mem += IOAPIC_RESOURCE_NAME_SIZE;
3834 }
3835 }
3836
3837 ioapic_resources = res;
3838
3839 return res;
3840}
54168ed7 3841
f3294a33
YL		 3842void __init ioapic_init_mappings(void)
3843{
3844 unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0;
3845 int i;
54168ed7 3846 struct resource *ioapic_res;
f3294a33 3847
54168ed7 3848 ioapic_res = ioapic_setup_resources();
f3294a33
YL		 3849 for (i = 0; i < nr_ioapics; i++) {
3850 if (smp_found_config) {
3851 ioapic_phys = mp_ioapics[i].mp_apicaddr;
54168ed7
IM		 3852#ifdef CONFIG_X86_32
3853 if (!ioapic_phys) {
3854 printk(KERN_ERR
3855 "WARNING: bogus zero IO-APIC "
3856 "address found in MPTABLE, "
3857 "disabling IO/APIC support!\n");
3858 smp_found_config = 0;
3859 skip_ioapic_setup = 1;
3860 goto fake_ioapic_page;
3861 }
3862#endif
f3294a33 3863 } else {
54168ed7 3864#ifdef CONFIG_X86_32
f3294a33 3865fake_ioapic_page:
54168ed7 3866#endif
f3294a33 3867 ioapic_phys = (unsigned long)
54168ed7 3868 alloc_bootmem_pages(PAGE_SIZE);
f3294a33
YL		 3869 ioapic_phys = __pa(ioapic_phys);
3870 }
3871 set_fixmap_nocache(idx, ioapic_phys);
54168ed7
IM		 3872 apic_printk(APIC_VERBOSE,
3873 "mapped IOAPIC to %08lx (%08lx)\n",
3874 __fix_to_virt(idx), ioapic_phys);
f3294a33 3875 idx++;
54168ed7 3876
54168ed7
IM		 3877 if (ioapic_res != NULL) {
3878 ioapic_res->start = ioapic_phys;
3879 ioapic_res->end = ioapic_phys + (4 * 1024) - 1;
3880 ioapic_res++;
3881 }
f3294a33
YL		 3882 }
3883}
3884
54168ed7
IM		 3885static int __init ioapic_insert_resources(void)
3886{
3887 int i;
3888 struct resource *r = ioapic_resources;
3889
3890 if (!r) {
3891 printk(KERN_ERR
3892 "IO APIC resources could be not be allocated.\n");
3893 return -1;
3894 }
3895
3896 for (i = 0; i < nr_ioapics; i++) {
3897 insert_resource(&iomem_resource, r);
3898 r++;
3899 }
3900
3901 return 0;
3902}
3903
3904/* Insert the IO APIC resources after PCI initialization has occured to handle
3905 * IO APICS that are mapped in on a BAR in PCI space. */
3906late_initcall(ioapic_insert_resources);
Clone of davem's net-next-2.6 tree