]> bbs.cooldavid.org Git - net-next-2.6.git/blame - arch/x86/kernel/io_apic.c
git://bbs.cooldavid.org / net-next-2.6.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
x86: fix HPET compiler error when not using CONFIG_PCI_MSI
[net-next-2.6.git] / arch / x86 / kernel / io_apic.c
CommitLineData
1da177e4
LT		 1/*
2 * Intel IO-APIC support for multi-Pentium hosts.
3 *
4 * Copyright (C) 1997, 1998, 1999, 2000 Ingo Molnar, Hajnalka Szabo
5 *
6 * Many thanks to Stig Venaas for trying out countless experimental
7 * patches and reporting/debugging problems patiently!
8 *
9 * (c) 1999, Multiple IO-APIC support, developed by
10 * Ken-ichi Yaku <yaku@css1.kbnes.nec.co.jp> and
11 * Hidemi Kishimoto <kisimoto@css1.kbnes.nec.co.jp>,
12 * further tested and cleaned up by Zach Brown <zab@redhat.com>
13 * and Ingo Molnar <mingo@redhat.com>
14 *
15 * Fixes
16 * Maciej W. Rozycki : Bits for genuine 82489DX APICs;
17 * thanks to Eric Gilmore
18 * and Rolf G. Tews
19 * for testing these extensively
20 * Paul Diefenbaugh : Added full ACPI support
21 */
22
23#include <linux/mm.h>
1da177e4
LT		 24#include <linux/interrupt.h>
25#include <linux/init.h>
26#include <linux/delay.h>
27#include <linux/sched.h>
d4057bdb 28#include <linux/pci.h>
1da177e4
LT		 29#include <linux/mc146818rtc.h>
30#include <linux/compiler.h>
31#include <linux/acpi.h>
129f6946 32#include <linux/module.h>
1da177e4 33#include <linux/sysdev.h>
3b7d1921 34#include <linux/msi.h>
95d77884 35#include <linux/htirq.h>
7dfb7103 36#include <linux/freezer.h>
f26d6a2b 37#include <linux/kthread.h>
54168ed7 38#include <linux/jiffies.h> /* time_after() */
d4057bdb
YL		 39#ifdef CONFIG_ACPI
40#include <acpi/acpi_bus.h>
41#endif
42#include <linux/bootmem.h>
43#include <linux/dmar.h>
58ac1e76 44#include <linux/hpet.h>
54d5d424 45
d4057bdb 46#include <asm/idle.h>
1da177e4
LT		 47#include <asm/io.h>
48#include <asm/smp.h>
49#include <asm/desc.h>
d4057bdb
YL		 50#include <asm/proto.h>
51#include <asm/acpi.h>
52#include <asm/dma.h>
1da177e4 53#include <asm/timer.h>
306e440d 54#include <asm/i8259.h>
3e4ff115 55#include <asm/nmi.h>
2d3fcc1c 56#include <asm/msidef.h>
8b955b0d 57#include <asm/hypertransport.h>
a4dbc34d 58#include <asm/setup.h>
d4057bdb 59#include <asm/irq_remapping.h>
58ac1e76 60#include <asm/hpet.h>
1da177e4 61
497c9a19 62#include <mach_ipi.h>
1da177e4 63#include <mach_apic.h>
874c4fe3 64#include <mach_apicdef.h>
1da177e4 65
32f71aff
MR		 66#define __apicdebuginit(type) static type __init
67
1da177e4 68/*
54168ed7
IM		 69 * Is the SiS APIC rmw bug present ?
70 * -1 = don't know, 0 = no, 1 = yes
1da177e4
LT		 71 */
72int sis_apic_bug = -1;
73
efa2559f
YL		 74static DEFINE_SPINLOCK(ioapic_lock);
75static DEFINE_SPINLOCK(vector_lock);
76
1da177e4
LT		 77/*
78 * # of IRQ routing registers
79 */
80int nr_ioapic_registers[MAX_IO_APICS];
81
9f640ccb 82/* I/O APIC entries */
ec2cd0a2 83struct mp_config_ioapic mp_ioapics[MAX_IO_APICS];
9f640ccb
AS		 84int nr_ioapics;
85
584f734d 86/* MP IRQ source entries */
2fddb6e2 87struct mp_config_intsrc mp_irqs[MAX_IRQ_SOURCES];
584f734d
AS		 88
89/* # of MP IRQ source entries */
90int mp_irq_entries;
91
8732fc4b
AS		 92#if defined (CONFIG_MCA) || defined (CONFIG_EISA)
93int mp_bus_id_to_type[MAX_MP_BUSSES];
94#endif
95
96DECLARE_BITMAP(mp_bus_not_pci, MAX_MP_BUSSES);
97
efa2559f
YL		 98int skip_ioapic_setup;
99
54168ed7 100static int __init parse_noapic(char *str)
efa2559f
YL		 101{
102 /* disable IO-APIC */
103 disable_ioapic_setup();
104 return 0;
105}
106early_param("noapic", parse_noapic);
66759a01 107
da51a821 108struct irq_cfg;
0f978f45 109struct irq_pin_list;
a1420f39 110struct irq_cfg {
da51a821 111 unsigned int irq;
8f09cd20 112#ifdef CONFIG_HAVE_SPARSE_IRQ
da51a821 113 struct irq_cfg *next;
8f09cd20 114#endif
0f978f45 115 struct irq_pin_list *irq_2_pin;
497c9a19
YL		 116 cpumask_t domain;
117 cpumask_t old_domain;
118 unsigned move_cleanup_count;
a1420f39 119 u8 vector;
497c9a19 120 u8 move_in_progress : 1;
a1420f39
YL		 121};
122
a1420f39
YL		 123/* irq_cfg is indexed by the sum of all RTEs in all I/O APICs. */
124static struct irq_cfg irq_cfg_legacy[] __initdata = {
497c9a19
YL		 125 [0] = { .irq = 0, .domain = CPU_MASK_ALL, .vector = IRQ0_VECTOR, },
126 [1] = { .irq = 1, .domain = CPU_MASK_ALL, .vector = IRQ1_VECTOR, },
127 [2] = { .irq = 2, .domain = CPU_MASK_ALL, .vector = IRQ2_VECTOR, },
128 [3] = { .irq = 3, .domain = CPU_MASK_ALL, .vector = IRQ3_VECTOR, },
129 [4] = { .irq = 4, .domain = CPU_MASK_ALL, .vector = IRQ4_VECTOR, },
130 [5] = { .irq = 5, .domain = CPU_MASK_ALL, .vector = IRQ5_VECTOR, },
131 [6] = { .irq = 6, .domain = CPU_MASK_ALL, .vector = IRQ6_VECTOR, },
132 [7] = { .irq = 7, .domain = CPU_MASK_ALL, .vector = IRQ7_VECTOR, },
133 [8] = { .irq = 8, .domain = CPU_MASK_ALL, .vector = IRQ8_VECTOR, },
134 [9] = { .irq = 9, .domain = CPU_MASK_ALL, .vector = IRQ9_VECTOR, },
135 [10] = { .irq = 10, .domain = CPU_MASK_ALL, .vector = IRQ10_VECTOR, },
136 [11] = { .irq = 11, .domain = CPU_MASK_ALL, .vector = IRQ11_VECTOR, },
137 [12] = { .irq = 12, .domain = CPU_MASK_ALL, .vector = IRQ12_VECTOR, },
138 [13] = { .irq = 13, .domain = CPU_MASK_ALL, .vector = IRQ13_VECTOR, },
139 [14] = { .irq = 14, .domain = CPU_MASK_ALL, .vector = IRQ14_VECTOR, },
140 [15] = { .irq = 15, .domain = CPU_MASK_ALL, .vector = IRQ15_VECTOR, },
a1420f39
YL		 141};
142
da51a821 143static struct irq_cfg irq_cfg_init = { .irq = -1U, };
da51a821
YL		 144
145static void init_one_irq_cfg(struct irq_cfg *cfg)
146{
147 memcpy(cfg, &irq_cfg_init, sizeof(struct irq_cfg));
148}
149
150static struct irq_cfg *irq_cfgx;
e89eb438 151
ac54a6c9 152#ifdef CONFIG_HAVE_SPARSE_IRQ
e89eb438
YL		 153/*
154 * Protect the irq_cfgx_free freelist:
155 */
156static DEFINE_SPINLOCK(irq_cfg_lock);
157
da51a821 158static struct irq_cfg *irq_cfgx_free;
8f09cd20 159#endif
ac54a6c9 160
a1420f39
YL		 161static void __init init_work(void *data)
162{
da51a821
YL		 163 struct dyn_array *da = data;
164 struct irq_cfg *cfg;
165 int legacy_count;
166 int i;
167
168 cfg = *da->name;
a1420f39 169
da51a821 170 memcpy(cfg, irq_cfg_legacy, sizeof(irq_cfg_legacy));
a1420f39 171
676f4a92 172 legacy_count = ARRAY_SIZE(irq_cfg_legacy);
da51a821
YL		 173 for (i = legacy_count; i < *da->nr; i++)
174 init_one_irq_cfg(&cfg[i]);
a1420f39 175
8f09cd20 176#ifdef CONFIG_HAVE_SPARSE_IRQ
da51a821
YL		 177 for (i = 1; i < *da->nr; i++)
178 cfg[i-1].next = &cfg[i];
a1420f39 179
da51a821
YL		 180 irq_cfgx_free = &irq_cfgx[legacy_count];
181 irq_cfgx[legacy_count - 1].next = NULL;
8f09cd20
YL		 182#endif
183}
184
185#ifdef CONFIG_HAVE_SPARSE_IRQ
186/* need to be biger than size of irq_cfg_legacy */
187static int nr_irq_cfg = 32;
188
189static int __init parse_nr_irq_cfg(char *arg)
190{
191 if (arg) {
192 nr_irq_cfg = simple_strtoul(arg, NULL, 0);
193 if (nr_irq_cfg < 32)
194 nr_irq_cfg = 32;
195 }
196 return 0;
a1420f39
YL		 197}
198
8f09cd20
YL		 199early_param("nr_irq_cfg", parse_nr_irq_cfg);
200
201#define for_each_irq_cfg(irqX, cfg) \
202 for (cfg = irq_cfgx, irqX = cfg->irq; cfg; cfg = cfg->next, irqX = cfg ? cfg->irq : -1U)
203
da51a821
YL		 204
205DEFINE_DYN_ARRAY(irq_cfgx, sizeof(struct irq_cfg), nr_irq_cfg, PAGE_SIZE, init_work);
a1420f39
YL		 206
207static struct irq_cfg *irq_cfg(unsigned int irq)
208{
da51a821
YL		 209 struct irq_cfg *cfg;
210
211 cfg = irq_cfgx;
212 while (cfg) {
213 if (cfg->irq == irq)
214 return cfg;
215
216 cfg = cfg->next;
217 }
218
219 return NULL;
220}
221
222static struct irq_cfg *irq_cfg_alloc(unsigned int irq)
223{
224 struct irq_cfg *cfg, *cfg_pri;
e89eb438 225 unsigned long flags;
da51a821 226 int count = 0;
e89eb438 227 int i;
da51a821
YL		 228
229 cfg_pri = cfg = irq_cfgx;
230 while (cfg) {
231 if (cfg->irq == irq)
232 return cfg;
233
234 cfg_pri = cfg;
235 cfg = cfg->next;
236 count++;
237 }
238
e89eb438 239 spin_lock_irqsave(&irq_cfg_lock, flags);
da51a821
YL		 240 if (!irq_cfgx_free) {
241 unsigned long phys;
242 unsigned long total_bytes;
243 /*
244 * we run out of pre-allocate ones, allocate more
245 */
246 printk(KERN_DEBUG "try to get more irq_cfg %d\n", nr_irq_cfg);
247
248 total_bytes = sizeof(struct irq_cfg) * nr_irq_cfg;
249 if (after_bootmem)
250 cfg = kzalloc(total_bytes, GFP_ATOMIC);
251 else
252 cfg = __alloc_bootmem_nopanic(total_bytes, PAGE_SIZE, 0);
a1420f39 253
da51a821
YL		 254 if (!cfg)
255 panic("please boot with nr_irq_cfg= %d\n", count * 2);
256
257 phys = __pa(cfg);
258 printk(KERN_DEBUG "irq_irq ==> [%#lx - %#lx]\n", phys, phys + total_bytes);
259
260 for (i = 0; i < nr_irq_cfg; i++)
261 init_one_irq_cfg(&cfg[i]);
262
263 for (i = 1; i < nr_irq_cfg; i++)
264 cfg[i-1].next = &cfg[i];
265
266 irq_cfgx_free = cfg;
267 }
268
269 cfg = irq_cfgx_free;
270 irq_cfgx_free = irq_cfgx_free->next;
271 cfg->next = NULL;
272 if (cfg_pri)
273 cfg_pri->next = cfg;
274 else
275 irq_cfgx = cfg;
276 cfg->irq = irq;
e89eb438
YL		 277
278 spin_unlock_irqrestore(&irq_cfg_lock, flags);
279
da51a821 280 printk(KERN_DEBUG "found new irq_cfg for irq %d\n", cfg->irq);
da51a821
YL		 281#ifdef CONFIG_HAVE_SPARSE_IRQ_DEBUG
282 {
283 /* dump the results */
284 struct irq_cfg *cfg;
285 unsigned long phys;
286 unsigned long bytes = sizeof(struct irq_cfg);
287
288 printk(KERN_DEBUG "=========================== %d\n", irq);
289 printk(KERN_DEBUG "irq_cfg dump after get that for %d\n", irq);
290 for_each_irq_cfg(cfg) {
291 phys = __pa(cfg);
292 printk(KERN_DEBUG "irq_cfg %d ==> [%#lx - %#lx]\n", cfg->irq, phys, phys + bytes);
293 }
294 printk(KERN_DEBUG "===========================\n");
295 }
296#endif
297 return cfg;
a1420f39 298}
8f09cd20
YL		 299#else
300
301#define for_each_irq_cfg(irq, cfg) \
302 for (irq = 0, cfg = &irq_cfgx[irq]; irq < nr_irqs; irq++, cfg = &irq_cfgx[irq])
303
304DEFINE_DYN_ARRAY(irq_cfgx, sizeof(struct irq_cfg), nr_irqs, PAGE_SIZE, init_work);
a1420f39 305
8f09cd20
YL		 306struct irq_cfg *irq_cfg(unsigned int irq)
307{
308 if (irq < nr_irqs)
309 return &irq_cfgx[irq];
310
311 return NULL;
312}
313struct irq_cfg *irq_cfg_alloc(unsigned int irq)
314{
315 return irq_cfg(irq);
316}
317
318#endif
1da177e4
LT		 319/*
320 * This is performance-critical, we want to do it O(1)
321 *
322 * the indexing order of this array favors 1:1 mappings
323 * between pins and IRQs.
324 */
325
0f978f45
YL		 326struct irq_pin_list {
327 int apic, pin;
328 struct irq_pin_list *next;
329};
330
331static struct irq_pin_list *irq_2_pin_head;
332/* fill one page ? */
333static int nr_irq_2_pin = 0x100;
334static struct irq_pin_list *irq_2_pin_ptr;
335static void __init irq_2_pin_init_work(void *data)
336{
337 struct dyn_array *da = data;
338 struct irq_pin_list *pin;
339 int i;
340
341 pin = *da->name;
342
343 for (i = 1; i < *da->nr; i++)
344 pin[i-1].next = &pin[i];
345
346 irq_2_pin_ptr = &pin[0];
347}
348DEFINE_DYN_ARRAY(irq_2_pin_head, sizeof(struct irq_pin_list), nr_irq_2_pin, PAGE_SIZE, irq_2_pin_init_work);
349
350static struct irq_pin_list *get_one_free_irq_2_pin(void)
351{
352 struct irq_pin_list *pin;
353 int i;
354
355 pin = irq_2_pin_ptr;
356
357 if (pin) {
358 irq_2_pin_ptr = pin->next;
359 pin->next = NULL;
360 return pin;
361 }
362
363 /*
364 * we run out of pre-allocate ones, allocate more
365 */
366 printk(KERN_DEBUG "try to get more irq_2_pin %d\n", nr_irq_2_pin);
367
368 if (after_bootmem)
369 pin = kzalloc(sizeof(struct irq_pin_list)*nr_irq_2_pin,
370 GFP_ATOMIC);
371 else
372 pin = __alloc_bootmem_nopanic(sizeof(struct irq_pin_list) *
373 nr_irq_2_pin, PAGE_SIZE, 0);
374
375 if (!pin)
376 panic("can not get more irq_2_pin\n");
301e6190 377
0f978f45
YL		 378 for (i = 1; i < nr_irq_2_pin; i++)
379 pin[i-1].next = &pin[i];
380
381 irq_2_pin_ptr = pin->next;
382 pin->next = NULL;
383
384 return pin;
385}
1da177e4 386
130fe05d
LT		 387struct io_apic {
388 unsigned int index;
389 unsigned int unused[3];
390 unsigned int data;
391};
392
393static __attribute_const__ struct io_apic __iomem *io_apic_base(int idx)
394{
395 return (void __iomem *) __fix_to_virt(FIX_IO_APIC_BASE_0 + idx)
ec2cd0a2 396 + (mp_ioapics[idx].mp_apicaddr & ~PAGE_MASK);
130fe05d
LT		 397}
398
399static inline unsigned int io_apic_read(unsigned int apic, unsigned int reg)
400{
401 struct io_apic __iomem *io_apic = io_apic_base(apic);
402 writel(reg, &io_apic->index);
403 return readl(&io_apic->data);
404}
405
406static inline void io_apic_write(unsigned int apic, unsigned int reg, unsigned int value)
407{
408 struct io_apic __iomem *io_apic = io_apic_base(apic);
409 writel(reg, &io_apic->index);
410 writel(value, &io_apic->data);
411}
412
413/*
414 * Re-write a value: to be used for read-modify-write
415 * cycles where the read already set up the index register.
416 *
417 * Older SiS APIC requires we rewrite the index register
418 */
419static inline void io_apic_modify(unsigned int apic, unsigned int reg, unsigned int value)
420{
54168ed7
IM		 421 struct io_apic __iomem *io_apic = io_apic_base(apic);
422 if (sis_apic_bug)
423 writel(reg, &io_apic->index);
130fe05d
LT		 424 writel(value, &io_apic->data);
425}
426
047c8fdb
YL		 427static bool io_apic_level_ack_pending(unsigned int irq)
428{
429 struct irq_pin_list *entry;
430 unsigned long flags;
431 struct irq_cfg *cfg = irq_cfg(irq);
432
433 spin_lock_irqsave(&ioapic_lock, flags);
434 entry = cfg->irq_2_pin;
435 for (;;) {
436 unsigned int reg;
437 int pin;
438
439 if (!entry)
440 break;
441 pin = entry->pin;
442 reg = io_apic_read(entry->apic, 0x10 + pin*2);
443 /* Is the remote IRR bit set? */
444 if (reg & IO_APIC_REDIR_REMOTE_IRR) {
445 spin_unlock_irqrestore(&ioapic_lock, flags);
446 return true;
447 }
448 if (!entry->next)
449 break;
450 entry = entry->next;
451 }
452 spin_unlock_irqrestore(&ioapic_lock, flags);
453
454 return false;
455}
047c8fdb 456
cf4c6a2f
AK		 457union entry_union {
458 struct { u32 w1, w2; };
459 struct IO_APIC_route_entry entry;
460};
461
462static struct IO_APIC_route_entry ioapic_read_entry(int apic, int pin)
463{
464 union entry_union eu;
465 unsigned long flags;
466 spin_lock_irqsave(&ioapic_lock, flags);
467 eu.w1 = io_apic_read(apic, 0x10 + 2 * pin);
468 eu.w2 = io_apic_read(apic, 0x11 + 2 * pin);
469 spin_unlock_irqrestore(&ioapic_lock, flags);
470 return eu.entry;
471}
472
f9dadfa7
LT		 473/*
474 * When we write a new IO APIC routing entry, we need to write the high
475 * word first! If the mask bit in the low word is clear, we will enable
476 * the interrupt, and we need to make sure the entry is fully populated
477 * before that happens.
478 */
d15512f4
AK		 479static void
480__ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
cf4c6a2f 481{
cf4c6a2f
AK		 482 union entry_union eu;
483 eu.entry = e;
f9dadfa7
LT		 484 io_apic_write(apic, 0x11 + 2*pin, eu.w2);
485 io_apic_write(apic, 0x10 + 2*pin, eu.w1);
d15512f4
AK		 486}
487
488static void ioapic_write_entry(int apic, int pin, struct IO_APIC_route_entry e)
489{
490 unsigned long flags;
491 spin_lock_irqsave(&ioapic_lock, flags);
492 __ioapic_write_entry(apic, pin, e);
f9dadfa7
LT		 493 spin_unlock_irqrestore(&ioapic_lock, flags);
494}
495
496/*
497 * When we mask an IO APIC routing entry, we need to write the low
498 * word first, in order to set the mask bit before we change the
499 * high bits!
500 */
501static void ioapic_mask_entry(int apic, int pin)
502{
503 unsigned long flags;
504 union entry_union eu = { .entry.mask = 1 };
505
cf4c6a2f
AK		 506 spin_lock_irqsave(&ioapic_lock, flags);
507 io_apic_write(apic, 0x10 + 2*pin, eu.w1);
508 io_apic_write(apic, 0x11 + 2*pin, eu.w2);
509 spin_unlock_irqrestore(&ioapic_lock, flags);
510}
511
497c9a19
YL		 512#ifdef CONFIG_SMP
513static void __target_IO_APIC_irq(unsigned int irq, unsigned int dest, u8 vector)
514{
515 int apic, pin;
516 struct irq_cfg *cfg;
517 struct irq_pin_list *entry;
518
519 cfg = irq_cfg(irq);
520 entry = cfg->irq_2_pin;
521 for (;;) {
522 unsigned int reg;
523
524 if (!entry)
525 break;
526
527 apic = entry->apic;
528 pin = entry->pin;
54168ed7
IM		 529#ifdef CONFIG_INTR_REMAP
530 /*
531 * With interrupt-remapping, destination information comes
532 * from interrupt-remapping table entry.
533 */
534 if (!irq_remapped(irq))
535 io_apic_write(apic, 0x11 + pin*2, dest);
536#else
497c9a19 537 io_apic_write(apic, 0x11 + pin*2, dest);
54168ed7 538#endif
497c9a19
YL		 539 reg = io_apic_read(apic, 0x10 + pin*2);
540 reg &= ~IO_APIC_REDIR_VECTOR_MASK;
541 reg |= vector;
54168ed7 542 io_apic_modify(apic, 0x10 + pin*2, reg);
497c9a19
YL		 543 if (!entry->next)
544 break;
545 entry = entry->next;
546 }
547}
efa2559f
YL		 548
549static int assign_irq_vector(int irq, cpumask_t mask);
550
497c9a19
YL		 551static void set_ioapic_affinity_irq(unsigned int irq, cpumask_t mask)
552{
553 struct irq_cfg *cfg;
554 unsigned long flags;
555 unsigned int dest;
556 cpumask_t tmp;
54168ed7 557 struct irq_desc *desc;
497c9a19 558
497c9a19
YL		 559 cpus_and(tmp, mask, cpu_online_map);
560 if (cpus_empty(tmp))
561 return;
562
047c8fdb 563 cfg = irq_cfg(irq);
497c9a19
YL		 564 if (assign_irq_vector(irq, mask))
565 return;
566
567 cpus_and(tmp, cfg->domain, mask);
497c9a19
YL		 568 dest = cpu_mask_to_apicid(tmp);
569 /*
570 * Only the high 8 bits are valid.
571 */
572 dest = SET_APIC_LOGICAL_ID(dest);
573
54168ed7 574 desc = irq_to_desc(irq);
497c9a19
YL		 575 spin_lock_irqsave(&ioapic_lock, flags);
576 __target_IO_APIC_irq(irq, dest, cfg->vector);
54168ed7 577 desc->affinity = mask;
497c9a19
YL		 578 spin_unlock_irqrestore(&ioapic_lock, flags);
579}
497c9a19
YL		 580#endif /* CONFIG_SMP */
581
1da177e4
LT		 582/*
583 * The common case is 1:1 IRQ<->pin mappings. Sometimes there are
584 * shared ISA-space IRQs, so we have to support them. We are super
585 * fast in the common case, and fast for shared ISA-space IRQs.
586 */
587static void add_pin_to_irq(unsigned int irq, int apic, int pin)
588{
0f978f45
YL		 589 struct irq_cfg *cfg;
590 struct irq_pin_list *entry;
591
592 /* first time to refer irq_cfg, so with new */
593 cfg = irq_cfg_alloc(irq);
594 entry = cfg->irq_2_pin;
595 if (!entry) {
596 entry = get_one_free_irq_2_pin();
597 cfg->irq_2_pin = entry;
598 entry->apic = apic;
599 entry->pin = pin;
600 printk(KERN_DEBUG " 0 add_pin_to_irq: irq %d --> apic %d pin %d\n", irq, apic, pin);
601 return;
602 }
1da177e4 603
0f978f45
YL		 604 while (entry->next) {
605 /* not again, please */
606 if (entry->apic == apic && entry->pin == pin)
607 return;
1da177e4 608
0f978f45 609 entry = entry->next;
1da177e4 610 }
0f978f45
YL		 611
612 entry->next = get_one_free_irq_2_pin();
613 entry = entry->next;
1da177e4
LT		 614 entry->apic = apic;
615 entry->pin = pin;
0f978f45 616 printk(KERN_DEBUG " x add_pin_to_irq: irq %d --> apic %d pin %d\n", irq, apic, pin);
1da177e4
LT		 617}
618
619/*
620 * Reroute an IRQ to a different pin.
621 */
622static void __init replace_pin_at_irq(unsigned int irq,
623 int oldapic, int oldpin,
624 int newapic, int newpin)
625{
0f978f45
YL		 626 struct irq_cfg *cfg = irq_cfg(irq);
627 struct irq_pin_list *entry = cfg->irq_2_pin;
628 int replaced = 0;
1da177e4 629
0f978f45 630 while (entry) {
1da177e4
LT		 631 if (entry->apic == oldapic && entry->pin == oldpin) {
632 entry->apic = newapic;
633 entry->pin = newpin;
0f978f45
YL		 634 replaced = 1;
635 /* every one is different, right? */
1da177e4 636 break;
0f978f45
YL		 637 }
638 entry = entry->next;
1da177e4 639 }
0f978f45
YL		 640
641 /* why? call replace before add? */
642 if (!replaced)
643 add_pin_to_irq(irq, newapic, newpin);
1da177e4
LT		 644}
645
4e738e2f 646#define __DO_ACTION(R, ACTION_ENABLE, ACTION_DISABLE, FINAL) \
047c8fdb
YL		 647 \
648{ \
649 int pin; \
650 struct irq_cfg *cfg; \
651 struct irq_pin_list *entry; \
652 \
653 cfg = irq_cfg(irq); \
654 entry = cfg->irq_2_pin; \
655 for (;;) { \
656 unsigned int reg; \
657 if (!entry) \
658 break; \
659 pin = entry->pin; \
660 reg = io_apic_read(entry->apic, 0x10 + R + pin*2); \
4e738e2f
YL		 661 reg ACTION_DISABLE; \
662 reg ACTION_ENABLE; \
047c8fdb
YL		 663 io_apic_modify(entry->apic, 0x10 + R + pin*2, reg); \
664 FINAL; \
665 if (!entry->next) \
666 break; \
667 entry = entry->next; \
668 } \
669}
670
4e738e2f 671#define DO_ACTION(name,R, ACTION_ENABLE, ACTION_DISABLE, FINAL) \
047c8fdb
YL		 672 \
673 static void name##_IO_APIC_irq (unsigned int irq) \
4e738e2f 674 __DO_ACTION(R, ACTION_ENABLE, ACTION_DISABLE, FINAL)
047c8fdb
YL		 675
676/* mask = 0 */
4e738e2f 677DO_ACTION(__unmask, 0, |= 0, &= ~IO_APIC_REDIR_MASKED, )
047c8fdb 678
4e738e2f
YL		 679#ifdef CONFIG_X86_64
680/*
681 * Synchronize the IO-APIC and the CPU by doing
682 * a dummy read from the IO-APIC
683 */
684static inline void io_apic_sync(unsigned int apic)
1da177e4 685{
4e738e2f
YL		 686 struct io_apic __iomem *io_apic = io_apic_base(apic);
687 readl(&io_apic->data);
1da177e4
LT		 688}
689
690/* mask = 1 */
4e738e2f 691DO_ACTION(__mask, 0, |= IO_APIC_REDIR_MASKED, &= ~0, io_apic_sync(entry->apic))
1da177e4 692
4e738e2f
YL		 693#else
694
695/* mask = 1 */
696DO_ACTION(__mask, 0, |= IO_APIC_REDIR_MASKED, &= ~0, )
1da177e4
LT		 697
698/* mask = 1, trigger = 0 */
4e738e2f 699DO_ACTION(__mask_and_edge, 0, |= IO_APIC_REDIR_MASKED, &= ~IO_APIC_REDIR_LEVEL_TRIGGER, )
1da177e4
LT		 700
701/* mask = 0, trigger = 1 */
4e738e2f 702DO_ACTION(__unmask_and_level, 0, |= IO_APIC_REDIR_LEVEL_TRIGGER, &= ~IO_APIC_REDIR_MASKED, )
1da177e4 703
047c8fdb
YL		 704#endif
705
54168ed7 706static void mask_IO_APIC_irq (unsigned int irq)
1da177e4
LT		 707{
708 unsigned long flags;
709
710 spin_lock_irqsave(&ioapic_lock, flags);
711 __mask_IO_APIC_irq(irq);
712 spin_unlock_irqrestore(&ioapic_lock, flags);
713}
714
54168ed7 715static void unmask_IO_APIC_irq (unsigned int irq)
1da177e4
LT		 716{
717 unsigned long flags;
718
719 spin_lock_irqsave(&ioapic_lock, flags);
720 __unmask_IO_APIC_irq(irq);
721 spin_unlock_irqrestore(&ioapic_lock, flags);
722}
723
724static void clear_IO_APIC_pin(unsigned int apic, unsigned int pin)
725{
726 struct IO_APIC_route_entry entry;
36062448 727
1da177e4 728 /* Check delivery_mode to be sure we're not clearing an SMI pin */
cf4c6a2f 729 entry = ioapic_read_entry(apic, pin);
1da177e4
LT		 730 if (entry.delivery_mode == dest_SMI)
731 return;
1da177e4
LT		 732 /*
733 * Disable it in the IO-APIC irq-routing table:
734 */
f9dadfa7 735 ioapic_mask_entry(apic, pin);
1da177e4
LT		 736}
737
54168ed7 738static void clear_IO_APIC (void)
1da177e4
LT		 739{
740 int apic, pin;
741
742 for (apic = 0; apic < nr_ioapics; apic++)
743 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
744 clear_IO_APIC_pin(apic, pin);
745}
746
54168ed7 747#if !defined(CONFIG_SMP) && defined(CONFIG_X86_32)
75604d7f 748void send_IPI_self(int vector)
1da177e4
LT		 749{
750 unsigned int cfg;
751
752 /*
753 * Wait for idle.
754 */
755 apic_wait_icr_idle();
756 cfg = APIC_DM_FIXED | APIC_DEST_SELF | vector | APIC_DEST_LOGICAL;
757 /*
758 * Send the IPI. The write to APIC_ICR fires this off.
759 */
593f4a78 760 apic_write(APIC_ICR, cfg);
1da177e4 761}
54168ed7 762#endif /* !CONFIG_SMP && CONFIG_X86_32*/
1da177e4 763
54168ed7 764#ifdef CONFIG_X86_32
1da177e4
LT		 765/*
766 * support for broken MP BIOSs, enables hand-redirection of PIRQ0-7 to
767 * specific CPU-side IRQs.
768 */
769
770#define MAX_PIRQS 8
771static int pirq_entries [MAX_PIRQS];
772static int pirqs_enabled;
1da177e4 773
1da177e4
LT		 774static int __init ioapic_pirq_setup(char *str)
775{
776 int i, max;
777 int ints[MAX_PIRQS+1];
778
779 get_options(str, ARRAY_SIZE(ints), ints);
780
781 for (i = 0; i < MAX_PIRQS; i++)
782 pirq_entries[i] = -1;
783
784 pirqs_enabled = 1;
785 apic_printk(APIC_VERBOSE, KERN_INFO
786 "PIRQ redirection, working around broken MP-BIOS.\n");
787 max = MAX_PIRQS;
788 if (ints[0] < MAX_PIRQS)
789 max = ints[0];
790
791 for (i = 0; i < max; i++) {
792 apic_printk(APIC_VERBOSE, KERN_DEBUG
793 "... PIRQ%d -> IRQ %d\n", i, ints[i+1]);
794 /*
795 * PIRQs are mapped upside down, usually.
796 */
797 pirq_entries[MAX_PIRQS-i-1] = ints[i+1];
798 }
799 return 1;
800}
801
802__setup("pirq=", ioapic_pirq_setup);
54168ed7
IM		 803#endif /* CONFIG_X86_32 */
804
805#ifdef CONFIG_INTR_REMAP
806/* I/O APIC RTE contents at the OS boot up */
807static struct IO_APIC_route_entry *early_ioapic_entries[MAX_IO_APICS];
808
809/*
810 * Saves and masks all the unmasked IO-APIC RTE's
811 */
812int save_mask_IO_APIC_setup(void)
813{
814 union IO_APIC_reg_01 reg_01;
815 unsigned long flags;
816 int apic, pin;
817
818 /*
819 * The number of IO-APIC IRQ registers (== #pins):
820 */
821 for (apic = 0; apic < nr_ioapics; apic++) {
822 spin_lock_irqsave(&ioapic_lock, flags);
823 reg_01.raw = io_apic_read(apic, 1);
824 spin_unlock_irqrestore(&ioapic_lock, flags);
825 nr_ioapic_registers[apic] = reg_01.bits.entries+1;
826 }
827
828 for (apic = 0; apic < nr_ioapics; apic++) {
829 early_ioapic_entries[apic] =
830 kzalloc(sizeof(struct IO_APIC_route_entry) *
831 nr_ioapic_registers[apic], GFP_KERNEL);
832 if (!early_ioapic_entries[apic])
833 return -ENOMEM;
834 }
835
836 for (apic = 0; apic < nr_ioapics; apic++)
837 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
838 struct IO_APIC_route_entry entry;
839
840 entry = early_ioapic_entries[apic][pin] =
841 ioapic_read_entry(apic, pin);
842 if (!entry.mask) {
843 entry.mask = 1;
844 ioapic_write_entry(apic, pin, entry);
845 }
846 }
847 return 0;
848}
849
850void restore_IO_APIC_setup(void)
851{
852 int apic, pin;
853
854 for (apic = 0; apic < nr_ioapics; apic++)
855 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++)
856 ioapic_write_entry(apic, pin,
857 early_ioapic_entries[apic][pin]);
858}
859
860void reinit_intr_remapped_IO_APIC(int intr_remapping)
861{
862 /*
863 * for now plain restore of previous settings.
864 * TBD: In the case of OS enabling interrupt-remapping,
865 * IO-APIC RTE's need to be setup to point to interrupt-remapping
866 * table entries. for now, do a plain restore, and wait for
867 * the setup_IO_APIC_irqs() to do proper initialization.
868 */
869 restore_IO_APIC_setup();
870}
871#endif
1da177e4
LT		 872
873/*
874 * Find the IRQ entry number of a certain pin.
875 */
876static int find_irq_entry(int apic, int pin, int type)
877{
878 int i;
879
880 for (i = 0; i < mp_irq_entries; i++)
2fddb6e2
AS		 881 if (mp_irqs[i].mp_irqtype == type &&
882 (mp_irqs[i].mp_dstapic == mp_ioapics[apic].mp_apicid ||
883 mp_irqs[i].mp_dstapic == MP_APIC_ALL) &&
884 mp_irqs[i].mp_dstirq == pin)
1da177e4
LT		 885 return i;
886
887 return -1;
888}
889
890/*
891 * Find the pin to which IRQ[irq] (ISA) is connected
892 */
fcfd636a 893static int __init find_isa_irq_pin(int irq, int type)
1da177e4
LT		 894{
895 int i;
896
897 for (i = 0; i < mp_irq_entries; i++) {
2fddb6e2 898 int lbus = mp_irqs[i].mp_srcbus;
1da177e4 899
d27e2b8e 900 if (test_bit(lbus, mp_bus_not_pci) &&
2fddb6e2
AS		 901 (mp_irqs[i].mp_irqtype == type) &&
902 (mp_irqs[i].mp_srcbusirq == irq))
1da177e4 903
2fddb6e2 904 return mp_irqs[i].mp_dstirq;
1da177e4
LT		 905 }
906 return -1;
907}
908
fcfd636a
EB		 909static int __init find_isa_irq_apic(int irq, int type)
910{
911 int i;
912
913 for (i = 0; i < mp_irq_entries; i++) {
2fddb6e2 914 int lbus = mp_irqs[i].mp_srcbus;
fcfd636a 915
73b2961b 916 if (test_bit(lbus, mp_bus_not_pci) &&
2fddb6e2
AS		 917 (mp_irqs[i].mp_irqtype == type) &&
918 (mp_irqs[i].mp_srcbusirq == irq))
fcfd636a
EB		 919 break;
920 }
921 if (i < mp_irq_entries) {
922 int apic;
54168ed7 923 for(apic = 0; apic < nr_ioapics; apic++) {
2fddb6e2 924 if (mp_ioapics[apic].mp_apicid == mp_irqs[i].mp_dstapic)
fcfd636a
EB		 925 return apic;
926 }
927 }
928
929 return -1;
930}
931
1da177e4
LT		 932/*
933 * Find a specific PCI IRQ entry.
934 * Not an __init, possibly needed by modules
935 */
936static int pin_2_irq(int idx, int apic, int pin);
937
938int IO_APIC_get_PCI_irq_vector(int bus, int slot, int pin)
939{
940 int apic, i, best_guess = -1;
941
54168ed7
IM		 942 apic_printk(APIC_DEBUG, "querying PCI -> IRQ mapping bus:%d, slot:%d, pin:%d.\n",
943 bus, slot, pin);
ce6444d3 944 if (test_bit(bus, mp_bus_not_pci)) {
54168ed7 945 apic_printk(APIC_VERBOSE, "PCI BIOS passed nonexistent PCI bus %d!\n", bus);
1da177e4
LT		 946 return -1;
947 }
948 for (i = 0; i < mp_irq_entries; i++) {
2fddb6e2 949 int lbus = mp_irqs[i].mp_srcbus;
1da177e4
LT		 950
951 for (apic = 0; apic < nr_ioapics; apic++)
2fddb6e2
AS		 952 if (mp_ioapics[apic].mp_apicid == mp_irqs[i].mp_dstapic ||
953 mp_irqs[i].mp_dstapic == MP_APIC_ALL)
1da177e4
LT		 954 break;
955
47cab822 956 if (!test_bit(lbus, mp_bus_not_pci) &&
2fddb6e2 957 !mp_irqs[i].mp_irqtype &&
1da177e4 958 (bus == lbus) &&
2fddb6e2 959 (slot == ((mp_irqs[i].mp_srcbusirq >> 2) & 0x1f))) {
54168ed7 960 int irq = pin_2_irq(i,apic,mp_irqs[i].mp_dstirq);
1da177e4
LT		 961
962 if (!(apic || IO_APIC_IRQ(irq)))
963 continue;
964
2fddb6e2 965 if (pin == (mp_irqs[i].mp_srcbusirq & 3))
1da177e4
LT		 966 return irq;
967 /*
968 * Use the first all-but-pin matching entry as a
969 * best-guess fuzzy result for broken mptables.
970 */
971 if (best_guess < 0)
972 best_guess = irq;
973 }
974 }
975 return best_guess;
976}
54168ed7 977
129f6946 978EXPORT_SYMBOL(IO_APIC_get_PCI_irq_vector);
1da177e4 979
c0a282c2 980#if defined(CONFIG_EISA) || defined(CONFIG_MCA)
1da177e4
LT		 981/*
982 * EISA Edge/Level control register, ELCR
983 */
984static int EISA_ELCR(unsigned int irq)
985{
986 if (irq < 16) {
987 unsigned int port = 0x4d0 + (irq >> 3);
988 return (inb(port) >> (irq & 7)) & 1;
989 }
990 apic_printk(APIC_VERBOSE, KERN_INFO
991 "Broken MPtable reports ISA irq %d\n", irq);
992 return 0;
993}
54168ed7 994
c0a282c2 995#endif
1da177e4 996
6728801d
AS		 997/* ISA interrupts are always polarity zero edge triggered,
998 * when listed as conforming in the MP table. */
999
1000#define default_ISA_trigger(idx) (0)
1001#define default_ISA_polarity(idx) (0)
1002
1da177e4
LT		 1003/* EISA interrupts are always polarity zero and can be edge or level
1004 * trigger depending on the ELCR value. If an interrupt is listed as
1005 * EISA conforming in the MP table, that means its trigger type must
1006 * be read in from the ELCR */
1007
2fddb6e2 1008#define default_EISA_trigger(idx) (EISA_ELCR(mp_irqs[idx].mp_srcbusirq))
6728801d 1009#define default_EISA_polarity(idx) default_ISA_polarity(idx)
1da177e4
LT		 1010
1011/* PCI interrupts are always polarity one level triggered,
1012 * when listed as conforming in the MP table. */
1013
1014#define default_PCI_trigger(idx) (1)
1015#define default_PCI_polarity(idx) (1)
1016
1017/* MCA interrupts are always polarity zero level triggered,
1018 * when listed as conforming in the MP table. */
1019
1020#define default_MCA_trigger(idx) (1)
6728801d 1021#define default_MCA_polarity(idx) default_ISA_polarity(idx)
1da177e4 1022
61fd47e0 1023static int MPBIOS_polarity(int idx)
1da177e4 1024{
2fddb6e2 1025 int bus = mp_irqs[idx].mp_srcbus;
1da177e4
LT		 1026 int polarity;
1027
1028 /*
1029 * Determine IRQ line polarity (high active or low active):
1030 */
54168ed7 1031 switch (mp_irqs[idx].mp_irqflag & 3)
36062448 1032 {
54168ed7
IM		 1033 case 0: /* conforms, ie. bus-type dependent polarity */
1034 if (test_bit(bus, mp_bus_not_pci))
1035 polarity = default_ISA_polarity(idx);
1036 else
1037 polarity = default_PCI_polarity(idx);
1038 break;
1039 case 1: /* high active */
1040 {
1041 polarity = 0;
1042 break;
1043 }
1044 case 2: /* reserved */
1045 {
1046 printk(KERN_WARNING "broken BIOS!!\n");
1047 polarity = 1;
1048 break;
1049 }
1050 case 3: /* low active */
1051 {
1052 polarity = 1;
1053 break;
1054 }
1055 default: /* invalid */
1056 {
1057 printk(KERN_WARNING "broken BIOS!!\n");
1058 polarity = 1;
1059 break;
1060 }
1da177e4
LT		 1061 }
1062 return polarity;
1063}
1064
1065static int MPBIOS_trigger(int idx)
1066{
2fddb6e2 1067 int bus = mp_irqs[idx].mp_srcbus;
1da177e4
LT		 1068 int trigger;
1069
1070 /*
1071 * Determine IRQ trigger mode (edge or level sensitive):
1072 */
54168ed7 1073 switch ((mp_irqs[idx].mp_irqflag>>2) & 3)
1da177e4 1074 {
54168ed7
IM		 1075 case 0: /* conforms, ie. bus-type dependent */
1076 if (test_bit(bus, mp_bus_not_pci))
1077 trigger = default_ISA_trigger(idx);
1078 else
1079 trigger = default_PCI_trigger(idx);
c0a282c2 1080#if defined(CONFIG_EISA) || defined(CONFIG_MCA)
54168ed7
IM		 1081 switch (mp_bus_id_to_type[bus]) {
1082 case MP_BUS_ISA: /* ISA pin */
1083 {
1084 /* set before the switch */
1085 break;
1086 }
1087 case MP_BUS_EISA: /* EISA pin */
1088 {
1089 trigger = default_EISA_trigger(idx);
1090 break;
1091 }
1092 case MP_BUS_PCI: /* PCI pin */
1093 {
1094 /* set before the switch */
1095 break;
1096 }
1097 case MP_BUS_MCA: /* MCA pin */
1098 {
1099 trigger = default_MCA_trigger(idx);
1100 break;
1101 }
1102 default:
1103 {
1104 printk(KERN_WARNING "broken BIOS!!\n");
1105 trigger = 1;
1106 break;
1107 }
1108 }
1109#endif
1da177e4 1110 break;
54168ed7 1111 case 1: /* edge */
1da177e4 1112 {
54168ed7 1113 trigger = 0;
1da177e4
LT		 1114 break;
1115 }
54168ed7 1116 case 2: /* reserved */
1da177e4 1117 {
54168ed7
IM		 1118 printk(KERN_WARNING "broken BIOS!!\n");
1119 trigger = 1;
1da177e4
LT		 1120 break;
1121 }
54168ed7 1122 case 3: /* level */
1da177e4 1123 {
54168ed7 1124 trigger = 1;
1da177e4
LT		 1125 break;
1126 }
54168ed7 1127 default: /* invalid */
1da177e4
LT		 1128 {
1129 printk(KERN_WARNING "broken BIOS!!\n");
54168ed7 1130 trigger = 0;
1da177e4
LT		 1131 break;
1132 }
1133 }
1134 return trigger;
1135}
1136
1137static inline int irq_polarity(int idx)
1138{
1139 return MPBIOS_polarity(idx);
1140}
1141
1142static inline int irq_trigger(int idx)
1143{
1144 return MPBIOS_trigger(idx);
1145}
1146
efa2559f 1147int (*ioapic_renumber_irq)(int ioapic, int irq);
1da177e4
LT		 1148static int pin_2_irq(int idx, int apic, int pin)
1149{
1150 int irq, i;
2fddb6e2 1151 int bus = mp_irqs[idx].mp_srcbus;
1da177e4
LT		 1152
1153 /*
1154 * Debugging check, we are in big trouble if this message pops up!
1155 */
2fddb6e2 1156 if (mp_irqs[idx].mp_dstirq != pin)
1da177e4
LT		 1157 printk(KERN_ERR "broken BIOS or MPTABLE parser, ayiee!!\n");
1158
54168ed7 1159 if (test_bit(bus, mp_bus_not_pci)) {
2fddb6e2 1160 irq = mp_irqs[idx].mp_srcbusirq;
54168ed7 1161 } else {
643befed
AS		 1162 /*
1163 * PCI IRQs are mapped in order
1164 */
1165 i = irq = 0;
1166 while (i < apic)
1167 irq += nr_ioapic_registers[i++];
1168 irq += pin;
54168ed7
IM		 1169 /*
1170 * For MPS mode, so far only needed by ES7000 platform
1171 */
1172 if (ioapic_renumber_irq)
1173 irq = ioapic_renumber_irq(apic, irq);
1da177e4
LT		 1174 }
1175
54168ed7 1176#ifdef CONFIG_X86_32
1da177e4
LT		 1177 /*
1178 * PCI IRQ command line redirection. Yes, limits are hardcoded.
1179 */
1180 if ((pin >= 16) && (pin <= 23)) {
1181 if (pirq_entries[pin-16] != -1) {
1182 if (!pirq_entries[pin-16]) {
1183 apic_printk(APIC_VERBOSE, KERN_DEBUG
1184 "disabling PIRQ%d\n", pin-16);
1185 } else {
1186 irq = pirq_entries[pin-16];
1187 apic_printk(APIC_VERBOSE, KERN_DEBUG
1188 "using PIRQ%d -> IRQ %d\n",
1189 pin-16, irq);
1190 }
1191 }
1192 }
54168ed7
IM		 1193#endif
1194
1da177e4
LT		 1195 return irq;
1196}
1197
497c9a19
YL		 1198void lock_vector_lock(void)
1199{
1200 /* Used to the online set of cpus does not change
1201 * during assign_irq_vector.
1202 */
1203 spin_lock(&vector_lock);
1204}
1da177e4 1205
497c9a19 1206void unlock_vector_lock(void)
1da177e4 1207{
497c9a19
YL		 1208 spin_unlock(&vector_lock);
1209}
1da177e4 1210
497c9a19
YL		 1211static int __assign_irq_vector(int irq, cpumask_t mask)
1212{
047c8fdb
YL		 1213 /*
1214 * NOTE! The local APIC isn't very good at handling
1215 * multiple interrupts at the same interrupt level.
1216 * As the interrupt level is determined by taking the
1217 * vector number and shifting that right by 4, we
1218 * want to spread these out a bit so that they don't
1219 * all fall in the same interrupt level.
1220 *
1221 * Also, we've got to be careful not to trash gate
1222 * 0x80, because int 0x80 is hm, kind of importantish. ;)
1223 */
54168ed7
IM		 1224 static int current_vector = FIRST_DEVICE_VECTOR, current_offset = 0;
1225 unsigned int old_vector;
1226 int cpu;
1227 struct irq_cfg *cfg;
ace80ab7 1228
54168ed7 1229 cfg = irq_cfg(irq);
8339f000 1230
54168ed7
IM		 1231 /* Only try and allocate irqs on cpus that are present */
1232 cpus_and(mask, mask, cpu_online_map);
ace80ab7 1233
54168ed7
IM		 1234 if ((cfg->move_in_progress) || cfg->move_cleanup_count)
1235 return -EBUSY;
0a1ad60d 1236
54168ed7
IM		 1237 old_vector = cfg->vector;
1238 if (old_vector) {
1239 cpumask_t tmp;
1240 cpus_and(tmp, cfg->domain, mask);
1241 if (!cpus_empty(tmp))
1242 return 0;
1243 }
497c9a19 1244
54168ed7
IM		 1245 for_each_cpu_mask_nr(cpu, mask) {
1246 cpumask_t domain, new_mask;
1247 int new_cpu;
1248 int vector, offset;
497c9a19 1249
54168ed7
IM		 1250 domain = vector_allocation_domain(cpu);
1251 cpus_and(new_mask, domain, cpu_online_map);
497c9a19 1252
54168ed7
IM		 1253 vector = current_vector;
1254 offset = current_offset;
497c9a19 1255next:
54168ed7
IM		 1256 vector += 8;
1257 if (vector >= first_system_vector) {
1258 /* If we run out of vectors on large boxen, must share them. */
1259 offset = (offset + 1) % 8;
1260 vector = FIRST_DEVICE_VECTOR + offset;
1261 }
1262 if (unlikely(current_vector == vector))
1263 continue;
047c8fdb 1264#ifdef CONFIG_X86_64
54168ed7
IM		 1265 if (vector == IA32_SYSCALL_VECTOR)
1266 goto next;
047c8fdb 1267#else
54168ed7
IM		 1268 if (vector == SYSCALL_VECTOR)
1269 goto next;
047c8fdb 1270#endif
54168ed7
IM		 1271 for_each_cpu_mask_nr(new_cpu, new_mask)
1272 if (per_cpu(vector_irq, new_cpu)[vector] != -1)
1273 goto next;
1274 /* Found one! */
1275 current_vector = vector;
1276 current_offset = offset;
1277 if (old_vector) {
1278 cfg->move_in_progress = 1;
1279 cfg->old_domain = cfg->domain;
7a959cff 1280 }
54168ed7
IM		 1281 for_each_cpu_mask_nr(new_cpu, new_mask)
1282 per_cpu(vector_irq, new_cpu)[vector] = irq;
1283 cfg->vector = vector;
1284 cfg->domain = domain;
1285 return 0;
1286 }
1287 return -ENOSPC;
497c9a19
YL		 1288}
1289
1290static int assign_irq_vector(int irq, cpumask_t mask)
1291{
1292 int err;
ace80ab7 1293 unsigned long flags;
ace80ab7
EB		 1294
1295 spin_lock_irqsave(&vector_lock, flags);
497c9a19 1296 err = __assign_irq_vector(irq, mask);
26a3c49c 1297 spin_unlock_irqrestore(&vector_lock, flags);
497c9a19
YL		 1298 return err;
1299}
1300
1301static void __clear_irq_vector(int irq)
1302{
1303 struct irq_cfg *cfg;
1304 cpumask_t mask;
1305 int cpu, vector;
1306
1307 cfg = irq_cfg(irq);
1308 BUG_ON(!cfg->vector);
1309
1310 vector = cfg->vector;
1311 cpus_and(mask, cfg->domain, cpu_online_map);
1312 for_each_cpu_mask_nr(cpu, mask)
1313 per_cpu(vector_irq, cpu)[vector] = -1;
1314
1315 cfg->vector = 0;
1316 cpus_clear(cfg->domain);
1317}
1318
1319void __setup_vector_irq(int cpu)
1320{
1321 /* Initialize vector_irq on a new cpu */
1322 /* This function must be called with vector_lock held */
1323 int irq, vector;
1324 struct irq_cfg *cfg;
1325
1326 /* Mark the inuse vectors */
8f09cd20 1327 for_each_irq_cfg(irq, cfg) {
497c9a19
YL		 1328 if (!cpu_isset(cpu, cfg->domain))
1329 continue;
1330 vector = cfg->vector;
497c9a19
YL		 1331 per_cpu(vector_irq, cpu)[vector] = irq;
1332 }
1333 /* Mark the free vectors */
1334 for (vector = 0; vector < NR_VECTORS; ++vector) {
1335 irq = per_cpu(vector_irq, cpu)[vector];
1336 if (irq < 0)
1337 continue;
1338
1339 cfg = irq_cfg(irq);
1340 if (!cpu_isset(cpu, cfg->domain))
1341 per_cpu(vector_irq, cpu)[vector] = -1;
54168ed7 1342 }
1da177e4 1343}
3fde6900 1344
f5b9ed7a 1345static struct irq_chip ioapic_chip;
54168ed7
IM		 1346#ifdef CONFIG_INTR_REMAP
1347static struct irq_chip ir_ioapic_chip;
1348#endif
1da177e4 1349
54168ed7
IM		 1350#define IOAPIC_AUTO -1
1351#define IOAPIC_EDGE 0
1352#define IOAPIC_LEVEL 1
1da177e4 1353
047c8fdb 1354#ifdef CONFIG_X86_32
1d025192
YL		 1355static inline int IO_APIC_irq_trigger(int irq)
1356{
54168ed7 1357 int apic, idx, pin;
1d025192 1358
54168ed7
IM		 1359 for (apic = 0; apic < nr_ioapics; apic++) {
1360 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
1361 idx = find_irq_entry(apic, pin, mp_INT);
1362 if ((idx != -1) && (irq == pin_2_irq(idx, apic, pin)))
1363 return irq_trigger(idx);
1364 }
1365 }
1366 /*
1367 * nonexistent IRQs are edge default
1368 */
1369 return 0;
1d025192 1370}
047c8fdb
YL		 1371#else
1372static inline int IO_APIC_irq_trigger(int irq)
1373{
54168ed7 1374 return 1;
047c8fdb
YL		 1375}
1376#endif
1d025192 1377
497c9a19 1378static void ioapic_register_intr(int irq, unsigned long trigger)
1da177e4 1379{
08678b08
YL		 1380 struct irq_desc *desc;
1381
199751d7
YL		 1382 /* first time to use this irq_desc */
1383 if (irq < 16)
1384 desc = irq_to_desc(irq);
1385 else
1386 desc = irq_to_desc_alloc(irq);
1387
6ebcc00e 1388 if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
047c8fdb 1389 trigger == IOAPIC_LEVEL)
08678b08 1390 desc->status |= IRQ_LEVEL;
047c8fdb
YL		 1391 else
1392 desc->status &= ~IRQ_LEVEL;
1393
54168ed7
IM		 1394#ifdef CONFIG_INTR_REMAP
1395 if (irq_remapped(irq)) {
1396 desc->status |= IRQ_MOVE_PCNTXT;
1397 if (trigger)
1398 set_irq_chip_and_handler_name(irq, &ir_ioapic_chip,
1399 handle_fasteoi_irq,
1400 "fasteoi");
1401 else
1402 set_irq_chip_and_handler_name(irq, &ir_ioapic_chip,
1403 handle_edge_irq, "edge");
1404 return;
1405 }
1406#endif
047c8fdb
YL		 1407 if ((trigger == IOAPIC_AUTO && IO_APIC_irq_trigger(irq)) ||
1408 trigger == IOAPIC_LEVEL)
a460e745 1409 set_irq_chip_and_handler_name(irq, &ioapic_chip,
54168ed7
IM		 1410 handle_fasteoi_irq,
1411 "fasteoi");
047c8fdb 1412 else
a460e745 1413 set_irq_chip_and_handler_name(irq, &ioapic_chip,
54168ed7 1414 handle_edge_irq, "edge");
1da177e4
LT		 1415}
1416
497c9a19
YL		 1417static int setup_ioapic_entry(int apic, int irq,
1418 struct IO_APIC_route_entry *entry,
1419 unsigned int destination, int trigger,
1420 int polarity, int vector)
1da177e4 1421{
497c9a19
YL		 1422 /*
1423 * add it to the IO-APIC irq-routing table:
1424 */
1425 memset(entry,0,sizeof(*entry));
1426
54168ed7
IM		 1427#ifdef CONFIG_INTR_REMAP
1428 if (intr_remapping_enabled) {
1429 struct intel_iommu *iommu = map_ioapic_to_ir(apic);
1430 struct irte irte;
1431 struct IR_IO_APIC_route_entry *ir_entry =
1432 (struct IR_IO_APIC_route_entry *) entry;
1433 int index;
1434
1435 if (!iommu)
1436 panic("No mapping iommu for ioapic %d\n", apic);
1437
1438 index = alloc_irte(iommu, irq, 1);
1439 if (index < 0)
1440 panic("Failed to allocate IRTE for ioapic %d\n", apic);
1441
1442 memset(&irte, 0, sizeof(irte));
1443
1444 irte.present = 1;
1445 irte.dst_mode = INT_DEST_MODE;
1446 irte.trigger_mode = trigger;
1447 irte.dlvry_mode = INT_DELIVERY_MODE;
1448 irte.vector = vector;
1449 irte.dest_id = IRTE_DEST(destination);
1450
1451 modify_irte(irq, &irte);
1452
1453 ir_entry->index2 = (index >> 15) & 0x1;
1454 ir_entry->zero = 0;
1455 ir_entry->format = 1;
1456 ir_entry->index = (index & 0x7fff);
1457 } else
1458#endif
1459 {
1460 entry->delivery_mode = INT_DELIVERY_MODE;
1461 entry->dest_mode = INT_DEST_MODE;
1462 entry->dest = destination;
1463 }
497c9a19 1464
54168ed7 1465 entry->mask = 0; /* enable IRQ */
497c9a19
YL		 1466 entry->trigger = trigger;
1467 entry->polarity = polarity;
1468 entry->vector = vector;
1469
1470 /* Mask level triggered irqs.
1471 * Use IRQ_DELAYED_DISABLE for edge triggered irqs.
1472 */
1473 if (trigger)
1474 entry->mask = 1;
497c9a19
YL		 1475 return 0;
1476}
1477
1478static void setup_IO_APIC_irq(int apic, int pin, unsigned int irq,
54168ed7 1479 int trigger, int polarity)
497c9a19
YL		 1480{
1481 struct irq_cfg *cfg;
1da177e4 1482 struct IO_APIC_route_entry entry;
497c9a19
YL		 1483 cpumask_t mask;
1484
1485 if (!IO_APIC_IRQ(irq))
1486 return;
1487
1488 cfg = irq_cfg(irq);
1489
1490 mask = TARGET_CPUS;
1491 if (assign_irq_vector(irq, mask))
1492 return;
1493
1494 cpus_and(mask, cfg->domain, mask);
1495
1496 apic_printk(APIC_VERBOSE,KERN_DEBUG
1497 "IOAPIC[%d]: Set routing entry (%d-%d -> 0x%x -> "
1498 "IRQ %d Mode:%i Active:%i)\n",
1499 apic, mp_ioapics[apic].mp_apicid, pin, cfg->vector,
1500 irq, trigger, polarity);
1501
1502
1503 if (setup_ioapic_entry(mp_ioapics[apic].mp_apicid, irq, &entry,
1504 cpu_mask_to_apicid(mask), trigger, polarity,
1505 cfg->vector)) {
1506 printk("Failed to setup ioapic entry for ioapic %d, pin %d\n",
1507 mp_ioapics[apic].mp_apicid, pin);
1508 __clear_irq_vector(irq);
1509 return;
1510 }
1511
1512 ioapic_register_intr(irq, trigger);
1513 if (irq < 16)
1514 disable_8259A_irq(irq);
1515
1516 ioapic_write_entry(apic, pin, entry);
1517}
1518
1519static void __init setup_IO_APIC_irqs(void)
1520{
3c2cbd24
CG		 1521 int apic, pin, idx, irq;
1522 int notcon = 0;
1da177e4
LT		 1523
1524 apic_printk(APIC_VERBOSE, KERN_DEBUG "init IO_APIC IRQs\n");
1525
1526 for (apic = 0; apic < nr_ioapics; apic++) {
3c2cbd24 1527 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
20d225b9 1528
3c2cbd24
CG		 1529 idx = find_irq_entry(apic, pin, mp_INT);
1530 if (idx == -1) {
2a554fb1 1531 if (!notcon) {
3c2cbd24 1532 notcon = 1;
2a554fb1
CG		 1533 apic_printk(APIC_VERBOSE,
1534 KERN_DEBUG " %d-%d",
1535 mp_ioapics[apic].mp_apicid,
1536 pin);
1537 } else
1538 apic_printk(APIC_VERBOSE, " %d-%d",
1539 mp_ioapics[apic].mp_apicid,
1540 pin);
3c2cbd24
CG		 1541 continue;
1542 }
1543
1544 irq = pin_2_irq(idx, apic, pin);
54168ed7 1545#ifdef CONFIG_X86_32
3c2cbd24
CG		 1546 if (multi_timer_check(apic, irq))
1547 continue;
54168ed7 1548#endif
3c2cbd24 1549 add_pin_to_irq(irq, apic, pin);
36062448 1550
3c2cbd24
CG		 1551 setup_IO_APIC_irq(apic, pin, irq,
1552 irq_trigger(idx), irq_polarity(idx));
1553 }
1554 if (notcon) {
1555 apic_printk(APIC_VERBOSE,
2a554fb1 1556 " (apicid-pin) not connected\n");
3c2cbd24
CG		 1557 notcon = 0;
1558 }
1da177e4
LT		 1559 }
1560
3c2cbd24
CG		 1561 if (notcon)
1562 apic_printk(APIC_VERBOSE,
2a554fb1 1563 " (apicid-pin) not connected\n");
1da177e4
LT		 1564}
1565
1566/*
f7633ce5 1567 * Set up the timer pin, possibly with the 8259A-master behind.
1da177e4 1568 */
f7633ce5
MR		 1569static void __init setup_timer_IRQ0_pin(unsigned int apic, unsigned int pin,
1570 int vector)
1da177e4
LT		 1571{
1572 struct IO_APIC_route_entry entry;
1da177e4 1573
54168ed7
IM		 1574#ifdef CONFIG_INTR_REMAP
1575 if (intr_remapping_enabled)
1576 return;
1577#endif
1578
36062448 1579 memset(&entry, 0, sizeof(entry));
1da177e4
LT		 1580
1581 /*
1582 * We use logical delivery to get the timer IRQ
1583 * to the first CPU.
1584 */
1585 entry.dest_mode = INT_DEST_MODE;
03be7505 1586 entry.mask = 1; /* mask IRQ now */
d83e94ac 1587 entry.dest = cpu_mask_to_apicid(TARGET_CPUS);
1da177e4
LT		 1588 entry.delivery_mode = INT_DELIVERY_MODE;
1589 entry.polarity = 0;
1590 entry.trigger = 0;
1591 entry.vector = vector;
1592
1593 /*
1594 * The timer IRQ doesn't have to know that behind the
f7633ce5 1595 * scene we may have a 8259A-master in AEOI mode ...
1da177e4 1596 */
54168ed7 1597 set_irq_chip_and_handler_name(0, &ioapic_chip, handle_edge_irq, "edge");
1da177e4
LT		 1598
1599 /*
1600 * Add it to the IO-APIC irq-routing table:
1601 */
cf4c6a2f 1602 ioapic_write_entry(apic, pin, entry);
1da177e4
LT		 1603}
1604
32f71aff
MR		 1605
1606__apicdebuginit(void) print_IO_APIC(void)
1da177e4
LT		 1607{
1608 int apic, i;
1609 union IO_APIC_reg_00 reg_00;
1610 union IO_APIC_reg_01 reg_01;
1611 union IO_APIC_reg_02 reg_02;
1612 union IO_APIC_reg_03 reg_03;
1613 unsigned long flags;
0f978f45 1614 struct irq_cfg *cfg;
8f09cd20 1615 unsigned int irq;
1da177e4
LT		 1616
1617 if (apic_verbosity == APIC_QUIET)
1618 return;
1619
36062448 1620 printk(KERN_DEBUG "number of MP IRQ sources: %d.\n", mp_irq_entries);
1da177e4
LT		 1621 for (i = 0; i < nr_ioapics; i++)
1622 printk(KERN_DEBUG "number of IO-APIC #%d registers: %d.\n",
ec2cd0a2 1623 mp_ioapics[i].mp_apicid, nr_ioapic_registers[i]);
1da177e4
LT		 1624
1625 /*
1626 * We are a bit conservative about what we expect. We have to
1627 * know about every hardware change ASAP.
1628 */
1629 printk(KERN_INFO "testing the IO APIC.......................\n");
1630
1631 for (apic = 0; apic < nr_ioapics; apic++) {
1632
1633 spin_lock_irqsave(&ioapic_lock, flags);
1634 reg_00.raw = io_apic_read(apic, 0);
1635 reg_01.raw = io_apic_read(apic, 1);
1636 if (reg_01.bits.version >= 0x10)
1637 reg_02.raw = io_apic_read(apic, 2);
54168ed7
IM		 1638 if (reg_01.bits.version >= 0x20)
1639 reg_03.raw = io_apic_read(apic, 3);
1da177e4
LT		 1640 spin_unlock_irqrestore(&ioapic_lock, flags);
1641
54168ed7 1642 printk("\n");
ec2cd0a2 1643 printk(KERN_DEBUG "IO APIC #%d......\n", mp_ioapics[apic].mp_apicid);
1da177e4
LT		 1644 printk(KERN_DEBUG ".... register #00: %08X\n", reg_00.raw);
1645 printk(KERN_DEBUG "....... : physical APIC id: %02X\n", reg_00.bits.ID);
1646 printk(KERN_DEBUG "....... : Delivery Type: %X\n", reg_00.bits.delivery_type);
1647 printk(KERN_DEBUG "....... : LTS : %X\n", reg_00.bits.LTS);
1da177e4 1648
54168ed7 1649 printk(KERN_DEBUG ".... register #01: %08X\n", *(int *)&reg_01);
1da177e4 1650 printk(KERN_DEBUG "....... : max redirection entries: %04X\n", reg_01.bits.entries);
1da177e4
LT		 1651
1652 printk(KERN_DEBUG "....... : PRQ implemented: %X\n", reg_01.bits.PRQ);
1653 printk(KERN_DEBUG "....... : IO APIC version: %04X\n", reg_01.bits.version);
1da177e4
LT		 1654
1655 /*
1656 * Some Intel chipsets with IO APIC VERSION of 0x1? don't have reg_02,
1657 * but the value of reg_02 is read as the previous read register
1658 * value, so ignore it if reg_02 == reg_01.
1659 */
1660 if (reg_01.bits.version >= 0x10 && reg_02.raw != reg_01.raw) {
1661 printk(KERN_DEBUG ".... register #02: %08X\n", reg_02.raw);
1662 printk(KERN_DEBUG "....... : arbitration: %02X\n", reg_02.bits.arbitration);
1da177e4
LT		 1663 }
1664
1665 /*
1666 * Some Intel chipsets with IO APIC VERSION of 0x2? don't have reg_02
1667 * or reg_03, but the value of reg_0[23] is read as the previous read
1668 * register value, so ignore it if reg_03 == reg_0[12].
1669 */
1670 if (reg_01.bits.version >= 0x20 && reg_03.raw != reg_02.raw &&
1671 reg_03.raw != reg_01.raw) {
1672 printk(KERN_DEBUG ".... register #03: %08X\n", reg_03.raw);
1673 printk(KERN_DEBUG "....... : Boot DT : %X\n", reg_03.bits.boot_DT);
1da177e4
LT		 1674 }
1675
1676 printk(KERN_DEBUG ".... IRQ redirection table:\n");
1677
d83e94ac
YL		 1678 printk(KERN_DEBUG " NR Dst Mask Trig IRR Pol"
1679 " Stat Dmod Deli Vect: \n");
1da177e4
LT		 1680
1681 for (i = 0; i <= reg_01.bits.entries; i++) {
1682 struct IO_APIC_route_entry entry;
1683
cf4c6a2f 1684 entry = ioapic_read_entry(apic, i);
1da177e4 1685
54168ed7
IM		 1686 printk(KERN_DEBUG " %02x %03X ",
1687 i,
1688 entry.dest
1689 );
1da177e4
LT		 1690
1691 printk("%1d %1d %1d %1d %1d %1d %1d %02X\n",
1692 entry.mask,
1693 entry.trigger,
1694 entry.irr,
1695 entry.polarity,
1696 entry.delivery_status,
1697 entry.dest_mode,
1698 entry.delivery_mode,
1699 entry.vector
1700 );
1701 }
1702 }
1da177e4 1703 printk(KERN_DEBUG "IRQ to pin mappings:\n");
8f09cd20 1704 for_each_irq_cfg(irq, cfg) {
0f978f45
YL		 1705 struct irq_pin_list *entry = cfg->irq_2_pin;
1706 if (!entry)
1da177e4 1707 continue;
8f09cd20 1708 printk(KERN_DEBUG "IRQ%d ", irq);
1da177e4
LT		 1709 for (;;) {
1710 printk("-> %d:%d", entry->apic, entry->pin);
1711 if (!entry->next)
1712 break;
0f978f45 1713 entry = entry->next;
1da177e4
LT		 1714 }
1715 printk("\n");
1716 }
1717
1718 printk(KERN_INFO ".................................... done.\n");
1719
1720 return;
1721}
1722
32f71aff 1723__apicdebuginit(void) print_APIC_bitfield(int base)
1da177e4
LT		 1724{
1725 unsigned int v;
1726 int i, j;
1727
1728 if (apic_verbosity == APIC_QUIET)
1729 return;
1730
1731 printk(KERN_DEBUG "0123456789abcdef0123456789abcdef\n" KERN_DEBUG);
1732 for (i = 0; i < 8; i++) {
1733 v = apic_read(base + i*0x10);
1734 for (j = 0; j < 32; j++) {
1735 if (v & (1<<j))
1736 printk("1");
1737 else
1738 printk("0");
1739 }
1740 printk("\n");
1741 }
1742}
1743
32f71aff 1744__apicdebuginit(void) print_local_APIC(void *dummy)
1da177e4
LT		 1745{
1746 unsigned int v, ver, maxlvt;
7ab6af7a 1747 u64 icr;
1da177e4
LT		 1748
1749 if (apic_verbosity == APIC_QUIET)
1750 return;
1751
1752 printk("\n" KERN_DEBUG "printing local APIC contents on CPU#%d/%d:\n",
1753 smp_processor_id(), hard_smp_processor_id());
66823114 1754 v = apic_read(APIC_ID);
54168ed7 1755 printk(KERN_INFO "... APIC ID: %08x (%01x)\n", v, read_apic_id());
1da177e4
LT		 1756 v = apic_read(APIC_LVR);
1757 printk(KERN_INFO "... APIC VERSION: %08x\n", v);
1758 ver = GET_APIC_VERSION(v);
e05d723f 1759 maxlvt = lapic_get_maxlvt();
1da177e4
LT		 1760
1761 v = apic_read(APIC_TASKPRI);
1762 printk(KERN_DEBUG "... APIC TASKPRI: %08x (%02x)\n", v, v & APIC_TPRI_MASK);
1763
54168ed7 1764 if (APIC_INTEGRATED(ver)) { /* !82489DX */
a11b5abe
YL		 1765 if (!APIC_XAPIC(ver)) {
1766 v = apic_read(APIC_ARBPRI);
1767 printk(KERN_DEBUG "... APIC ARBPRI: %08x (%02x)\n", v,
1768 v & APIC_ARBPRI_MASK);
1769 }
1da177e4
LT		 1770 v = apic_read(APIC_PROCPRI);
1771 printk(KERN_DEBUG "... APIC PROCPRI: %08x\n", v);
1772 }
1773
a11b5abe
YL		 1774 /*
1775 * Remote read supported only in the 82489DX and local APIC for
1776 * Pentium processors.
1777 */
1778 if (!APIC_INTEGRATED(ver) || maxlvt == 3) {
1779 v = apic_read(APIC_RRR);
1780 printk(KERN_DEBUG "... APIC RRR: %08x\n", v);
1781 }
1782
1da177e4
LT		 1783 v = apic_read(APIC_LDR);
1784 printk(KERN_DEBUG "... APIC LDR: %08x\n", v);
a11b5abe
YL		 1785 if (!x2apic_enabled()) {
1786 v = apic_read(APIC_DFR);
1787 printk(KERN_DEBUG "... APIC DFR: %08x\n", v);
1788 }
1da177e4
LT		 1789 v = apic_read(APIC_SPIV);
1790 printk(KERN_DEBUG "... APIC SPIV: %08x\n", v);
1791
1792 printk(KERN_DEBUG "... APIC ISR field:\n");
1793 print_APIC_bitfield(APIC_ISR);
1794 printk(KERN_DEBUG "... APIC TMR field:\n");
1795 print_APIC_bitfield(APIC_TMR);
1796 printk(KERN_DEBUG "... APIC IRR field:\n");
1797 print_APIC_bitfield(APIC_IRR);
1798
54168ed7
IM		 1799 if (APIC_INTEGRATED(ver)) { /* !82489DX */
1800 if (maxlvt > 3) /* Due to the Pentium erratum 3AP. */
1da177e4 1801 apic_write(APIC_ESR, 0);
54168ed7 1802
1da177e4
LT		 1803 v = apic_read(APIC_ESR);
1804 printk(KERN_DEBUG "... APIC ESR: %08x\n", v);
1805 }
1806
7ab6af7a 1807 icr = apic_icr_read();
0c425cec
IM		 1808 printk(KERN_DEBUG "... APIC ICR: %08x\n", (u32)icr);
1809 printk(KERN_DEBUG "... APIC ICR2: %08x\n", (u32)(icr >> 32));
1da177e4
LT		 1810
1811 v = apic_read(APIC_LVTT);
1812 printk(KERN_DEBUG "... APIC LVTT: %08x\n", v);
1813
1814 if (maxlvt > 3) { /* PC is LVT#4. */
1815 v = apic_read(APIC_LVTPC);
1816 printk(KERN_DEBUG "... APIC LVTPC: %08x\n", v);
1817 }
1818 v = apic_read(APIC_LVT0);
1819 printk(KERN_DEBUG "... APIC LVT0: %08x\n", v);
1820 v = apic_read(APIC_LVT1);
1821 printk(KERN_DEBUG "... APIC LVT1: %08x\n", v);
1822
1823 if (maxlvt > 2) { /* ERR is LVT#3. */
1824 v = apic_read(APIC_LVTERR);
1825 printk(KERN_DEBUG "... APIC LVTERR: %08x\n", v);
1826 }
1827
1828 v = apic_read(APIC_TMICT);
1829 printk(KERN_DEBUG "... APIC TMICT: %08x\n", v);
1830 v = apic_read(APIC_TMCCT);
1831 printk(KERN_DEBUG "... APIC TMCCT: %08x\n", v);
1832 v = apic_read(APIC_TDCR);
1833 printk(KERN_DEBUG "... APIC TDCR: %08x\n", v);
1834 printk("\n");
1835}
1836
32f71aff 1837__apicdebuginit(void) print_all_local_APICs(void)
1da177e4 1838{
ffd5aae7
YL		 1839 int cpu;
1840
1841 preempt_disable();
1842 for_each_online_cpu(cpu)
1843 smp_call_function_single(cpu, print_local_APIC, NULL, 1);
1844 preempt_enable();
1da177e4
LT		 1845}
1846
32f71aff 1847__apicdebuginit(void) print_PIC(void)
1da177e4 1848{
1da177e4
LT		 1849 unsigned int v;
1850 unsigned long flags;
1851
1852 if (apic_verbosity == APIC_QUIET)
1853 return;
1854
1855 printk(KERN_DEBUG "\nprinting PIC contents\n");
1856
1857 spin_lock_irqsave(&i8259A_lock, flags);
1858
1859 v = inb(0xa1) << 8 | inb(0x21);
1860 printk(KERN_DEBUG "... PIC IMR: %04x\n", v);
1861
1862 v = inb(0xa0) << 8 | inb(0x20);
1863 printk(KERN_DEBUG "... PIC IRR: %04x\n", v);
1864
54168ed7
IM		 1865 outb(0x0b,0xa0);
1866 outb(0x0b,0x20);
1da177e4 1867 v = inb(0xa0) << 8 | inb(0x20);
54168ed7
IM		 1868 outb(0x0a,0xa0);
1869 outb(0x0a,0x20);
1da177e4
LT		 1870
1871 spin_unlock_irqrestore(&i8259A_lock, flags);
1872
1873 printk(KERN_DEBUG "... PIC ISR: %04x\n", v);
1874
1875 v = inb(0x4d1) << 8 | inb(0x4d0);
1876 printk(KERN_DEBUG "... PIC ELCR: %04x\n", v);
1877}
1878
32f71aff
MR		 1879__apicdebuginit(int) print_all_ICs(void)
1880{
1881 print_PIC();
1882 print_all_local_APICs();
1883 print_IO_APIC();
1884
1885 return 0;
1886}
1887
1888fs_initcall(print_all_ICs);
1889
1da177e4 1890
efa2559f
YL		 1891/* Where if anywhere is the i8259 connect in external int mode */
1892static struct { int pin, apic; } ioapic_i8259 = { -1, -1 };
1893
54168ed7 1894void __init enable_IO_APIC(void)
1da177e4
LT		 1895{
1896 union IO_APIC_reg_01 reg_01;
fcfd636a 1897 int i8259_apic, i8259_pin;
54168ed7 1898 int apic;
1da177e4
LT		 1899 unsigned long flags;
1900
54168ed7
IM		 1901#ifdef CONFIG_X86_32
1902 int i;
1da177e4
LT		 1903 if (!pirqs_enabled)
1904 for (i = 0; i < MAX_PIRQS; i++)
1905 pirq_entries[i] = -1;
54168ed7 1906#endif
1da177e4
LT		 1907
1908 /*
1909 * The number of IO-APIC IRQ registers (== #pins):
1910 */
fcfd636a 1911 for (apic = 0; apic < nr_ioapics; apic++) {
1da177e4 1912 spin_lock_irqsave(&ioapic_lock, flags);
fcfd636a 1913 reg_01.raw = io_apic_read(apic, 1);
1da177e4 1914 spin_unlock_irqrestore(&ioapic_lock, flags);
fcfd636a
EB		 1915 nr_ioapic_registers[apic] = reg_01.bits.entries+1;
1916 }
54168ed7 1917 for(apic = 0; apic < nr_ioapics; apic++) {
fcfd636a
EB		 1918 int pin;
1919 /* See if any of the pins is in ExtINT mode */
1008fddc 1920 for (pin = 0; pin < nr_ioapic_registers[apic]; pin++) {
fcfd636a 1921 struct IO_APIC_route_entry entry;
cf4c6a2f 1922 entry = ioapic_read_entry(apic, pin);
fcfd636a 1923
fcfd636a
EB		 1924 /* If the interrupt line is enabled and in ExtInt mode
1925 * I have found the pin where the i8259 is connected.
1926 */
1927 if ((entry.mask == 0) && (entry.delivery_mode == dest_ExtINT)) {
1928 ioapic_i8259.apic = apic;
1929 ioapic_i8259.pin = pin;
1930 goto found_i8259;
1931 }
1932 }
1933 }
1934 found_i8259:
1935 /* Look to see what if the MP table has reported the ExtINT */
1936 /* If we could not find the appropriate pin by looking at the ioapic
1937 * the i8259 probably is not connected the ioapic but give the
1938 * mptable a chance anyway.
1939 */
1940 i8259_pin = find_isa_irq_pin(0, mp_ExtINT);
1941 i8259_apic = find_isa_irq_apic(0, mp_ExtINT);
1942 /* Trust the MP table if nothing is setup in the hardware */
1943 if ((ioapic_i8259.pin == -1) && (i8259_pin >= 0)) {
1944 printk(KERN_WARNING "ExtINT not setup in hardware but reported by MP table\n");
1945 ioapic_i8259.pin = i8259_pin;
1946 ioapic_i8259.apic = i8259_apic;
1947 }
1948 /* Complain if the MP table and the hardware disagree */
1949 if (((ioapic_i8259.apic != i8259_apic) || (ioapic_i8259.pin != i8259_pin)) &&
1950 (i8259_pin >= 0) && (ioapic_i8259.pin >= 0))
1951 {
1952 printk(KERN_WARNING "ExtINT in hardware and MP table differ\n");
1da177e4
LT		 1953 }
1954
1955 /*
1956 * Do not trust the IO-APIC being empty at bootup
1957 */
1958 clear_IO_APIC();
1959}
1960
1961/*
1962 * Not an __init, needed by the reboot code
1963 */
1964void disable_IO_APIC(void)
1965{
1966 /*
1967 * Clear the IO-APIC before rebooting:
1968 */
1969 clear_IO_APIC();
1970
650927ef 1971 /*
0b968d23 1972 * If the i8259 is routed through an IOAPIC
650927ef 1973 * Put that IOAPIC in virtual wire mode
0b968d23 1974 * so legacy interrupts can be delivered.
650927ef 1975 */
fcfd636a 1976 if (ioapic_i8259.pin != -1) {
650927ef 1977 struct IO_APIC_route_entry entry;
650927ef
EB		 1978
1979 memset(&entry, 0, sizeof(entry));
1980 entry.mask = 0; /* Enabled */
1981 entry.trigger = 0; /* Edge */
1982 entry.irr = 0;
1983 entry.polarity = 0; /* High */
1984 entry.delivery_status = 0;
1985 entry.dest_mode = 0; /* Physical */
fcfd636a 1986 entry.delivery_mode = dest_ExtINT; /* ExtInt */
650927ef 1987 entry.vector = 0;
54168ed7 1988 entry.dest = read_apic_id();
650927ef
EB		 1989
1990 /*
1991 * Add it to the IO-APIC irq-routing table:
1992 */
cf4c6a2f 1993 ioapic_write_entry(ioapic_i8259.apic, ioapic_i8259.pin, entry);
650927ef 1994 }
54168ed7 1995
fcfd636a 1996 disconnect_bsp_APIC(ioapic_i8259.pin != -1);
1da177e4
LT		 1997}
1998
54168ed7 1999#ifdef CONFIG_X86_32
1da177e4
LT		 2000/*
2001 * function to set the IO-APIC physical IDs based on the
2002 * values stored in the MPC table.
2003 *
2004 * by Matt Domsch <Matt_Domsch@dell.com> Tue Dec 21 12:25:05 CST 1999
2005 */
2006
1da177e4
LT		 2007static void __init setup_ioapic_ids_from_mpc(void)
2008{
2009 union IO_APIC_reg_00 reg_00;
2010 physid_mask_t phys_id_present_map;
2011 int apic;
2012 int i;
2013 unsigned char old_id;
2014 unsigned long flags;
2015
a4dbc34d 2016 if (x86_quirks->setup_ioapic_ids && x86_quirks->setup_ioapic_ids())
d49c4288 2017 return;
d49c4288 2018
ca05fea6
NP		 2019 /*
2020 * Don't check I/O APIC IDs for xAPIC systems. They have
2021 * no meaning without the serial APIC bus.
2022 */
7c5c1e42
SL		 2023 if (!(boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
2024 || APIC_XAPIC(apic_version[boot_cpu_physical_apicid]))
ca05fea6 2025 return;
1da177e4
LT		 2026 /*
2027 * This is broken; anything with a real cpu count has to
2028 * circumvent this idiocy regardless.
2029 */
2030 phys_id_present_map = ioapic_phys_id_map(phys_cpu_present_map);
2031
2032 /*
2033 * Set the IOAPIC ID to the value stored in the MPC table.
2034 */
2035 for (apic = 0; apic < nr_ioapics; apic++) {
2036
2037 /* Read the register 0 value */
2038 spin_lock_irqsave(&ioapic_lock, flags);
2039 reg_00.raw = io_apic_read(apic, 0);
2040 spin_unlock_irqrestore(&ioapic_lock, flags);
36062448 2041
ec2cd0a2 2042 old_id = mp_ioapics[apic].mp_apicid;
1da177e4 2043
ec2cd0a2 2044 if (mp_ioapics[apic].mp_apicid >= get_physical_broadcast()) {
1da177e4 2045 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID is %d in the MPC table!...\n",
ec2cd0a2 2046 apic, mp_ioapics[apic].mp_apicid);
1da177e4
LT		 2047 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
2048 reg_00.bits.ID);
ec2cd0a2 2049 mp_ioapics[apic].mp_apicid = reg_00.bits.ID;
1da177e4
LT		 2050 }
2051
1da177e4
LT		 2052 /*
2053 * Sanity check, is the ID really free? Every APIC in a
2054 * system must have a unique ID or we get lots of nice
2055 * 'stuck on smp_invalidate_needed IPI wait' messages.
2056 */
2057 if (check_apicid_used(phys_id_present_map,
ec2cd0a2 2058 mp_ioapics[apic].mp_apicid)) {
1da177e4 2059 printk(KERN_ERR "BIOS bug, IO-APIC#%d ID %d is already used!...\n",
ec2cd0a2 2060 apic, mp_ioapics[apic].mp_apicid);
1da177e4
LT		 2061 for (i = 0; i < get_physical_broadcast(); i++)
2062 if (!physid_isset(i, phys_id_present_map))
2063 break;
2064 if (i >= get_physical_broadcast())
2065 panic("Max APIC ID exceeded!\n");
2066 printk(KERN_ERR "... fixing up to %d. (tell your hw vendor)\n",
2067 i);
2068 physid_set(i, phys_id_present_map);
ec2cd0a2 2069 mp_ioapics[apic].mp_apicid = i;
1da177e4
LT		 2070 } else {
2071 physid_mask_t tmp;
ec2cd0a2 2072 tmp = apicid_to_cpu_present(mp_ioapics[apic].mp_apicid);
1da177e4
LT		 2073 apic_printk(APIC_VERBOSE, "Setting %d in the "
2074 "phys_id_present_map\n",
ec2cd0a2 2075 mp_ioapics[apic].mp_apicid);
1da177e4
LT		 2076 physids_or(phys_id_present_map, phys_id_present_map, tmp);
2077 }
2078
2079
2080 /*
2081 * We need to adjust the IRQ routing table
2082 * if the ID changed.
2083 */
ec2cd0a2 2084 if (old_id != mp_ioapics[apic].mp_apicid)
1da177e4 2085 for (i = 0; i < mp_irq_entries; i++)
2fddb6e2
AS		 2086 if (mp_irqs[i].mp_dstapic == old_id)
2087 mp_irqs[i].mp_dstapic
ec2cd0a2 2088 = mp_ioapics[apic].mp_apicid;
1da177e4
LT		 2089
2090 /*
2091 * Read the right value from the MPC table and
2092 * write it into the ID register.
36062448 2093 */
1da177e4
LT		 2094 apic_printk(APIC_VERBOSE, KERN_INFO
2095 "...changing IO-APIC physical APIC ID to %d ...",
ec2cd0a2 2096 mp_ioapics[apic].mp_apicid);
1da177e4 2097
ec2cd0a2 2098 reg_00.bits.ID = mp_ioapics[apic].mp_apicid;
1da177e4 2099 spin_lock_irqsave(&ioapic_lock, flags);
a2d332fa
YL		 2100 io_apic_write(apic, 0, reg_00.raw);
2101 spin_unlock_irqrestore(&ioapic_lock, flags);
1da177e4
LT		 2102
2103 /*
2104 * Sanity check
2105 */
2106 spin_lock_irqsave(&ioapic_lock, flags);
2107 reg_00.raw = io_apic_read(apic, 0);
2108 spin_unlock_irqrestore(&ioapic_lock, flags);
ec2cd0a2 2109 if (reg_00.bits.ID != mp_ioapics[apic].mp_apicid)
1da177e4
LT		 2110 printk("could not set ID!\n");
2111 else
2112 apic_printk(APIC_VERBOSE, " ok.\n");
2113 }
2114}
54168ed7 2115#endif
1da177e4 2116
7ce0bcfd 2117int no_timer_check __initdata;
8542b200
ZA		 2118
2119static int __init notimercheck(char *s)
2120{
2121 no_timer_check = 1;
2122 return 1;
2123}
2124__setup("no_timer_check", notimercheck);
2125
1da177e4
LT		 2126/*
2127 * There is a nasty bug in some older SMP boards, their mptable lies
2128 * about the timer IRQ. We do the following to work around the situation:
2129 *
2130 * - timer IRQ defaults to IO-APIC IRQ
2131 * - if this function detects that timer IRQs are defunct, then we fall
2132 * back to ISA timer IRQs
2133 */
f0a7a5c9 2134static int __init timer_irq_works(void)
1da177e4
LT		 2135{
2136 unsigned long t1 = jiffies;
4aae0702 2137 unsigned long flags;
1da177e4 2138
8542b200
ZA		 2139 if (no_timer_check)
2140 return 1;
2141
4aae0702 2142 local_save_flags(flags);
1da177e4
LT		 2143 local_irq_enable();
2144 /* Let ten ticks pass... */
2145 mdelay((10 * 1000) / HZ);
4aae0702 2146 local_irq_restore(flags);
1da177e4
LT		 2147
2148 /*
2149 * Expect a few ticks at least, to be sure some possible
2150 * glue logic does not lock up after one or two first
2151 * ticks in a non-ExtINT mode. Also the local APIC
2152 * might have cached one ExtINT interrupt. Finally, at
2153 * least one tick may be lost due to delays.
2154 */
54168ed7
IM		 2155
2156 /* jiffies wrap? */
1d16b53e 2157 if (time_after(jiffies, t1 + 4))
1da177e4 2158 return 1;
1da177e4
LT		 2159 return 0;
2160}
2161
2162/*
2163 * In the SMP+IOAPIC case it might happen that there are an unspecified
2164 * number of pending IRQ events unhandled. These cases are very rare,
2165 * so we 'resend' these IRQs via IPIs, to the same CPU. It's much
2166 * better to do it this way as thus we do not have to be aware of
2167 * 'pending' interrupts in the IRQ path, except at this point.
2168 */
2169/*
2170 * Edge triggered needs to resend any interrupt
2171 * that was delayed but this is now handled in the device
2172 * independent code.
2173 */
2174
2175/*
2176 * Starting up a edge-triggered IO-APIC interrupt is
2177 * nasty - we need to make sure that we get the edge.
2178 * If it is already asserted for some reason, we need
2179 * return 1 to indicate that is was pending.
2180 *
2181 * This is not complete - we should be able to fake
2182 * an edge even if it isn't on the 8259A...
2183 */
54168ed7 2184
f5b9ed7a 2185static unsigned int startup_ioapic_irq(unsigned int irq)
1da177e4
LT		 2186{
2187 int was_pending = 0;
2188 unsigned long flags;
2189
2190 spin_lock_irqsave(&ioapic_lock, flags);
2191 if (irq < 16) {
2192 disable_8259A_irq(irq);
2193 if (i8259A_irq_pending(irq))
2194 was_pending = 1;
2195 }
2196 __unmask_IO_APIC_irq(irq);
2197 spin_unlock_irqrestore(&ioapic_lock, flags);
2198
2199 return was_pending;
2200}
2201
54168ed7 2202#ifdef CONFIG_X86_64
ace80ab7 2203static int ioapic_retrigger_irq(unsigned int irq)
1da177e4 2204{
54168ed7
IM		 2205
2206 struct irq_cfg *cfg = irq_cfg(irq);
2207 unsigned long flags;
2208
2209 spin_lock_irqsave(&vector_lock, flags);
2210 send_IPI_mask(cpumask_of_cpu(first_cpu(cfg->domain)), cfg->vector);
2211 spin_unlock_irqrestore(&vector_lock, flags);
c0ad90a3
IM		 2212
2213 return 1;
2214}
54168ed7
IM		 2215#else
2216static int ioapic_retrigger_irq(unsigned int irq)
497c9a19 2217{
54168ed7 2218 send_IPI_self(irq_cfg(irq)->vector);
497c9a19 2219
54168ed7
IM		 2220 return 1;
2221}
2222#endif
497c9a19 2223
54168ed7
IM		 2224/*
2225 * Level and edge triggered IO-APIC interrupts need different handling,
2226 * so we use two separate IRQ descriptors. Edge triggered IRQs can be
2227 * handled with the level-triggered descriptor, but that one has slightly
2228 * more overhead. Level-triggered interrupts cannot be handled with the
2229 * edge-triggered handler, without risking IRQ storms and other ugly
2230 * races.
2231 */
497c9a19 2232
54168ed7 2233#ifdef CONFIG_SMP
497c9a19 2234
54168ed7
IM		 2235#ifdef CONFIG_INTR_REMAP
2236static void ir_irq_migration(struct work_struct *work);
497c9a19 2237
54168ed7 2238static DECLARE_DELAYED_WORK(ir_migration_work, ir_irq_migration);
497c9a19 2239
54168ed7
IM		 2240/*
2241 * Migrate the IO-APIC irq in the presence of intr-remapping.
2242 *
2243 * For edge triggered, irq migration is a simple atomic update(of vector
2244 * and cpu destination) of IRTE and flush the hardware cache.
2245 *
2246 * For level triggered, we need to modify the io-apic RTE aswell with the update
2247 * vector information, along with modifying IRTE with vector and destination.
2248 * So irq migration for level triggered is little bit more complex compared to
2249 * edge triggered migration. But the good news is, we use the same algorithm
2250 * for level triggered migration as we have today, only difference being,
2251 * we now initiate the irq migration from process context instead of the
2252 * interrupt context.
2253 *
2254 * In future, when we do a directed EOI (combined with cpu EOI broadcast
2255 * suppression) to the IO-APIC, level triggered irq migration will also be
2256 * as simple as edge triggered migration and we can do the irq migration
2257 * with a simple atomic update to IO-APIC RTE.
2258 */
2259static void migrate_ioapic_irq(int irq, cpumask_t mask)
497c9a19 2260{
54168ed7
IM		 2261 struct irq_cfg *cfg;
2262 struct irq_desc *desc;
2263 cpumask_t tmp, cleanup_mask;
2264 struct irte irte;
2265 int modify_ioapic_rte;
2266 unsigned int dest;
2267 unsigned long flags;
497c9a19 2268
54168ed7
IM		 2269 cpus_and(tmp, mask, cpu_online_map);
2270 if (cpus_empty(tmp))
497c9a19
YL		 2271 return;
2272
54168ed7
IM		 2273 if (get_irte(irq, &irte))
2274 return;
497c9a19 2275
54168ed7
IM		 2276 if (assign_irq_vector(irq, mask))
2277 return;
2278
2279 cfg = irq_cfg(irq);
2280 cpus_and(tmp, cfg->domain, mask);
2281 dest = cpu_mask_to_apicid(tmp);
2282
2283 desc = irq_to_desc(irq);
2284 modify_ioapic_rte = desc->status & IRQ_LEVEL;
2285 if (modify_ioapic_rte) {
2286 spin_lock_irqsave(&ioapic_lock, flags);
2287 __target_IO_APIC_irq(irq, dest, cfg->vector);
2288 spin_unlock_irqrestore(&ioapic_lock, flags);
2289 }
2290
2291 irte.vector = cfg->vector;
2292 irte.dest_id = IRTE_DEST(dest);
2293
2294 /*
2295 * Modified the IRTE and flushes the Interrupt entry cache.
2296 */
2297 modify_irte(irq, &irte);
2298
2299 if (cfg->move_in_progress) {
2300 cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map);
2301 cfg->move_cleanup_count = cpus_weight(cleanup_mask);
2302 send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
2303 cfg->move_in_progress = 0;
2304 }
2305
2306 desc->affinity = mask;
2307}
2308
2309static int migrate_irq_remapped_level(int irq)
2310{
2311 int ret = -1;
2312 struct irq_desc *desc = irq_to_desc(irq);
2313
2314 mask_IO_APIC_irq(irq);
2315
2316 if (io_apic_level_ack_pending(irq)) {
2317 /*
2318 * Interrupt in progress. Migrating irq now will change the
2319 * vector information in the IO-APIC RTE and that will confuse
2320 * the EOI broadcast performed by cpu.
2321 * So, delay the irq migration to the next instance.
2322 */
2323 schedule_delayed_work(&ir_migration_work, 1);
2324 goto unmask;
2325 }
2326
2327 /* everthing is clear. we have right of way */
2328 migrate_ioapic_irq(irq, desc->pending_mask);
2329
2330 ret = 0;
2331 desc->status &= ~IRQ_MOVE_PENDING;
2332 cpus_clear(desc->pending_mask);
2333
2334unmask:
2335 unmask_IO_APIC_irq(irq);
2336 return ret;
2337}
2338
2339static void ir_irq_migration(struct work_struct *work)
2340{
2341 unsigned int irq;
2342 struct irq_desc *desc;
2343
2344 for_each_irq_desc(irq, desc) {
2345 if (desc->status & IRQ_MOVE_PENDING) {
2346 unsigned long flags;
2347
2348 spin_lock_irqsave(&desc->lock, flags);
2349 if (!desc->chip->set_affinity ||
2350 !(desc->status & IRQ_MOVE_PENDING)) {
2351 desc->status &= ~IRQ_MOVE_PENDING;
2352 spin_unlock_irqrestore(&desc->lock, flags);
2353 continue;
2354 }
2355
2356 desc->chip->set_affinity(irq, desc->pending_mask);
2357 spin_unlock_irqrestore(&desc->lock, flags);
2358 }
2359 }
2360}
2361
2362/*
2363 * Migrates the IRQ destination in the process context.
2364 */
2365static void set_ir_ioapic_affinity_irq(unsigned int irq, cpumask_t mask)
2366{
2367 struct irq_desc *desc = irq_to_desc(irq);
2368
2369 if (desc->status & IRQ_LEVEL) {
2370 desc->status |= IRQ_MOVE_PENDING;
2371 desc->pending_mask = mask;
2372 migrate_irq_remapped_level(irq);
2373 return;
2374 }
2375
2376 migrate_ioapic_irq(irq, mask);
2377}
2378#endif
2379
2380asmlinkage void smp_irq_move_cleanup_interrupt(void)
2381{
2382 unsigned vector, me;
2383 ack_APIC_irq();
2384#ifdef CONFIG_X86_64
2385 exit_idle();
2386#endif
2387 irq_enter();
2388
2389 me = smp_processor_id();
2390 for (vector = FIRST_EXTERNAL_VECTOR; vector < NR_VECTORS; vector++) {
2391 unsigned int irq;
2392 struct irq_desc *desc;
2393 struct irq_cfg *cfg;
2394 irq = __get_cpu_var(vector_irq)[vector];
2395
2396 desc = irq_to_desc(irq);
2397 if (!desc)
2398 continue;
2399
2400 cfg = irq_cfg(irq);
2401 spin_lock(&desc->lock);
2402 if (!cfg->move_cleanup_count)
2403 goto unlock;
2404
2405 if ((vector == cfg->vector) && cpu_isset(me, cfg->domain))
2406 goto unlock;
2407
2408 __get_cpu_var(vector_irq)[vector] = -1;
2409 cfg->move_cleanup_count--;
2410unlock:
2411 spin_unlock(&desc->lock);
2412 }
2413
2414 irq_exit();
2415}
2416
2417static void irq_complete_move(unsigned int irq)
2418{
2419 struct irq_cfg *cfg = irq_cfg(irq);
2420 unsigned vector, me;
2421
2422 if (likely(!cfg->move_in_progress))
2423 return;
2424
2425 vector = ~get_irq_regs()->orig_ax;
2426 me = smp_processor_id();
2427 if ((vector == cfg->vector) && cpu_isset(me, cfg->domain)) {
2428 cpumask_t cleanup_mask;
2429
2430 cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map);
2431 cfg->move_cleanup_count = cpus_weight(cleanup_mask);
2432 send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
497c9a19
YL		 2433 cfg->move_in_progress = 0;
2434 }
2435}
2436#else
2437static inline void irq_complete_move(unsigned int irq) {}
2438#endif
54168ed7
IM		 2439#ifdef CONFIG_INTR_REMAP
2440static void ack_x2apic_level(unsigned int irq)
2441{
2442 ack_x2APIC_irq();
2443}
2444
2445static void ack_x2apic_edge(unsigned int irq)
2446{
2447 ack_x2APIC_irq();
2448}
2449#endif
497c9a19 2450
1d025192
YL		 2451static void ack_apic_edge(unsigned int irq)
2452{
2453 irq_complete_move(irq);
2454 move_native_irq(irq);
2455 ack_APIC_irq();
2456}
2457
3eb2cce8
YL		 2458#ifdef CONFIG_X86_32
2459atomic_t irq_mis_count;
2460#endif
2461
047c8fdb
YL		 2462static void ack_apic_level(unsigned int irq)
2463{
3eb2cce8
YL		 2464#ifdef CONFIG_X86_32
2465 unsigned long v;
2466 int i;
2467#endif
54168ed7 2468 int do_unmask_irq = 0;
047c8fdb 2469
54168ed7 2470 irq_complete_move(irq);
047c8fdb 2471#ifdef CONFIG_GENERIC_PENDING_IRQ
54168ed7
IM		 2472 /* If we are moving the irq we need to mask it */
2473 if (unlikely(irq_to_desc(irq)->status & IRQ_MOVE_PENDING)) {
2474 do_unmask_irq = 1;
2475 mask_IO_APIC_irq(irq);
2476 }
047c8fdb
YL		 2477#endif
2478
3eb2cce8
YL		 2479#ifdef CONFIG_X86_32
2480 /*
2481 * It appears there is an erratum which affects at least version 0x11
2482 * of I/O APIC (that's the 82093AA and cores integrated into various
2483 * chipsets). Under certain conditions a level-triggered interrupt is
2484 * erroneously delivered as edge-triggered one but the respective IRR
2485 * bit gets set nevertheless. As a result the I/O unit expects an EOI
2486 * message but it will never arrive and further interrupts are blocked
2487 * from the source. The exact reason is so far unknown, but the
2488 * phenomenon was observed when two consecutive interrupt requests
2489 * from a given source get delivered to the same CPU and the source is
2490 * temporarily disabled in between.
2491 *
2492 * A workaround is to simulate an EOI message manually. We achieve it
2493 * by setting the trigger mode to edge and then to level when the edge
2494 * trigger mode gets detected in the TMR of a local APIC for a
2495 * level-triggered interrupt. We mask the source for the time of the
2496 * operation to prevent an edge-triggered interrupt escaping meanwhile.
2497 * The idea is from Manfred Spraul. --macro
2498 */
2499 i = irq_cfg(irq)->vector;
2500
2501 v = apic_read(APIC_TMR + ((i & ~0x1f) >> 1));
2502#endif
2503
54168ed7
IM		 2504 /*
2505 * We must acknowledge the irq before we move it or the acknowledge will
2506 * not propagate properly.
2507 */
2508 ack_APIC_irq();
2509
2510 /* Now we can move and renable the irq */
2511 if (unlikely(do_unmask_irq)) {
2512 /* Only migrate the irq if the ack has been received.
2513 *
2514 * On rare occasions the broadcast level triggered ack gets
2515 * delayed going to ioapics, and if we reprogram the
2516 * vector while Remote IRR is still set the irq will never
2517 * fire again.
2518 *
2519 * To prevent this scenario we read the Remote IRR bit
2520 * of the ioapic. This has two effects.
2521 * - On any sane system the read of the ioapic will
2522 * flush writes (and acks) going to the ioapic from
2523 * this cpu.
2524 * - We get to see if the ACK has actually been delivered.
2525 *
2526 * Based on failed experiments of reprogramming the
2527 * ioapic entry from outside of irq context starting
2528 * with masking the ioapic entry and then polling until
2529 * Remote IRR was clear before reprogramming the
2530 * ioapic I don't trust the Remote IRR bit to be
2531 * completey accurate.
2532 *
2533 * However there appears to be no other way to plug
2534 * this race, so if the Remote IRR bit is not
2535 * accurate and is causing problems then it is a hardware bug
2536 * and you can go talk to the chipset vendor about it.
2537 */
2538 if (!io_apic_level_ack_pending(irq))
2539 move_masked_irq(irq);
2540 unmask_IO_APIC_irq(irq);
2541 }
1d025192 2542
3eb2cce8 2543#ifdef CONFIG_X86_32
1d025192
YL		 2544 if (!(v & (1 << (i & 0x1f)))) {
2545 atomic_inc(&irq_mis_count);
2546 spin_lock(&ioapic_lock);
2547 __mask_and_edge_IO_APIC_irq(irq);
2548 __unmask_and_level_IO_APIC_irq(irq);
2549 spin_unlock(&ioapic_lock);
2550 }
047c8fdb 2551#endif
3eb2cce8 2552}
1d025192 2553
f5b9ed7a
IM		 2554static struct irq_chip ioapic_chip __read_mostly = {
2555 .name = "IO-APIC",
ace80ab7
EB		 2556 .startup = startup_ioapic_irq,
2557 .mask = mask_IO_APIC_irq,
2558 .unmask = unmask_IO_APIC_irq,
1d025192
YL		 2559 .ack = ack_apic_edge,
2560 .eoi = ack_apic_level,
54d5d424 2561#ifdef CONFIG_SMP
ace80ab7 2562 .set_affinity = set_ioapic_affinity_irq,
54d5d424 2563#endif
ace80ab7 2564 .retrigger = ioapic_retrigger_irq,
1da177e4
LT		 2565};
2566
54168ed7
IM		 2567#ifdef CONFIG_INTR_REMAP
2568static struct irq_chip ir_ioapic_chip __read_mostly = {
2569 .name = "IR-IO-APIC",
2570 .startup = startup_ioapic_irq,
2571 .mask = mask_IO_APIC_irq,
2572 .unmask = unmask_IO_APIC_irq,
2573 .ack = ack_x2apic_edge,
2574 .eoi = ack_x2apic_level,
2575#ifdef CONFIG_SMP
2576 .set_affinity = set_ir_ioapic_affinity_irq,
2577#endif
2578 .retrigger = ioapic_retrigger_irq,
2579};
2580#endif
1da177e4
LT		 2581
2582static inline void init_IO_APIC_traps(void)
2583{
2584 int irq;
08678b08 2585 struct irq_desc *desc;
da51a821 2586 struct irq_cfg *cfg;
1da177e4
LT		 2587
2588 /*
2589 * NOTE! The local APIC isn't very good at handling
2590 * multiple interrupts at the same interrupt level.
2591 * As the interrupt level is determined by taking the
2592 * vector number and shifting that right by 4, we
2593 * want to spread these out a bit so that they don't
2594 * all fall in the same interrupt level.
2595 *
2596 * Also, we've got to be careful not to trash gate
2597 * 0x80, because int 0x80 is hm, kind of importantish. ;)
2598 */
8f09cd20 2599 for_each_irq_cfg(irq, cfg) {
da51a821 2600 if (IO_APIC_IRQ(irq) && !cfg->vector) {
1da177e4
LT		 2601 /*
2602 * Hmm.. We don't have an entry for this,
2603 * so default to an old-fashioned 8259
2604 * interrupt if we can..
2605 */
2606 if (irq < 16)
2607 make_8259A_irq(irq);
08678b08
YL		 2608 else {
2609 desc = irq_to_desc(irq);
1da177e4 2610 /* Strange. Oh, well.. */
08678b08
YL		 2611 desc->chip = &no_irq_chip;
2612 }
1da177e4
LT		 2613 }
2614 }
2615}
2616
f5b9ed7a
IM		 2617/*
2618 * The local APIC irq-chip implementation:
2619 */
1da177e4 2620
36062448 2621static void mask_lapic_irq(unsigned int irq)
1da177e4
LT		 2622{
2623 unsigned long v;
2624
2625 v = apic_read(APIC_LVT0);
593f4a78 2626 apic_write(APIC_LVT0, v | APIC_LVT_MASKED);
1da177e4
LT		 2627}
2628
36062448 2629static void unmask_lapic_irq(unsigned int irq)
1da177e4 2630{
f5b9ed7a 2631 unsigned long v;
1da177e4 2632
f5b9ed7a 2633 v = apic_read(APIC_LVT0);
593f4a78 2634 apic_write(APIC_LVT0, v & ~APIC_LVT_MASKED);
f5b9ed7a 2635}
1da177e4 2636
54168ed7 2637static void ack_lapic_irq (unsigned int irq)
1d025192
YL		 2638{
2639 ack_APIC_irq();
2640}
2641
f5b9ed7a 2642static struct irq_chip lapic_chip __read_mostly = {
9a1c6192 2643 .name = "local-APIC",
f5b9ed7a
IM		 2644 .mask = mask_lapic_irq,
2645 .unmask = unmask_lapic_irq,
c88ac1df 2646 .ack = ack_lapic_irq,
1da177e4
LT		 2647};
2648
497c9a19 2649static void lapic_register_intr(int irq)
c88ac1df 2650{
08678b08
YL		 2651 struct irq_desc *desc;
2652
2653 desc = irq_to_desc(irq);
2654 desc->status &= ~IRQ_LEVEL;
c88ac1df
MR		 2655 set_irq_chip_and_handler_name(irq, &lapic_chip, handle_edge_irq,
2656 "edge");
c88ac1df
MR		 2657}
2658
e9427101 2659static void __init setup_nmi(void)
1da177e4
LT		 2660{
2661 /*
36062448 2662 * Dirty trick to enable the NMI watchdog ...
1da177e4
LT		 2663 * We put the 8259A master into AEOI mode and
2664 * unmask on all local APICs LVT0 as NMI.
2665 *
2666 * The idea to use the 8259A in AEOI mode ('8259A Virtual Wire')
2667 * is from Maciej W. Rozycki - so we do not have to EOI from
2668 * the NMI handler or the timer interrupt.
36062448 2669 */
1da177e4
LT		 2670 apic_printk(APIC_VERBOSE, KERN_INFO "activating NMI Watchdog ...");
2671
e9427101 2672 enable_NMI_through_LVT0();
1da177e4
LT		 2673
2674 apic_printk(APIC_VERBOSE, " done.\n");
2675}
2676
2677/*
2678 * This looks a bit hackish but it's about the only one way of sending
2679 * a few INTA cycles to 8259As and any associated glue logic. ICR does
2680 * not support the ExtINT mode, unfortunately. We need to send these
2681 * cycles as some i82489DX-based boards have glue logic that keeps the
2682 * 8259A interrupt line asserted until INTA. --macro
2683 */
28acf285 2684static inline void __init unlock_ExtINT_logic(void)
1da177e4 2685{
fcfd636a 2686 int apic, pin, i;
1da177e4
LT		 2687 struct IO_APIC_route_entry entry0, entry1;
2688 unsigned char save_control, save_freq_select;
1da177e4 2689
fcfd636a 2690 pin = find_isa_irq_pin(8, mp_INT);
956fb531
AB		 2691 if (pin == -1) {
2692 WARN_ON_ONCE(1);
2693 return;
2694 }
fcfd636a 2695 apic = find_isa_irq_apic(8, mp_INT);
956fb531
AB		 2696 if (apic == -1) {
2697 WARN_ON_ONCE(1);
1da177e4 2698 return;
956fb531 2699 }
1da177e4 2700
cf4c6a2f 2701 entry0 = ioapic_read_entry(apic, pin);
fcfd636a 2702 clear_IO_APIC_pin(apic, pin);
1da177e4
LT		 2703
2704 memset(&entry1, 0, sizeof(entry1));
2705
2706 entry1.dest_mode = 0; /* physical delivery */
2707 entry1.mask = 0; /* unmask IRQ now */
d83e94ac 2708 entry1.dest = hard_smp_processor_id();
1da177e4
LT		 2709 entry1.delivery_mode = dest_ExtINT;
2710 entry1.polarity = entry0.polarity;
2711 entry1.trigger = 0;
2712 entry1.vector = 0;
2713
cf4c6a2f 2714 ioapic_write_entry(apic, pin, entry1);
1da177e4
LT		 2715
2716 save_control = CMOS_READ(RTC_CONTROL);
2717 save_freq_select = CMOS_READ(RTC_FREQ_SELECT);
2718 CMOS_WRITE((save_freq_select & ~RTC_RATE_SELECT) | 0x6,
2719 RTC_FREQ_SELECT);
2720 CMOS_WRITE(save_control | RTC_PIE, RTC_CONTROL);
2721
2722 i = 100;
2723 while (i-- > 0) {
2724 mdelay(10);
2725 if ((CMOS_READ(RTC_INTR_FLAGS) & RTC_PF) == RTC_PF)
2726 i -= 10;
2727 }
2728
2729 CMOS_WRITE(save_control, RTC_CONTROL);
2730 CMOS_WRITE(save_freq_select, RTC_FREQ_SELECT);
fcfd636a 2731 clear_IO_APIC_pin(apic, pin);
1da177e4 2732
cf4c6a2f 2733 ioapic_write_entry(apic, pin, entry0);
1da177e4
LT		 2734}
2735
efa2559f 2736static int disable_timer_pin_1 __initdata;
047c8fdb 2737/* Actually the next is obsolete, but keep it for paranoid reasons -AK */
54168ed7 2738static int __init disable_timer_pin_setup(char *arg)
efa2559f
YL		 2739{
2740 disable_timer_pin_1 = 1;
2741 return 0;
2742}
54168ed7 2743early_param("disable_timer_pin_1", disable_timer_pin_setup);
efa2559f
YL		 2744
2745int timer_through_8259 __initdata;
2746
1da177e4
LT		 2747/*
2748 * This code may look a bit paranoid, but it's supposed to cooperate with
2749 * a wide range of boards and BIOS bugs. Fortunately only the timer IRQ
2750 * is so screwy. Thanks to Brian Perkins for testing/hacking this beast
2751 * fanatically on his truly buggy board.
54168ed7
IM		 2752 *
2753 * FIXME: really need to revamp this for all platforms.
1da177e4 2754 */
8542b200 2755static inline void __init check_timer(void)
1da177e4 2756{
497c9a19 2757 struct irq_cfg *cfg = irq_cfg(0);
fcfd636a 2758 int apic1, pin1, apic2, pin2;
4aae0702 2759 unsigned long flags;
047c8fdb
YL		 2760 unsigned int ver;
2761 int no_pin1 = 0;
4aae0702
IM		 2762
2763 local_irq_save(flags);
d4d25dec 2764
54168ed7
IM		 2765 ver = apic_read(APIC_LVR);
2766 ver = GET_APIC_VERSION(ver);
6e908947 2767
1da177e4
LT		 2768 /*
2769 * get/set the timer IRQ vector:
2770 */
2771 disable_8259A_irq(0);
497c9a19 2772 assign_irq_vector(0, TARGET_CPUS);
1da177e4
LT		 2773
2774 /*
d11d5794
MR		 2775 * As IRQ0 is to be enabled in the 8259A, the virtual
2776 * wire has to be disabled in the local APIC. Also
2777 * timer interrupts need to be acknowledged manually in
2778 * the 8259A for the i82489DX when using the NMI
2779 * watchdog as that APIC treats NMIs as level-triggered.
2780 * The AEOI mode will finish them in the 8259A
2781 * automatically.
1da177e4 2782 */
593f4a78 2783 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_EXTINT);
1da177e4 2784 init_8259A(1);
54168ed7 2785#ifdef CONFIG_X86_32
d11d5794 2786 timer_ack = (nmi_watchdog == NMI_IO_APIC && !APIC_INTEGRATED(ver));
54168ed7 2787#endif
1da177e4 2788
fcfd636a
EB		 2789 pin1 = find_isa_irq_pin(0, mp_INT);
2790 apic1 = find_isa_irq_apic(0, mp_INT);
2791 pin2 = ioapic_i8259.pin;
2792 apic2 = ioapic_i8259.apic;
1da177e4 2793
49a66a0b
MR		 2794 apic_printk(APIC_QUIET, KERN_INFO "..TIMER: vector=0x%02X "
2795 "apic1=%d pin1=%d apic2=%d pin2=%d\n",
497c9a19 2796 cfg->vector, apic1, pin1, apic2, pin2);
1da177e4 2797
691874fa
MR		 2798 /*
2799 * Some BIOS writers are clueless and report the ExtINTA
2800 * I/O APIC input from the cascaded 8259A as the timer
2801 * interrupt input. So just in case, if only one pin
2802 * was found above, try it both directly and through the
2803 * 8259A.
2804 */
2805 if (pin1 == -1) {
54168ed7
IM		 2806#ifdef CONFIG_INTR_REMAP
2807 if (intr_remapping_enabled)
2808 panic("BIOS bug: timer not connected to IO-APIC");
2809#endif
691874fa
MR		 2810 pin1 = pin2;
2811 apic1 = apic2;
2812 no_pin1 = 1;
2813 } else if (pin2 == -1) {
2814 pin2 = pin1;
2815 apic2 = apic1;
2816 }
2817
1da177e4
LT		 2818 if (pin1 != -1) {
2819 /*
2820 * Ok, does IRQ0 through the IOAPIC work?
2821 */
691874fa
MR		 2822 if (no_pin1) {
2823 add_pin_to_irq(0, apic1, pin1);
497c9a19 2824 setup_timer_IRQ0_pin(apic1, pin1, cfg->vector);
691874fa 2825 }
1da177e4
LT		 2826 unmask_IO_APIC_irq(0);
2827 if (timer_irq_works()) {
2828 if (nmi_watchdog == NMI_IO_APIC) {
1da177e4
LT		 2829 setup_nmi();
2830 enable_8259A_irq(0);
1da177e4 2831 }
66759a01
CE		 2832 if (disable_timer_pin_1 > 0)
2833 clear_IO_APIC_pin(0, pin1);
4aae0702 2834 goto out;
1da177e4 2835 }
54168ed7
IM		 2836#ifdef CONFIG_INTR_REMAP
2837 if (intr_remapping_enabled)
2838 panic("timer doesn't work through Interrupt-remapped IO-APIC");
2839#endif
fcfd636a 2840 clear_IO_APIC_pin(apic1, pin1);
691874fa 2841 if (!no_pin1)
49a66a0b
MR		 2842 apic_printk(APIC_QUIET, KERN_ERR "..MP-BIOS bug: "
2843 "8254 timer not connected to IO-APIC\n");
1da177e4 2844
49a66a0b
MR		 2845 apic_printk(APIC_QUIET, KERN_INFO "...trying to set up timer "
2846 "(IRQ0) through the 8259A ...\n");
2847 apic_printk(APIC_QUIET, KERN_INFO
2848 "..... (found apic %d pin %d) ...\n", apic2, pin2);
1da177e4
LT		 2849 /*
2850 * legacy devices should be connected to IO APIC #0
2851 */
691874fa 2852 replace_pin_at_irq(0, apic1, pin1, apic2, pin2);
497c9a19 2853 setup_timer_IRQ0_pin(apic2, pin2, cfg->vector);
24742ece 2854 unmask_IO_APIC_irq(0);
ecd29476 2855 enable_8259A_irq(0);
1da177e4 2856 if (timer_irq_works()) {
49a66a0b 2857 apic_printk(APIC_QUIET, KERN_INFO "....... works.\n");
35542c5e 2858 timer_through_8259 = 1;
1da177e4 2859 if (nmi_watchdog == NMI_IO_APIC) {
60134ebe 2860 disable_8259A_irq(0);
1da177e4 2861 setup_nmi();
60134ebe 2862 enable_8259A_irq(0);
1da177e4 2863 }
4aae0702 2864 goto out;
1da177e4
LT		 2865 }
2866 /*
2867 * Cleanup, just in case ...
2868 */
ecd29476 2869 disable_8259A_irq(0);
fcfd636a 2870 clear_IO_APIC_pin(apic2, pin2);
49a66a0b 2871 apic_printk(APIC_QUIET, KERN_INFO "....... failed.\n");
1da177e4 2872 }
1da177e4
LT		 2873
2874 if (nmi_watchdog == NMI_IO_APIC) {
49a66a0b
MR		 2875 apic_printk(APIC_QUIET, KERN_WARNING "timer doesn't work "
2876 "through the IO-APIC - disabling NMI Watchdog!\n");
067fa0ff 2877 nmi_watchdog = NMI_NONE;
1da177e4 2878 }
54168ed7 2879#ifdef CONFIG_X86_32
d11d5794 2880 timer_ack = 0;
54168ed7 2881#endif
1da177e4 2882
49a66a0b
MR		 2883 apic_printk(APIC_QUIET, KERN_INFO
2884 "...trying to set up timer as Virtual Wire IRQ...\n");
1da177e4 2885
497c9a19
YL		 2886 lapic_register_intr(0);
2887 apic_write(APIC_LVT0, APIC_DM_FIXED | cfg->vector); /* Fixed mode */
1da177e4
LT		 2888 enable_8259A_irq(0);
2889
2890 if (timer_irq_works()) {
49a66a0b 2891 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
4aae0702 2892 goto out;
1da177e4 2893 }
e67465f1 2894 disable_8259A_irq(0);
497c9a19 2895 apic_write(APIC_LVT0, APIC_LVT_MASKED | APIC_DM_FIXED | cfg->vector);
49a66a0b 2896 apic_printk(APIC_QUIET, KERN_INFO "..... failed.\n");
1da177e4 2897
49a66a0b
MR		 2898 apic_printk(APIC_QUIET, KERN_INFO
2899 "...trying to set up timer as ExtINT IRQ...\n");
1da177e4 2900
1da177e4
LT		 2901 init_8259A(0);
2902 make_8259A_irq(0);
593f4a78 2903 apic_write(APIC_LVT0, APIC_DM_EXTINT);
1da177e4
LT		 2904
2905 unlock_ExtINT_logic();
2906
2907 if (timer_irq_works()) {
49a66a0b 2908 apic_printk(APIC_QUIET, KERN_INFO "..... works.\n");
4aae0702 2909 goto out;
1da177e4 2910 }
49a66a0b 2911 apic_printk(APIC_QUIET, KERN_INFO "..... failed :(.\n");
1da177e4 2912 panic("IO-APIC + timer doesn't work! Boot with apic=debug and send a "
49a66a0b 2913 "report. Then try booting with the 'noapic' option.\n");
4aae0702
IM		 2914out:
2915 local_irq_restore(flags);
1da177e4
LT		 2916}
2917
2918/*
af174783
MR		 2919 * Traditionally ISA IRQ2 is the cascade IRQ, and is not available
2920 * to devices. However there may be an I/O APIC pin available for
2921 * this interrupt regardless. The pin may be left unconnected, but
2922 * typically it will be reused as an ExtINT cascade interrupt for
2923 * the master 8259A. In the MPS case such a pin will normally be
2924 * reported as an ExtINT interrupt in the MP table. With ACPI
2925 * there is no provision for ExtINT interrupts, and in the absence
2926 * of an override it would be treated as an ordinary ISA I/O APIC
2927 * interrupt, that is edge-triggered and unmasked by default. We
2928 * used to do this, but it caused problems on some systems because
2929 * of the NMI watchdog and sometimes IRQ0 of the 8254 timer using
2930 * the same ExtINT cascade interrupt to drive the local APIC of the
2931 * bootstrap processor. Therefore we refrain from routing IRQ2 to
2932 * the I/O APIC in all cases now. No actual device should request
2933 * it anyway. --macro
1da177e4
LT		 2934 */
2935#define PIC_IRQS (1 << PIC_CASCADE_IR)
2936
2937void __init setup_IO_APIC(void)
2938{
54168ed7
IM		 2939
2940#ifdef CONFIG_X86_32
1da177e4 2941 enable_IO_APIC();
54168ed7
IM		 2942#else
2943 /*
2944 * calling enable_IO_APIC() is moved to setup_local_APIC for BP
2945 */
2946#endif
1da177e4 2947
af174783 2948 io_apic_irqs = ~PIC_IRQS;
1da177e4 2949
54168ed7
IM		 2950 apic_printk(APIC_VERBOSE, "ENABLING IO-APIC IRQs\n");
2951 /*
2952 * Set up IO-APIC IRQ routing.
2953 */
2954#ifdef CONFIG_X86_32
2955 if (!acpi_ioapic)
2956 setup_ioapic_ids_from_mpc();
2957#endif
1da177e4
LT		 2958 sync_Arb_IDs();
2959 setup_IO_APIC_irqs();
2960 init_IO_APIC_traps();
1e4c85f9 2961 check_timer();
1da177e4
LT		 2962}
2963
2964/*
54168ed7
IM		 2965 * Called after all the initialization is done. If we didnt find any
2966 * APIC bugs then we can allow the modify fast path
1da177e4 2967 */
36062448 2968
1da177e4
LT		 2969static int __init io_apic_bug_finalize(void)
2970{
54168ed7
IM		 2971 if (sis_apic_bug == -1)
2972 sis_apic_bug = 0;
2973 return 0;
1da177e4
LT		 2974}
2975
2976late_initcall(io_apic_bug_finalize);
2977
2978struct sysfs_ioapic_data {
2979 struct sys_device dev;
2980 struct IO_APIC_route_entry entry[0];
2981};
54168ed7 2982static struct sysfs_ioapic_data * mp_ioapic_data[MAX_IO_APICS];
1da177e4 2983
438510f6 2984static int ioapic_suspend(struct sys_device *dev, pm_message_t state)
1da177e4
LT		 2985{
2986 struct IO_APIC_route_entry *entry;
2987 struct sysfs_ioapic_data *data;
1da177e4 2988 int i;
36062448 2989
1da177e4
LT		 2990 data = container_of(dev, struct sysfs_ioapic_data, dev);
2991 entry = data->entry;
54168ed7
IM		 2992 for (i = 0; i < nr_ioapic_registers[dev->id]; i ++, entry ++ )
2993 *entry = ioapic_read_entry(dev->id, i);
1da177e4
LT		 2994
2995 return 0;
2996}
2997
2998static int ioapic_resume(struct sys_device *dev)
2999{
3000 struct IO_APIC_route_entry *entry;
3001 struct sysfs_ioapic_data *data;
3002 unsigned long flags;
3003 union IO_APIC_reg_00 reg_00;
3004 int i;
36062448 3005
1da177e4
LT		 3006 data = container_of(dev, struct sysfs_ioapic_data, dev);
3007 entry = data->entry;
3008
3009 spin_lock_irqsave(&ioapic_lock, flags);
3010 reg_00.raw = io_apic_read(dev->id, 0);
ec2cd0a2
AS		 3011 if (reg_00.bits.ID != mp_ioapics[dev->id].mp_apicid) {
3012 reg_00.bits.ID = mp_ioapics[dev->id].mp_apicid;
1da177e4
LT		 3013 io_apic_write(dev->id, 0, reg_00.raw);
3014 }
1da177e4 3015 spin_unlock_irqrestore(&ioapic_lock, flags);
36062448 3016 for (i = 0; i < nr_ioapic_registers[dev->id]; i++)
cf4c6a2f 3017 ioapic_write_entry(dev->id, i, entry[i]);
1da177e4
LT		 3018
3019 return 0;
3020}
3021
3022static struct sysdev_class ioapic_sysdev_class = {
af5ca3f4 3023 .name = "ioapic",
1da177e4
LT		 3024 .suspend = ioapic_suspend,
3025 .resume = ioapic_resume,
3026};
3027
3028static int __init ioapic_init_sysfs(void)
3029{
54168ed7
IM		 3030 struct sys_device * dev;
3031 int i, size, error;
1da177e4
LT		 3032
3033 error = sysdev_class_register(&ioapic_sysdev_class);
3034 if (error)
3035 return error;
3036
54168ed7 3037 for (i = 0; i < nr_ioapics; i++ ) {
36062448 3038 size = sizeof(struct sys_device) + nr_ioapic_registers[i]
1da177e4 3039 * sizeof(struct IO_APIC_route_entry);
25556c16 3040 mp_ioapic_data[i] = kzalloc(size, GFP_KERNEL);
1da177e4
LT		 3041 if (!mp_ioapic_data[i]) {
3042 printk(KERN_ERR "Can't suspend/resume IOAPIC %d\n", i);
3043 continue;
3044 }
1da177e4 3045 dev = &mp_ioapic_data[i]->dev;
36062448 3046 dev->id = i;
1da177e4
LT		 3047 dev->cls = &ioapic_sysdev_class;
3048 error = sysdev_register(dev);
3049 if (error) {
3050 kfree(mp_ioapic_data[i]);
3051 mp_ioapic_data[i] = NULL;
3052 printk(KERN_ERR "Can't suspend/resume IOAPIC %d\n", i);
3053 continue;
3054 }
3055 }
3056
3057 return 0;
3058}
3059
3060device_initcall(ioapic_init_sysfs);
3061
3fc471ed 3062/*
95d77884 3063 * Dynamic irq allocate and deallocation
3fc471ed 3064 */
199751d7 3065unsigned int create_irq_nr(unsigned int irq_want)
3fc471ed 3066{
ace80ab7 3067 /* Allocate an unused irq */
54168ed7
IM		 3068 unsigned int irq;
3069 unsigned int new;
3fc471ed 3070 unsigned long flags;
da51a821 3071 struct irq_cfg *cfg_new;
3fc471ed 3072
497c9a19 3073#ifndef CONFIG_HAVE_SPARSE_IRQ
199751d7 3074 irq_want = nr_irqs - 1;
497c9a19 3075#endif
199751d7
YL		 3076
3077 irq = 0;
ace80ab7 3078 spin_lock_irqsave(&vector_lock, flags);
54168ed7 3079 for (new = irq_want; new > 0; new--) {
ace80ab7
EB		 3080 if (platform_legacy_irq(new))
3081 continue;
da51a821
YL		 3082 cfg_new = irq_cfg(new);
3083 if (cfg_new && cfg_new->vector != 0)
ace80ab7 3084 continue;
047c8fdb 3085 /* check if need to create one */
da51a821
YL		 3086 if (!cfg_new)
3087 cfg_new = irq_cfg_alloc(new);
497c9a19 3088 if (__assign_irq_vector(new, TARGET_CPUS) == 0)
ace80ab7
EB		 3089 irq = new;
3090 break;
3091 }
3092 spin_unlock_irqrestore(&vector_lock, flags);
3fc471ed 3093
199751d7 3094 if (irq > 0) {
3fc471ed
EB		 3095 dynamic_irq_init(irq);
3096 }
3097 return irq;
3098}
3099
199751d7
YL		 3100int create_irq(void)
3101{
54168ed7
IM		 3102 int irq;
3103
3104 irq = create_irq_nr(nr_irqs - 1);
3105
3106 if (irq == 0)
3107 irq = -1;
3108
3109 return irq;
199751d7
YL		 3110}
3111
3fc471ed
EB		 3112void destroy_irq(unsigned int irq)
3113{
3114 unsigned long flags;
3fc471ed
EB		 3115
3116 dynamic_irq_cleanup(irq);
3117
54168ed7
IM		 3118#ifdef CONFIG_INTR_REMAP
3119 free_irte(irq);
3120#endif
3fc471ed 3121 spin_lock_irqsave(&vector_lock, flags);
497c9a19 3122 __clear_irq_vector(irq);
3fc471ed
EB		 3123 spin_unlock_irqrestore(&vector_lock, flags);
3124}
3fc471ed 3125
2d3fcc1c 3126/*
27b46d76 3127 * MSI message composition
2d3fcc1c
EB		 3128 */
3129#ifdef CONFIG_PCI_MSI
3b7d1921 3130static int msi_compose_msg(struct pci_dev *pdev, unsigned int irq, struct msi_msg *msg)
2d3fcc1c 3131{
497c9a19
YL		 3132 struct irq_cfg *cfg;
3133 int err;
2d3fcc1c 3134 unsigned dest;
497c9a19 3135 cpumask_t tmp;
2d3fcc1c 3136
497c9a19
YL		 3137 tmp = TARGET_CPUS;
3138 err = assign_irq_vector(irq, tmp);
3139 if (err)
3140 return err;
2d3fcc1c 3141
497c9a19
YL		 3142 cfg = irq_cfg(irq);
3143 cpus_and(tmp, cfg->domain, tmp);
3144 dest = cpu_mask_to_apicid(tmp);
3145
54168ed7
IM		 3146#ifdef CONFIG_INTR_REMAP
3147 if (irq_remapped(irq)) {
3148 struct irte irte;
3149 int ir_index;
3150 u16 sub_handle;
3151
3152 ir_index = map_irq_to_irte_handle(irq, &sub_handle);
3153 BUG_ON(ir_index == -1);
3154
3155 memset (&irte, 0, sizeof(irte));
3156
3157 irte.present = 1;
3158 irte.dst_mode = INT_DEST_MODE;
3159 irte.trigger_mode = 0; /* edge */
3160 irte.dlvry_mode = INT_DELIVERY_MODE;
3161 irte.vector = cfg->vector;
3162 irte.dest_id = IRTE_DEST(dest);
3163
3164 modify_irte(irq, &irte);
3165
3166 msg->address_hi = MSI_ADDR_BASE_HI;
3167 msg->data = sub_handle;
3168 msg->address_lo = MSI_ADDR_BASE_LO | MSI_ADDR_IR_EXT_INT |
3169 MSI_ADDR_IR_SHV |
3170 MSI_ADDR_IR_INDEX1(ir_index) |
3171 MSI_ADDR_IR_INDEX2(ir_index);
3172 } else
3173#endif
3174 {
3175 msg->address_hi = MSI_ADDR_BASE_HI;
3176 msg->address_lo =
3177 MSI_ADDR_BASE_LO |
3178 ((INT_DEST_MODE == 0) ?
3179 MSI_ADDR_DEST_MODE_PHYSICAL:
3180 MSI_ADDR_DEST_MODE_LOGICAL) |
3181 ((INT_DELIVERY_MODE != dest_LowestPrio) ?
3182 MSI_ADDR_REDIRECTION_CPU:
3183 MSI_ADDR_REDIRECTION_LOWPRI) |
3184 MSI_ADDR_DEST_ID(dest);
497c9a19 3185
54168ed7
IM		 3186 msg->data =
3187 MSI_DATA_TRIGGER_EDGE |
3188 MSI_DATA_LEVEL_ASSERT |
3189 ((INT_DELIVERY_MODE != dest_LowestPrio) ?
3190 MSI_DATA_DELIVERY_FIXED:
3191 MSI_DATA_DELIVERY_LOWPRI) |
3192 MSI_DATA_VECTOR(cfg->vector);
3193 }
497c9a19 3194 return err;
2d3fcc1c
EB		 3195}
3196
3b7d1921
EB		 3197#ifdef CONFIG_SMP
3198static void set_msi_irq_affinity(unsigned int irq, cpumask_t mask)
2d3fcc1c 3199{
497c9a19 3200 struct irq_cfg *cfg;
3b7d1921
EB		 3201 struct msi_msg msg;
3202 unsigned int dest;
3203 cpumask_t tmp;
54168ed7 3204 struct irq_desc *desc;
3b7d1921
EB		 3205
3206 cpus_and(tmp, mask, cpu_online_map);
3207 if (cpus_empty(tmp))
497c9a19 3208 return;
2d3fcc1c 3209
497c9a19 3210 if (assign_irq_vector(irq, mask))
3b7d1921 3211 return;
2d3fcc1c 3212
497c9a19
YL		 3213 cfg = irq_cfg(irq);
3214 cpus_and(tmp, cfg->domain, mask);
3215 dest = cpu_mask_to_apicid(tmp);
3b7d1921
EB		 3216
3217 read_msi_msg(irq, &msg);
3218
3219 msg.data &= ~MSI_DATA_VECTOR_MASK;
497c9a19 3220 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3b7d1921
EB		 3221 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3222 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3223
3224 write_msi_msg(irq, &msg);
54168ed7
IM		 3225 desc = irq_to_desc(irq);
3226 desc->affinity = mask;
2d3fcc1c 3227}
54168ed7
IM		 3228
3229#ifdef CONFIG_INTR_REMAP
3230/*
3231 * Migrate the MSI irq to another cpumask. This migration is
3232 * done in the process context using interrupt-remapping hardware.
3233 */
3234static void ir_set_msi_irq_affinity(unsigned int irq, cpumask_t mask)
3235{
3236 struct irq_cfg *cfg;
3237 unsigned int dest;
3238 cpumask_t tmp, cleanup_mask;
3239 struct irte irte;
3240 struct irq_desc *desc;
3241
3242 cpus_and(tmp, mask, cpu_online_map);
3243 if (cpus_empty(tmp))
3244 return;
3245
3246 if (get_irte(irq, &irte))
3247 return;
3248
3249 if (assign_irq_vector(irq, mask))
3250 return;
3251
3252 cfg = irq_cfg(irq);
3253 cpus_and(tmp, cfg->domain, mask);
3254 dest = cpu_mask_to_apicid(tmp);
3255
3256 irte.vector = cfg->vector;
3257 irte.dest_id = IRTE_DEST(dest);
3258
3259 /*
3260 * atomically update the IRTE with the new destination and vector.
3261 */
3262 modify_irte(irq, &irte);
3263
3264 /*
3265 * After this point, all the interrupts will start arriving
3266 * at the new destination. So, time to cleanup the previous
3267 * vector allocation.
3268 */
3269 if (cfg->move_in_progress) {
3270 cpus_and(cleanup_mask, cfg->old_domain, cpu_online_map);
3271 cfg->move_cleanup_count = cpus_weight(cleanup_mask);
3272 send_IPI_mask(cleanup_mask, IRQ_MOVE_CLEANUP_VECTOR);
3273 cfg->move_in_progress = 0;
3274 }
3275
3276 desc = irq_to_desc(irq);
3277 desc->affinity = mask;
3278}
3279#endif
3b7d1921 3280#endif /* CONFIG_SMP */
2d3fcc1c 3281
3b7d1921
EB		 3282/*
3283 * IRQ Chip for MSI PCI/PCI-X/PCI-Express Devices,
3284 * which implement the MSI or MSI-X Capability Structure.
3285 */
3286static struct irq_chip msi_chip = {
3287 .name = "PCI-MSI",
3288 .unmask = unmask_msi_irq,
3289 .mask = mask_msi_irq,
1d025192 3290 .ack = ack_apic_edge,
3b7d1921
EB		 3291#ifdef CONFIG_SMP
3292 .set_affinity = set_msi_irq_affinity,
3293#endif
3294 .retrigger = ioapic_retrigger_irq,
2d3fcc1c
EB		 3295};
3296
54168ed7
IM		 3297#ifdef CONFIG_INTR_REMAP
3298static struct irq_chip msi_ir_chip = {
3299 .name = "IR-PCI-MSI",
3300 .unmask = unmask_msi_irq,
3301 .mask = mask_msi_irq,
3302 .ack = ack_x2apic_edge,
3303#ifdef CONFIG_SMP
3304 .set_affinity = ir_set_msi_irq_affinity,
3305#endif
3306 .retrigger = ioapic_retrigger_irq,
3307};
3308
3309/*
3310 * Map the PCI dev to the corresponding remapping hardware unit
3311 * and allocate 'nvec' consecutive interrupt-remapping table entries
3312 * in it.
3313 */
3314static int msi_alloc_irte(struct pci_dev *dev, int irq, int nvec)
3315{
3316 struct intel_iommu *iommu;
3317 int index;
3318
3319 iommu = map_dev_to_ir(dev);
3320 if (!iommu) {
3321 printk(KERN_ERR
3322 "Unable to map PCI %s to iommu\n", pci_name(dev));
3323 return -ENOENT;
3324 }
3325
3326 index = alloc_irte(iommu, irq, nvec);
3327 if (index < 0) {
3328 printk(KERN_ERR
3329 "Unable to allocate %d IRTE for PCI %s\n", nvec,
3330 pci_name(dev));
3331 return -ENOSPC;
3332 }
3333 return index;
3334}
3335#endif
1d025192
YL		 3336
3337static int setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc, int irq)
3338{
3339 int ret;
3340 struct msi_msg msg;
3341
3342 ret = msi_compose_msg(dev, irq, &msg);
3343 if (ret < 0)
3344 return ret;
3345
3346 set_irq_msi(irq, desc);
3347 write_msi_msg(irq, &msg);
3348
54168ed7
IM		 3349#ifdef CONFIG_INTR_REMAP
3350 if (irq_remapped(irq)) {
3351 struct irq_desc *desc = irq_to_desc(irq);
3352 /*
3353 * irq migration in process context
3354 */
3355 desc->status |= IRQ_MOVE_PCNTXT;
3356 set_irq_chip_and_handler_name(irq, &msi_ir_chip, handle_edge_irq, "edge");
3357 } else
3358#endif
3359 set_irq_chip_and_handler_name(irq, &msi_chip, handle_edge_irq, "edge");
1d025192
YL		 3360
3361 return 0;
3362}
3363
199751d7
YL		 3364static unsigned int build_irq_for_pci_dev(struct pci_dev *dev)
3365{
3366 unsigned int irq;
3367
3368 irq = dev->bus->number;
3369 irq <<= 8;
3370 irq |= dev->devfn;
3371 irq <<= 12;
3372
3373 return irq;
3374}
3375
f7feaca7 3376int arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc)
3b7d1921 3377{
54168ed7
IM		 3378 unsigned int irq;
3379 int ret;
199751d7
YL		 3380 unsigned int irq_want;
3381
3382 irq_want = build_irq_for_pci_dev(dev) + 0x100;
3383
3384 irq = create_irq_nr(irq_want);
199751d7
YL		 3385 if (irq == 0)
3386 return -1;
f7feaca7 3387
54168ed7
IM		 3388#ifdef CONFIG_INTR_REMAP
3389 if (!intr_remapping_enabled)
3390 goto no_ir;
3391
3392 ret = msi_alloc_irte(dev, irq, 1);
3393 if (ret < 0)
3394 goto error;
3395no_ir:
3396#endif
1d025192 3397 ret = setup_msi_irq(dev, desc, irq);
f7feaca7
EB		 3398 if (ret < 0) {
3399 destroy_irq(irq);
3b7d1921 3400 return ret;
54168ed7 3401 }
7fe3730d 3402 return 0;
54168ed7
IM		 3403
3404#ifdef CONFIG_INTR_REMAP
3405error:
3406 destroy_irq(irq);
3407 return ret;
3408#endif
3b7d1921
EB		 3409}
3410
047c8fdb
YL		 3411int arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
3412{
54168ed7
IM		 3413 unsigned int irq;
3414 int ret, sub_handle;
3415 struct msi_desc *desc;
3416 unsigned int irq_want;
3417
3418#ifdef CONFIG_INTR_REMAP
3419 struct intel_iommu *iommu = 0;
3420 int index = 0;
3421#endif
3422
3423 irq_want = build_irq_for_pci_dev(dev) + 0x100;
3424 sub_handle = 0;
3425 list_for_each_entry(desc, &dev->msi_list, list) {
3426 irq = create_irq_nr(irq_want--);
3427 if (irq == 0)
3428 return -1;
3429#ifdef CONFIG_INTR_REMAP
3430 if (!intr_remapping_enabled)
3431 goto no_ir;
3432
3433 if (!sub_handle) {
3434 /*
3435 * allocate the consecutive block of IRTE's
3436 * for 'nvec'
3437 */
3438 index = msi_alloc_irte(dev, irq, nvec);
3439 if (index < 0) {
3440 ret = index;
3441 goto error;
3442 }
3443 } else {
3444 iommu = map_dev_to_ir(dev);
3445 if (!iommu) {
3446 ret = -ENOENT;
3447 goto error;
3448 }
3449 /*
3450 * setup the mapping between the irq and the IRTE
3451 * base index, the sub_handle pointing to the
3452 * appropriate interrupt remap table entry.
3453 */
3454 set_irte_irq(irq, iommu, index, sub_handle);
3455 }
3456no_ir:
3457#endif
3458 ret = setup_msi_irq(dev, desc, irq);
3459 if (ret < 0)
3460 goto error;
3461 sub_handle++;
3462 }
3463 return 0;
047c8fdb
YL		 3464
3465error:
54168ed7
IM		 3466 destroy_irq(irq);
3467 return ret;
047c8fdb
YL		 3468}
3469
3b7d1921
EB		 3470void arch_teardown_msi_irq(unsigned int irq)
3471{
f7feaca7 3472 destroy_irq(irq);
3b7d1921
EB		 3473}
3474
54168ed7
IM		 3475#ifdef CONFIG_DMAR
3476#ifdef CONFIG_SMP
3477static void dmar_msi_set_affinity(unsigned int irq, cpumask_t mask)
3478{
3479 struct irq_cfg *cfg;
3480 struct msi_msg msg;
3481 unsigned int dest;
3482 cpumask_t tmp;
3483 struct irq_desc *desc;
3484
3485 cpus_and(tmp, mask, cpu_online_map);
3486 if (cpus_empty(tmp))
3487 return;
3488
3489 if (assign_irq_vector(irq, mask))
3490 return;
3491
3492 cfg = irq_cfg(irq);
3493 cpus_and(tmp, cfg->domain, mask);
3494 dest = cpu_mask_to_apicid(tmp);
3495
3496 dmar_msi_read(irq, &msg);
3497
3498 msg.data &= ~MSI_DATA_VECTOR_MASK;
3499 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3500 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3501 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3502
3503 dmar_msi_write(irq, &msg);
3504 desc = irq_to_desc(irq);
3505 desc->affinity = mask;
3506}
3507#endif /* CONFIG_SMP */
3508
3509struct irq_chip dmar_msi_type = {
3510 .name = "DMAR_MSI",
3511 .unmask = dmar_msi_unmask,
3512 .mask = dmar_msi_mask,
3513 .ack = ack_apic_edge,
3514#ifdef CONFIG_SMP
3515 .set_affinity = dmar_msi_set_affinity,
3516#endif
3517 .retrigger = ioapic_retrigger_irq,
3518};
3519
3520int arch_setup_dmar_msi(unsigned int irq)
3521{
3522 int ret;
3523 struct msi_msg msg;
2d3fcc1c 3524
54168ed7
IM		 3525 ret = msi_compose_msg(NULL, irq, &msg);
3526 if (ret < 0)
3527 return ret;
3528 dmar_msi_write(irq, &msg);
3529 set_irq_chip_and_handler_name(irq, &dmar_msi_type, handle_edge_irq,
3530 "edge");
3531 return 0;
3532}
3533#endif
3534
58ac1e76 3535#ifdef CONFIG_HPET_TIMER
3536
3537#ifdef CONFIG_SMP
3538static void hpet_msi_set_affinity(unsigned int irq, cpumask_t mask)
3539{
3540 struct irq_cfg *cfg;
3541 struct irq_desc *desc;
3542 struct msi_msg msg;
3543 unsigned int dest;
3544 cpumask_t tmp;
3545
3546 cpus_and(tmp, mask, cpu_online_map);
3547 if (cpus_empty(tmp))
3548 return;
3549
3550 if (assign_irq_vector(irq, mask))
3551 return;
3552
3553 cfg = irq_cfg(irq);
3554 cpus_and(tmp, cfg->domain, mask);
3555 dest = cpu_mask_to_apicid(tmp);
3556
3557 hpet_msi_read(irq, &msg);
3558
3559 msg.data &= ~MSI_DATA_VECTOR_MASK;
3560 msg.data |= MSI_DATA_VECTOR(cfg->vector);
3561 msg.address_lo &= ~MSI_ADDR_DEST_ID_MASK;
3562 msg.address_lo |= MSI_ADDR_DEST_ID(dest);
3563
3564 hpet_msi_write(irq, &msg);
3565 desc = irq_to_desc(irq);
3566 desc->affinity = mask;
3567}
3568#endif /* CONFIG_SMP */
3569
3570struct irq_chip hpet_msi_type = {
3571 .name = "HPET_MSI",
3572 .unmask = hpet_msi_unmask,
3573 .mask = hpet_msi_mask,
3574 .ack = ack_apic_edge,
3575#ifdef CONFIG_SMP
3576 .set_affinity = hpet_msi_set_affinity,
3577#endif
3578 .retrigger = ioapic_retrigger_irq,
3579};
3580
3581int arch_setup_hpet_msi(unsigned int irq)
3582{
3583 int ret;
3584 struct msi_msg msg;
3585
3586 ret = msi_compose_msg(NULL, irq, &msg);
3587 if (ret < 0)
3588 return ret;
3589
3590 hpet_msi_write(irq, &msg);
3591 set_irq_chip_and_handler_name(irq, &hpet_msi_type, handle_edge_irq,
3592 "edge");
3593 return 0;
3594}
3595#endif
3596
54168ed7 3597#endif /* CONFIG_PCI_MSI */
8b955b0d
EB		 3598/*
3599 * Hypertransport interrupt support
3600 */
3601#ifdef CONFIG_HT_IRQ
3602
3603#ifdef CONFIG_SMP
3604
497c9a19 3605static void target_ht_irq(unsigned int irq, unsigned int dest, u8 vector)
8b955b0d 3606{
ec68307c
EB		 3607 struct ht_irq_msg msg;
3608 fetch_ht_irq_msg(irq, &msg);
8b955b0d 3609
497c9a19 3610 msg.address_lo &= ~(HT_IRQ_LOW_VECTOR_MASK | HT_IRQ_LOW_DEST_ID_MASK);
ec68307c 3611 msg.address_hi &= ~(HT_IRQ_HIGH_DEST_ID_MASK);
8b955b0d 3612
497c9a19 3613 msg.address_lo |= HT_IRQ_LOW_VECTOR(vector) | HT_IRQ_LOW_DEST_ID(dest);
ec68307c 3614 msg.address_hi |= HT_IRQ_HIGH_DEST_ID(dest);
8b955b0d 3615
ec68307c 3616 write_ht_irq_msg(irq, &msg);
8b955b0d
EB		 3617}
3618
3619static void set_ht_irq_affinity(unsigned int irq, cpumask_t mask)
3620{
497c9a19 3621 struct irq_cfg *cfg;
8b955b0d
EB		 3622 unsigned int dest;
3623 cpumask_t tmp;
54168ed7 3624 struct irq_desc *desc;
8b955b0d
EB		 3625
3626 cpus_and(tmp, mask, cpu_online_map);
3627 if (cpus_empty(tmp))
497c9a19 3628 return;
8b955b0d 3629
497c9a19
YL		 3630 if (assign_irq_vector(irq, mask))
3631 return;
8b955b0d 3632
497c9a19
YL		 3633 cfg = irq_cfg(irq);
3634 cpus_and(tmp, cfg->domain, mask);
3635 dest = cpu_mask_to_apicid(tmp);
8b955b0d 3636
497c9a19 3637 target_ht_irq(irq, dest, cfg->vector);
54168ed7
IM		 3638 desc = irq_to_desc(irq);
3639 desc->affinity = mask;
8b955b0d
EB		 3640}
3641#endif
3642
c37e108d 3643static struct irq_chip ht_irq_chip = {
8b955b0d
EB		 3644 .name = "PCI-HT",
3645 .mask = mask_ht_irq,
3646 .unmask = unmask_ht_irq,
1d025192 3647 .ack = ack_apic_edge,
8b955b0d
EB		 3648#ifdef CONFIG_SMP
3649 .set_affinity = set_ht_irq_affinity,
3650#endif
3651 .retrigger = ioapic_retrigger_irq,
3652};
3653
3654int arch_setup_ht_irq(unsigned int irq, struct pci_dev *dev)
3655{
497c9a19
YL		 3656 struct irq_cfg *cfg;
3657 int err;
3658 cpumask_t tmp;
8b955b0d 3659
497c9a19
YL		 3660 tmp = TARGET_CPUS;
3661 err = assign_irq_vector(irq, tmp);
54168ed7 3662 if (!err) {
ec68307c 3663 struct ht_irq_msg msg;
8b955b0d 3664 unsigned dest;
8b955b0d 3665
497c9a19
YL		 3666 cfg = irq_cfg(irq);
3667 cpus_and(tmp, cfg->domain, tmp);
8b955b0d
EB		 3668 dest = cpu_mask_to_apicid(tmp);
3669
ec68307c 3670 msg.address_hi = HT_IRQ_HIGH_DEST_ID(dest);
8b955b0d 3671
ec68307c
EB		 3672 msg.address_lo =
3673 HT_IRQ_LOW_BASE |
8b955b0d 3674 HT_IRQ_LOW_DEST_ID(dest) |
497c9a19 3675 HT_IRQ_LOW_VECTOR(cfg->vector) |
8b955b0d
EB		 3676 ((INT_DEST_MODE == 0) ?
3677 HT_IRQ_LOW_DM_PHYSICAL :
3678 HT_IRQ_LOW_DM_LOGICAL) |
3679 HT_IRQ_LOW_RQEOI_EDGE |
3680 ((INT_DELIVERY_MODE != dest_LowestPrio) ?
3681 HT_IRQ_LOW_MT_FIXED :
3682 HT_IRQ_LOW_MT_ARBITRATED) |
3683 HT_IRQ_LOW_IRQ_MASKED;
3684
ec68307c 3685 write_ht_irq_msg(irq, &msg);
8b955b0d 3686
a460e745
IM		 3687 set_irq_chip_and_handler_name(irq, &ht_irq_chip,
3688 handle_edge_irq, "edge");
8b955b0d 3689 }
497c9a19 3690 return err;
8b955b0d
EB		 3691}
3692#endif /* CONFIG_HT_IRQ */
3693
9d6a4d08
YL		 3694int __init io_apic_get_redir_entries (int ioapic)
3695{
3696 union IO_APIC_reg_01 reg_01;
3697 unsigned long flags;
3698
3699 spin_lock_irqsave(&ioapic_lock, flags);
3700 reg_01.raw = io_apic_read(ioapic, 1);
3701 spin_unlock_irqrestore(&ioapic_lock, flags);
3702
3703 return reg_01.bits.entries;
3704}
3705
3706int __init probe_nr_irqs(void)
3707{
3708 int idx;
3709 int nr = 0;
052c0bff
YL		 3710#ifndef CONFIG_XEN
3711 int nr_min = 32;
3712#else
3713 int nr_min = NR_IRQS;
3714#endif
9d6a4d08
YL		 3715
3716 for (idx = 0; idx < nr_ioapics; idx++)
052c0bff 3717 nr += io_apic_get_redir_entries(idx) + 1;
9d6a4d08
YL		 3718
3719 /* double it for hotplug and msi and nmi */
3720 nr <<= 1;
3721
3722 /* something wrong ? */
052c0bff
YL		 3723 if (nr < nr_min)
3724 nr = nr_min;
9d6a4d08
YL		 3725
3726 return nr;
3727}
3728
1da177e4 3729/* --------------------------------------------------------------------------
54168ed7 3730 ACPI-based IOAPIC Configuration
1da177e4
LT		 3731 -------------------------------------------------------------------------- */
3732
888ba6c6 3733#ifdef CONFIG_ACPI
1da177e4 3734
54168ed7 3735#ifdef CONFIG_X86_32
36062448 3736int __init io_apic_get_unique_id(int ioapic, int apic_id)
1da177e4
LT		 3737{
3738 union IO_APIC_reg_00 reg_00;
3739 static physid_mask_t apic_id_map = PHYSID_MASK_NONE;
3740 physid_mask_t tmp;
3741 unsigned long flags;
3742 int i = 0;
3743
3744 /*
36062448
PC		 3745 * The P4 platform supports up to 256 APIC IDs on two separate APIC
3746 * buses (one for LAPICs, one for IOAPICs), where predecessors only
1da177e4 3747 * supports up to 16 on one shared APIC bus.
36062448 3748 *
1da177e4
LT		 3749 * TBD: Expand LAPIC/IOAPIC support on P4-class systems to take full
3750 * advantage of new APIC bus architecture.
3751 */
3752
3753 if (physids_empty(apic_id_map))
3754 apic_id_map = ioapic_phys_id_map(phys_cpu_present_map);
3755
3756 spin_lock_irqsave(&ioapic_lock, flags);
3757 reg_00.raw = io_apic_read(ioapic, 0);
3758 spin_unlock_irqrestore(&ioapic_lock, flags);
3759
3760 if (apic_id >= get_physical_broadcast()) {
3761 printk(KERN_WARNING "IOAPIC[%d]: Invalid apic_id %d, trying "
3762 "%d\n", ioapic, apic_id, reg_00.bits.ID);
3763 apic_id = reg_00.bits.ID;
3764 }
3765
3766 /*
36062448 3767 * Every APIC in a system must have a unique ID or we get lots of nice
1da177e4
LT		 3768 * 'stuck on smp_invalidate_needed IPI wait' messages.
3769 */
3770 if (check_apicid_used(apic_id_map, apic_id)) {
3771
3772 for (i = 0; i < get_physical_broadcast(); i++) {
3773 if (!check_apicid_used(apic_id_map, i))
3774 break;
3775 }
3776
3777 if (i == get_physical_broadcast())
3778 panic("Max apic_id exceeded!\n");
3779
3780 printk(KERN_WARNING "IOAPIC[%d]: apic_id %d already used, "
3781 "trying %d\n", ioapic, apic_id, i);
3782
3783 apic_id = i;
36062448 3784 }
1da177e4
LT		 3785
3786 tmp = apicid_to_cpu_present(apic_id);
3787 physids_or(apic_id_map, apic_id_map, tmp);
3788
3789 if (reg_00.bits.ID != apic_id) {
3790 reg_00.bits.ID = apic_id;
3791
3792 spin_lock_irqsave(&ioapic_lock, flags);
3793 io_apic_write(ioapic, 0, reg_00.raw);
3794 reg_00.raw = io_apic_read(ioapic, 0);
3795 spin_unlock_irqrestore(&ioapic_lock, flags);
3796
3797 /* Sanity check */
6070f9ec
AD		 3798 if (reg_00.bits.ID != apic_id) {
3799 printk("IOAPIC[%d]: Unable to change apic_id!\n", ioapic);
3800 return -1;
3801 }
1da177e4
LT		 3802 }
3803
3804 apic_printk(APIC_VERBOSE, KERN_INFO
3805 "IOAPIC[%d]: Assigned apic_id %d\n", ioapic, apic_id);
3806
3807 return apic_id;
3808}
3809
36062448 3810int __init io_apic_get_version(int ioapic)
1da177e4
LT		 3811{
3812 union IO_APIC_reg_01 reg_01;
3813 unsigned long flags;
3814
3815 spin_lock_irqsave(&ioapic_lock, flags);
3816 reg_01.raw = io_apic_read(ioapic, 1);
3817 spin_unlock_irqrestore(&ioapic_lock, flags);
3818
3819 return reg_01.bits.version;
3820}
54168ed7 3821#endif
1da177e4 3822
54168ed7 3823int io_apic_set_pci_routing (int ioapic, int pin, int irq, int triggering, int polarity)
1da177e4 3824{
1da177e4 3825 if (!IO_APIC_IRQ(irq)) {
54168ed7 3826 apic_printk(APIC_QUIET,KERN_ERR "IOAPIC[%d]: Invalid reference to IRQ 0\n",
1da177e4
LT		 3827 ioapic);
3828 return -EINVAL;
3829 }
3830
1da177e4
LT		 3831 /*
3832 * IRQs < 16 are already in the irq_2_pin[] map
3833 */
3834 if (irq >= 16)
3835 add_pin_to_irq(irq, ioapic, pin);
3836
497c9a19 3837 setup_IO_APIC_irq(ioapic, pin, irq, triggering, polarity);
1da177e4
LT		 3838
3839 return 0;
3840}
3841
54168ed7 3842
61fd47e0
SL		 3843int acpi_get_override_irq(int bus_irq, int *trigger, int *polarity)
3844{
3845 int i;
3846
3847 if (skip_ioapic_setup)
3848 return -1;
3849
3850 for (i = 0; i < mp_irq_entries; i++)
2fddb6e2
AS		 3851 if (mp_irqs[i].mp_irqtype == mp_INT &&
3852 mp_irqs[i].mp_srcbusirq == bus_irq)
61fd47e0
SL		 3853 break;
3854 if (i >= mp_irq_entries)
3855 return -1;
3856
3857 *trigger = irq_trigger(i);
3858 *polarity = irq_polarity(i);
3859 return 0;
3860}
3861
888ba6c6 3862#endif /* CONFIG_ACPI */
1a3f239d 3863
497c9a19
YL		 3864/*
3865 * This function currently is only a helper for the i386 smp boot process where
3866 * we need to reprogram the ioredtbls to cater for the cpus which have come online
3867 * so mask in all cases should simply be TARGET_CPUS
3868 */
3869#ifdef CONFIG_SMP
3870void __init setup_ioapic_dest(void)
3871{
3872 int pin, ioapic, irq, irq_entry;
3873 struct irq_cfg *cfg;
497c9a19
YL		 3874
3875 if (skip_ioapic_setup == 1)
3876 return;
3877
3878 for (ioapic = 0; ioapic < nr_ioapics; ioapic++) {
3879 for (pin = 0; pin < nr_ioapic_registers[ioapic]; pin++) {
3880 irq_entry = find_irq_entry(ioapic, pin, mp_INT);
3881 if (irq_entry == -1)
3882 continue;
3883 irq = pin_2_irq(irq_entry, ioapic, pin);
3884
3885 /* setup_IO_APIC_irqs could fail to get vector for some device
3886 * when you have too many devices, because at that time only boot
3887 * cpu is online.
3888 */
3889 cfg = irq_cfg(irq);
3890 if (!cfg->vector)
3891 setup_IO_APIC_irq(ioapic, pin, irq,
3892 irq_trigger(irq_entry),
3893 irq_polarity(irq_entry));
54168ed7
IM		 3894#ifdef CONFIG_INTR_REMAP
3895 else if (intr_remapping_enabled)
3896 set_ir_ioapic_affinity_irq(irq, TARGET_CPUS);
3897#endif
3898 else
497c9a19 3899 set_ioapic_affinity_irq(irq, TARGET_CPUS);
497c9a19
YL		 3900 }
3901
3902 }
3903}
3904#endif
3905
54168ed7
IM		 3906#define IOAPIC_RESOURCE_NAME_SIZE 11
3907
3908static struct resource *ioapic_resources;
3909
3910static struct resource * __init ioapic_setup_resources(void)
3911{
3912 unsigned long n;
3913 struct resource *res;
3914 char *mem;
3915 int i;
3916
3917 if (nr_ioapics <= 0)
3918 return NULL;
3919
3920 n = IOAPIC_RESOURCE_NAME_SIZE + sizeof(struct resource);
3921 n *= nr_ioapics;
3922
3923 mem = alloc_bootmem(n);
3924 res = (void *)mem;
3925
3926 if (mem != NULL) {
3927 mem += sizeof(struct resource) * nr_ioapics;
3928
3929 for (i = 0; i < nr_ioapics; i++) {
3930 res[i].name = mem;
3931 res[i].flags = IORESOURCE_MEM | IORESOURCE_BUSY;
3932 sprintf(mem, "IOAPIC %u", i);
3933 mem += IOAPIC_RESOURCE_NAME_SIZE;
3934 }
3935 }
3936
3937 ioapic_resources = res;
3938
3939 return res;
3940}
54168ed7 3941
f3294a33
YL		 3942void __init ioapic_init_mappings(void)
3943{
3944 unsigned long ioapic_phys, idx = FIX_IO_APIC_BASE_0;
3945 int i;
54168ed7 3946 struct resource *ioapic_res;
f3294a33 3947
54168ed7 3948 ioapic_res = ioapic_setup_resources();
f3294a33
YL		 3949 for (i = 0; i < nr_ioapics; i++) {
3950 if (smp_found_config) {
3951 ioapic_phys = mp_ioapics[i].mp_apicaddr;
54168ed7
IM		 3952#ifdef CONFIG_X86_32
3953 if (!ioapic_phys) {
3954 printk(KERN_ERR
3955 "WARNING: bogus zero IO-APIC "
3956 "address found in MPTABLE, "
3957 "disabling IO/APIC support!\n");
3958 smp_found_config = 0;
3959 skip_ioapic_setup = 1;
3960 goto fake_ioapic_page;
3961 }
3962#endif
f3294a33 3963 } else {
54168ed7 3964#ifdef CONFIG_X86_32
f3294a33 3965fake_ioapic_page:
54168ed7 3966#endif
f3294a33 3967 ioapic_phys = (unsigned long)
54168ed7 3968 alloc_bootmem_pages(PAGE_SIZE);
f3294a33
YL		 3969 ioapic_phys = __pa(ioapic_phys);
3970 }
3971 set_fixmap_nocache(idx, ioapic_phys);
54168ed7
IM		 3972 apic_printk(APIC_VERBOSE,
3973 "mapped IOAPIC to %08lx (%08lx)\n",
3974 __fix_to_virt(idx), ioapic_phys);
f3294a33 3975 idx++;
54168ed7 3976
54168ed7
IM		 3977 if (ioapic_res != NULL) {
3978 ioapic_res->start = ioapic_phys;
3979 ioapic_res->end = ioapic_phys + (4 * 1024) - 1;
3980 ioapic_res++;
3981 }
f3294a33
YL		 3982 }
3983}
3984
54168ed7
IM		 3985static int __init ioapic_insert_resources(void)
3986{
3987 int i;
3988 struct resource *r = ioapic_resources;
3989
3990 if (!r) {
3991 printk(KERN_ERR
3992 "IO APIC resources could be not be allocated.\n");
3993 return -1;
3994 }
3995
3996 for (i = 0; i < nr_ioapics; i++) {
3997 insert_resource(&iomem_resource, r);
3998 r++;
3999 }
4000
4001 return 0;
4002}
4003
4004/* Insert the IO APIC resources after PCI initialization has occured to handle
4005 * IO APICS that are mapped in on a BAR in PCI space. */
4006late_initcall(ioapic_insert_resources);
Clone of davem's net-next-2.6 tree