[net-next-2.6.git] / drivers / pci / intr_remapping.c

#include <linux/interrupt.h>
#include <linux/dmar.h>
#include <linux/spinlock.h>
#include <linux/slab.h>
#include <linux/jiffies.h>
#include <linux/hpet.h>
#include <linux/pci.h>
#include <linux/irq.h>
#include <asm/io_apic.h>
#include <asm/smp.h>
#include <asm/cpu.h>
#include <linux/intel-iommu.h>
#include "intr_remapping.h"
#include <acpi/acpi.h>
#include <asm/pci-direct.h>
#include "pci.h"

static struct ioapic_scope ir_ioapic[MAX_IO_APICS];
static struct hpet_scope ir_hpet[MAX_HPET_TBS];
static int ir_ioapic_num, ir_hpet_num;
int intr_remapping_enabled;

static int disable_intremap;
static int disable_sourceid_checking;

static __init int setup_nointremap(char *str)
{
	disable_intremap = 1;
	return 0;
}
early_param("nointremap", setup_nointremap);

static __init int setup_intremap(char *str)
{
	if (!str)
		return -EINVAL;

	if (!strncmp(str, "on", 2))
		disable_intremap = 0;
	else if (!strncmp(str, "off", 3))
		disable_intremap = 1;
	else if (!strncmp(str, "nosid", 5))
		disable_sourceid_checking = 1;

	return 0;
}
early_param("intremap", setup_intremap);

static DEFINE_SPINLOCK(irq_2_ir_lock);

static struct irq_2_iommu *irq_2_iommu(unsigned int irq)
{
	struct irq_cfg *cfg = get_irq_chip_data(irq);
	return cfg ? &cfg->irq_2_iommu : NULL;
}

int irq_remapped(int irq)
{
	struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);

	return irq_iommu ? irq_iommu->iommu != NULL : 0;
}

int get_irte(int irq, struct irte *entry)
{
	struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
	unsigned long flags;
	int index;

	if (!entry || !irq_iommu)
		return -1;

	spin_lock_irqsave(&irq_2_ir_lock, flags);

	index = irq_iommu->irte_index + irq_iommu->sub_handle;
	*entry = *(irq_iommu->iommu->ir_table->base + index);

	spin_unlock_irqrestore(&irq_2_ir_lock, flags);
	return 0;
}

int alloc_irte(struct intel_iommu *iommu, int irq, u16 count)
{
	struct ir_table *table = iommu->ir_table;
	struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
	u16 index, start_index;
	unsigned int mask = 0;
	unsigned long flags;
	int i;

	if (!count || !irq_iommu)
		return -1;

	/*
	 * start the IRTE search from index 0.
	 */
	index = start_index = 0;

	if (count > 1) {
		count = __roundup_pow_of_two(count);
		mask = ilog2(count);
	}

	if (mask > ecap_max_handle_mask(iommu->ecap)) {
		printk(KERN_ERR
		       "Requested mask %x exceeds the max invalidation handle"
		       " mask value %Lx\n", mask,
		       ecap_max_handle_mask(iommu->ecap));
		return -1;
	}

	spin_lock_irqsave(&irq_2_ir_lock, flags);
	do {
		for (i = index; i < index + count; i++)
			if  (table->base[i].present)
				break;
		/* empty index found */
		if (i == index + count)
			break;

		index = (index + count) % INTR_REMAP_TABLE_ENTRIES;

		if (index == start_index) {
			spin_unlock_irqrestore(&irq_2_ir_lock, flags);
			printk(KERN_ERR "can't allocate an IRTE\n");
			return -1;
		}
	} while (1);

	for (i = index; i < index + count; i++)
		table->base[i].present = 1;

	irq_iommu->iommu = iommu;
	irq_iommu->irte_index =  index;
	irq_iommu->sub_handle = 0;
	irq_iommu->irte_mask = mask;

	spin_unlock_irqrestore(&irq_2_ir_lock, flags);

	return index;
}

static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask)
{
	struct qi_desc desc;

	desc.low = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask)
		   | QI_IEC_SELECTIVE;
	desc.high = 0;

	return qi_submit_sync(&desc, iommu);
}

int map_irq_to_irte_handle(int irq, u16 *sub_handle)
{
	struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
	unsigned long flags;
	int index;

	if (!irq_iommu)
		return -1;

	spin_lock_irqsave(&irq_2_ir_lock, flags);
	*sub_handle = irq_iommu->sub_handle;
	index = irq_iommu->irte_index;
	spin_unlock_irqrestore(&irq_2_ir_lock, flags);
	return index;
}

int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle)
{
	struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
	unsigned long flags;

	if (!irq_iommu)
		return -1;

	spin_lock_irqsave(&irq_2_ir_lock, flags);

	irq_iommu->iommu = iommu;
	irq_iommu->irte_index = index;
	irq_iommu->sub_handle = subhandle;
	irq_iommu->irte_mask = 0;

	spin_unlock_irqrestore(&irq_2_ir_lock, flags);

	return 0;
}

int modify_irte(int irq, struct irte *irte_modified)
{
	struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
	struct intel_iommu *iommu;
	unsigned long flags;
	struct irte *irte;
	int rc, index;

	if (!irq_iommu)
		return -1;

	spin_lock_irqsave(&irq_2_ir_lock, flags);

	iommu = irq_iommu->iommu;

	index = irq_iommu->irte_index + irq_iommu->sub_handle;
	irte = &iommu->ir_table->base[index];

	set_64bit(&irte->low, irte_modified->low);
	set_64bit(&irte->high, irte_modified->high);
	__iommu_flush_cache(iommu, irte, sizeof(*irte));

	rc = qi_flush_iec(iommu, index, 0);
	spin_unlock_irqrestore(&irq_2_ir_lock, flags);

	return rc;
}

struct intel_iommu *map_hpet_to_ir(u8 hpet_id)
{
	int i;

	for (i = 0; i < MAX_HPET_TBS; i++)
		if (ir_hpet[i].id == hpet_id)
			return ir_hpet[i].iommu;
	return NULL;
}

struct intel_iommu *map_ioapic_to_ir(int apic)
{
	int i;

	for (i = 0; i < MAX_IO_APICS; i++)
		if (ir_ioapic[i].id == apic)
			return ir_ioapic[i].iommu;
	return NULL;
}

struct intel_iommu *map_dev_to_ir(struct pci_dev *dev)
{
	struct dmar_drhd_unit *drhd;

	drhd = dmar_find_matched_drhd_unit(dev);
	if (!drhd)
		return NULL;

	return drhd->iommu;
}

static int clear_entries(struct irq_2_iommu *irq_iommu)
{
	struct irte *start, *entry, *end;
	struct intel_iommu *iommu;
	int index;

	if (irq_iommu->sub_handle)
		return 0;

	iommu = irq_iommu->iommu;
	index = irq_iommu->irte_index + irq_iommu->sub_handle;

	start = iommu->ir_table->base + index;
	end = start + (1 << irq_iommu->irte_mask);

	for (entry = start; entry < end; entry++) {
		set_64bit(&entry->low, 0);
		set_64bit(&entry->high, 0);
	}

	return qi_flush_iec(iommu, index, irq_iommu->irte_mask);
}

int free_irte(int irq)
{
	struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
	unsigned long flags;
	int rc;

	if (!irq_iommu)
		return -1;

	spin_lock_irqsave(&irq_2_ir_lock, flags);

	rc = clear_entries(irq_iommu);

	irq_iommu->iommu = NULL;
	irq_iommu->irte_index = 0;
	irq_iommu->sub_handle = 0;
	irq_iommu->irte_mask = 0;

	spin_unlock_irqrestore(&irq_2_ir_lock, flags);

	return rc;
}

/*
 * source validation type
 */
#define SVT_NO_VERIFY		0x0  /* no verification is required */
#define SVT_VERIFY_SID_SQ	0x1  /* verify using SID and SQ fiels */
#define SVT_VERIFY_BUS		0x2  /* verify bus of request-id */

/*
 * source-id qualifier
 */
#define SQ_ALL_16	0x0  /* verify all 16 bits of request-id */
#define SQ_13_IGNORE_1	0x1  /* verify most significant 13 bits, ignore
			      * the third least significant bit
			      */
#define SQ_13_IGNORE_2	0x2  /* verify most significant 13 bits, ignore
			      * the second and third least significant bits
			      */
#define SQ_13_IGNORE_3	0x3  /* verify most significant 13 bits, ignore
			      * the least three significant bits
			      */

/*
 * set SVT, SQ and SID fields of irte to verify
 * source ids of interrupt requests
 */
static void set_irte_sid(struct irte *irte, unsigned int svt,
			 unsigned int sq, unsigned int sid)
{
	if (disable_sourceid_checking)
		svt = SVT_NO_VERIFY;
	irte->svt = svt;
	irte->sq = sq;
	irte->sid = sid;
}

int set_ioapic_sid(struct irte *irte, int apic)
{
	int i;
	u16 sid = 0;

	if (!irte)
		return -1;

	for (i = 0; i < MAX_IO_APICS; i++) {
		if (ir_ioapic[i].id == apic) {
			sid = (ir_ioapic[i].bus << 8) | ir_ioapic[i].devfn;
			break;
		}
	}

	if (sid == 0) {
		pr_warning("Failed to set source-id of IOAPIC (%d)\n", apic);
		return -1;
	}

	set_irte_sid(irte, 1, 0, sid);

	return 0;
}

int set_hpet_sid(struct irte *irte, u8 id)
{
	int i;
	u16 sid = 0;

	if (!irte)
		return -1;

	for (i = 0; i < MAX_HPET_TBS; i++) {
		if (ir_hpet[i].id == id) {
			sid = (ir_hpet[i].bus << 8) | ir_hpet[i].devfn;
			break;
		}
	}

	if (sid == 0) {
		pr_warning("Failed to set source-id of HPET block (%d)\n", id);
		return -1;
	}

	/*
	 * Should really use SQ_ALL_16. Some platforms are broken.
	 * While we figure out the right quirks for these broken platforms, use
	 * SQ_13_IGNORE_3 for now.
	 */
	set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_13_IGNORE_3, sid);

	return 0;
}

int set_msi_sid(struct irte *irte, struct pci_dev *dev)
{
	struct pci_dev *bridge;

	if (!irte || !dev)
		return -1;

	/* PCIe device or Root Complex integrated PCI device */
	if (pci_is_pcie(dev) || !dev->bus->parent) {
		set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16,
			     (dev->bus->number << 8) | dev->devfn);
		return 0;
	}

	bridge = pci_find_upstream_pcie_bridge(dev);
	if (bridge) {
		if (pci_is_pcie(bridge))/* this is a PCIe-to-PCI/PCIX bridge */
			set_irte_sid(irte, SVT_VERIFY_BUS, SQ_ALL_16,
				(bridge->bus->number << 8) | dev->bus->number);
		else /* this is a legacy PCI bridge */
			set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16,
				(bridge->bus->number << 8) | bridge->devfn);
	}

	return 0;
}

static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode)
{
	u64 addr;
	u32 sts;
	unsigned long flags;

	addr = virt_to_phys((void *)iommu->ir_table->base);

	spin_lock_irqsave(&iommu->register_lock, flags);

	dmar_writeq(iommu->reg + DMAR_IRTA_REG,
		    (addr) | IR_X2APIC_MODE(mode) | INTR_REMAP_TABLE_REG_SIZE);

	/* Set interrupt-remapping table pointer */
	iommu->gcmd |= DMA_GCMD_SIRTP;
	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
		      readl, (sts & DMA_GSTS_IRTPS), sts);
	spin_unlock_irqrestore(&iommu->register_lock, flags);

	/*
	 * global invalidation of interrupt entry cache before enabling
	 * interrupt-remapping.
	 */
	qi_global_iec(iommu);

	spin_lock_irqsave(&iommu->register_lock, flags);

	/* Enable interrupt-remapping */
	iommu->gcmd |= DMA_GCMD_IRE;
	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
		      readl, (sts & DMA_GSTS_IRES), sts);

	spin_unlock_irqrestore(&iommu->register_lock, flags);
}


static int setup_intr_remapping(struct intel_iommu *iommu, int mode)
{
	struct ir_table *ir_table;
	struct page *pages;

	ir_table = iommu->ir_table = kzalloc(sizeof(struct ir_table),
					     GFP_ATOMIC);

	if (!iommu->ir_table)
		return -ENOMEM;

	pages = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO,
				 INTR_REMAP_PAGE_ORDER);

	if (!pages) {
		printk(KERN_ERR "failed to allocate pages of order %d\n",
		       INTR_REMAP_PAGE_ORDER);
		kfree(iommu->ir_table);
		return -ENOMEM;
	}

	ir_table->base = page_address(pages);

	iommu_set_intr_remapping(iommu, mode);
	return 0;
}

/*
 * Disable Interrupt Remapping.
 */
static void iommu_disable_intr_remapping(struct intel_iommu *iommu)
{
	unsigned long flags;
	u32 sts;

	if (!ecap_ir_support(iommu->ecap))
		return;

	/*
	 * global invalidation of interrupt entry cache before disabling
	 * interrupt-remapping.
	 */
	qi_global_iec(iommu);

	spin_lock_irqsave(&iommu->register_lock, flags);

	sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
	if (!(sts & DMA_GSTS_IRES))
		goto end;

	iommu->gcmd &= ~DMA_GCMD_IRE;
	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
		      readl, !(sts & DMA_GSTS_IRES), sts);

end:
	spin_unlock_irqrestore(&iommu->register_lock, flags);
}

int __init intr_remapping_supported(void)
{
	struct dmar_drhd_unit *drhd;

	if (disable_intremap)
		return 0;

	if (!dmar_ir_support())
		return 0;

	for_each_drhd_unit(drhd) {
		struct intel_iommu *iommu = drhd->iommu;

		if (!ecap_ir_support(iommu->ecap))
			return 0;
	}

	return 1;
}

int __init enable_intr_remapping(int eim)
{
	struct dmar_drhd_unit *drhd;
	int setup = 0;

	if (parse_ioapics_under_ir() != 1) {
		printk(KERN_INFO "Not enable interrupt remapping\n");
		return -1;
	}

	for_each_drhd_unit(drhd) {
		struct intel_iommu *iommu = drhd->iommu;

		/*
		 * If the queued invalidation is already initialized,
		 * shouldn't disable it.
		 */
		if (iommu->qi)
			continue;

		/*
		 * Clear previous faults.
		 */
		dmar_fault(-1, iommu);

		/*
		 * Disable intr remapping and queued invalidation, if already
		 * enabled prior to OS handover.
		 */
		iommu_disable_intr_remapping(iommu);

		dmar_disable_qi(iommu);
	}

	/*
	 * check for the Interrupt-remapping support
	 */
	for_each_drhd_unit(drhd) {
		struct intel_iommu *iommu = drhd->iommu;

		if (!ecap_ir_support(iommu->ecap))
			continue;

		if (eim && !ecap_eim_support(iommu->ecap)) {
			printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
			       " ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
			return -1;
		}
	}

	/*
	 * Enable queued invalidation for all the DRHD's.
	 */
	for_each_drhd_unit(drhd) {
		int ret;
		struct intel_iommu *iommu = drhd->iommu;
		ret = dmar_enable_qi(iommu);

		if (ret) {
			printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
			       " invalidation, ecap %Lx, ret %d\n",
			       drhd->reg_base_addr, iommu->ecap, ret);
			return -1;
		}
	}

	/*
	 * Setup Interrupt-remapping for all the DRHD's now.
	 */
	for_each_drhd_unit(drhd) {
		struct intel_iommu *iommu = drhd->iommu;

		if (!ecap_ir_support(iommu->ecap))
			continue;

		if (setup_intr_remapping(iommu, eim))
			goto error;

		setup = 1;
	}

	if (!setup)
		goto error;

	intr_remapping_enabled = 1;

	return 0;

error:
	/*
	 * handle error condition gracefully here!
	 */
	return -1;
}

static void ir_parse_one_hpet_scope(struct acpi_dmar_device_scope *scope,
				      struct intel_iommu *iommu)
{
	struct acpi_dmar_pci_path *path;
	u8 bus;
	int count;

	bus = scope->bus;
	path = (struct acpi_dmar_pci_path *)(scope + 1);
	count = (scope->length - sizeof(struct acpi_dmar_device_scope))
		/ sizeof(struct acpi_dmar_pci_path);

	while (--count > 0) {
		/*
		 * Access PCI directly due to the PCI
		 * subsystem isn't initialized yet.
		 */
		bus = read_pci_config_byte(bus, path->dev, path->fn,
					   PCI_SECONDARY_BUS);
		path++;
	}
	ir_hpet[ir_hpet_num].bus   = bus;
	ir_hpet[ir_hpet_num].devfn = PCI_DEVFN(path->dev, path->fn);
	ir_hpet[ir_hpet_num].iommu = iommu;
	ir_hpet[ir_hpet_num].id    = scope->enumeration_id;
	ir_hpet_num++;
}

static void ir_parse_one_ioapic_scope(struct acpi_dmar_device_scope *scope,
				      struct intel_iommu *iommu)
{
	struct acpi_dmar_pci_path *path;
	u8 bus;
	int count;

	bus = scope->bus;
	path = (struct acpi_dmar_pci_path *)(scope + 1);
	count = (scope->length - sizeof(struct acpi_dmar_device_scope))
		/ sizeof(struct acpi_dmar_pci_path);

	while (--count > 0) {
		/*
		 * Access PCI directly due to the PCI
		 * subsystem isn't initialized yet.
		 */
		bus = read_pci_config_byte(bus, path->dev, path->fn,
					   PCI_SECONDARY_BUS);
		path++;
	}

	ir_ioapic[ir_ioapic_num].bus   = bus;
	ir_ioapic[ir_ioapic_num].devfn = PCI_DEVFN(path->dev, path->fn);
	ir_ioapic[ir_ioapic_num].iommu = iommu;
	ir_ioapic[ir_ioapic_num].id    = scope->enumeration_id;
	ir_ioapic_num++;
}

static int ir_parse_ioapic_hpet_scope(struct acpi_dmar_header *header,
				      struct intel_iommu *iommu)
{
	struct acpi_dmar_hardware_unit *drhd;
	struct acpi_dmar_device_scope *scope;
	void *start, *end;

	drhd = (struct acpi_dmar_hardware_unit *)header;

	start = (void *)(drhd + 1);
	end = ((void *)drhd) + header->length;

	while (start < end) {
		scope = start;
		if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC) {
			if (ir_ioapic_num == MAX_IO_APICS) {
				printk(KERN_WARNING "Exceeded Max IO APICS\n");
				return -1;
			}

			printk(KERN_INFO "IOAPIC id %d under DRHD base "
			       " 0x%Lx IOMMU %d\n", scope->enumeration_id,
			       drhd->address, iommu->seq_id);

			ir_parse_one_ioapic_scope(scope, iommu);
		} else if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_HPET) {
			if (ir_hpet_num == MAX_HPET_TBS) {
				printk(KERN_WARNING "Exceeded Max HPET blocks\n");
				return -1;
			}

			printk(KERN_INFO "HPET id %d under DRHD base"
			       " 0x%Lx\n", scope->enumeration_id,
			       drhd->address);

			ir_parse_one_hpet_scope(scope, iommu);
		}
		start += scope->length;
	}

	return 0;
}

/*
 * Finds the assocaition between IOAPIC's and its Interrupt-remapping
 * hardware unit.
 */
int __init parse_ioapics_under_ir(void)
{
	struct dmar_drhd_unit *drhd;
	int ir_supported = 0;

	for_each_drhd_unit(drhd) {
		struct intel_iommu *iommu = drhd->iommu;

		if (ecap_ir_support(iommu->ecap)) {
			if (ir_parse_ioapic_hpet_scope(drhd->hdr, iommu))
				return -1;

			ir_supported = 1;
		}
	}

	if (ir_supported && ir_ioapic_num != nr_ioapics) {
		printk(KERN_WARNING
		       "Not all IO-APIC's listed under remapping hardware\n");
		return -1;
	}

	return ir_supported;
}

void disable_intr_remapping(void)
{
	struct dmar_drhd_unit *drhd;
	struct intel_iommu *iommu = NULL;

	/*
	 * Disable Interrupt-remapping for all the DRHD's now.
	 */
	for_each_iommu(iommu, drhd) {
		if (!ecap_ir_support(iommu->ecap))
			continue;

		iommu_disable_intr_remapping(iommu);
	}
}

int reenable_intr_remapping(int eim)
{
	struct dmar_drhd_unit *drhd;
	int setup = 0;
	struct intel_iommu *iommu = NULL;

	for_each_iommu(iommu, drhd)
		if (iommu->qi)
			dmar_reenable_qi(iommu);

	/*
	 * Setup Interrupt-remapping for all the DRHD's now.
	 */
	for_each_iommu(iommu, drhd) {
		if (!ecap_ir_support(iommu->ecap))
			continue;

		/* Set up interrupt remapping for iommu.*/
		iommu_set_intr_remapping(iommu, eim);
		setup = 1;
	}

	if (!setup)
		goto error;

	return 0;

error:
	/*
	 * handle error condition gracefully here!
	 */
	return -1;
}
Commit	Line	Data
5aeecaf4	1	#include <linux/interrupt.h>
ad3ad3f6	2	#include <linux/dmar.h>
2ae21010	3	#include <linux/spinlock.h>
5a0e3ad6	4	#include <linux/slab.h>
2ae21010	5	#include <linux/jiffies.h>
20f3097b	6	#include <linux/hpet.h>
2ae21010	7	#include <linux/pci.h>
b6fcb33a	8	#include <linux/irq.h>
ad3ad3f6	9	#include <asm/io_apic.h>
17483a1f	10	#include <asm/smp.h>
6d652ea1	11	#include <asm/cpu.h>
38717946	12	#include <linux/intel-iommu.h>
ad3ad3f6	13	#include "intr_remapping.h"
46f06b72	14	#include <acpi/acpi.h>
f007e99c WH	15	#include <asm/pci-direct.h>
f007e99c WH	16	#include "pci.h"
ad3ad3f6 SS	17
ad3ad3f6 SS	18	static struct ioapic_scope ir_ioapic[MAX_IO_APICS];
20f3097b SS	19	static struct hpet_scope ir_hpet[MAX_HPET_TBS];
20f3097b SS	20	static int ir_ioapic_num, ir_hpet_num;
2ae21010 SS	21	int intr_remapping_enabled;
2ae21010 SS	22
03ea8155	23	static int disable_intremap;
d1423d56 CW	24	static int disable_sourceid_checking;
d1423d56 CW	25
03ea8155 WH	26	static __init int setup_nointremap(char *str)
	27	{
	28	disable_intremap = 1;
	29	return 0;
	30	}
	31	early_param("nointremap", setup_nointremap);
	32
d1423d56 CW	33	static __init int setup_intremap(char *str)
	34	{
	35	if (!str)
	36	return -EINVAL;
	37
	38	if (!strncmp(str, "on", 2))
	39	disable_intremap = 0;
	40	else if (!strncmp(str, "off", 3))
	41	disable_intremap = 1;
	42	else if (!strncmp(str, "nosid", 5))
	43	disable_sourceid_checking = 1;
	44
	45	return 0;
	46	}
	47	early_param("intremap", setup_intremap);
	48
d585d060 TG	49	static DEFINE_SPINLOCK(irq_2_ir_lock);
d585d060 TG	50
e420dfb4 YL	51	static struct irq_2_iommu *irq_2_iommu(unsigned int irq)
e420dfb4 YL	52	{
349d6767 TG	53	struct irq_cfg *cfg = get_irq_chip_data(irq);
349d6767 TG	54	return cfg ? &cfg->irq_2_iommu : NULL;
0b8f1efa YL	55	}
0b8f1efa YL	56
e420dfb4 YL	57	int irq_remapped(int irq)
e420dfb4 YL	58	{
d585d060 TG	59	struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
	60
	61	return irq_iommu ? irq_iommu->iommu != NULL : 0;
b6fcb33a SS	62	}
	63
	64	int get_irte(int irq, struct irte *entry)
	65	{
d585d060	66	struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
4c5502b1	67	unsigned long flags;
d585d060	68	int index;
b6fcb33a	69
d585d060	70	if (!entry \|\| !irq_iommu)
b6fcb33a SS	71	return -1;
b6fcb33a SS	72
4c5502b1	73	spin_lock_irqsave(&irq_2_ir_lock, flags);
b6fcb33a	74
e420dfb4 YL	75	index = irq_iommu->irte_index + irq_iommu->sub_handle;
e420dfb4 YL	76	entry = (irq_iommu->iommu->ir_table->base + index);
b6fcb33a	77
4c5502b1	78	spin_unlock_irqrestore(&irq_2_ir_lock, flags);
b6fcb33a SS	79	return 0;
	80	}
	81
	82	int alloc_irte(struct intel_iommu *iommu, int irq, u16 count)
	83	{
	84	struct ir_table *table = iommu->ir_table;
d585d060	85	struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
b6fcb33a SS	86	u16 index, start_index;
b6fcb33a SS	87	unsigned int mask = 0;
4c5502b1	88	unsigned long flags;
b6fcb33a SS	89	int i;
b6fcb33a SS	90
d585d060	91	if (!count \|\| !irq_iommu)
e420dfb4	92	return -1;
e420dfb4	93
b6fcb33a SS	94	/*
	95	* start the IRTE search from index 0.
	96	*/
	97	index = start_index = 0;
	98
	99	if (count > 1) {
	100	count = __roundup_pow_of_two(count);
	101	mask = ilog2(count);
	102	}
	103
	104	if (mask > ecap_max_handle_mask(iommu->ecap)) {
	105	printk(KERN_ERR
	106	"Requested mask %x exceeds the max invalidation handle"
	107	" mask value %Lx\n", mask,
	108	ecap_max_handle_mask(iommu->ecap));
	109	return -1;
	110	}
	111
4c5502b1	112	spin_lock_irqsave(&irq_2_ir_lock, flags);
b6fcb33a SS	113	do {
	114	for (i = index; i < index + count; i++)
	115	if (table->base[i].present)
	116	break;
	117	/* empty index found */
	118	if (i == index + count)
	119	break;
	120
	121	index = (index + count) % INTR_REMAP_TABLE_ENTRIES;
	122
	123	if (index == start_index) {
4c5502b1	124	spin_unlock_irqrestore(&irq_2_ir_lock, flags);
b6fcb33a SS	125	printk(KERN_ERR "can't allocate an IRTE\n");
	126	return -1;
	127	}
	128	} while (1);
	129
	130	for (i = index; i < index + count; i++)
	131	table->base[i].present = 1;
	132
e420dfb4 YL	133	irq_iommu->iommu = iommu;
	134	irq_iommu->irte_index = index;
	135	irq_iommu->sub_handle = 0;
	136	irq_iommu->irte_mask = mask;
b6fcb33a	137
4c5502b1	138	spin_unlock_irqrestore(&irq_2_ir_lock, flags);
b6fcb33a SS	139
	140	return index;
	141	}
	142
704126ad	143	static int qi_flush_iec(struct intel_iommu *iommu, int index, int mask)
b6fcb33a SS	144	{
	145	struct qi_desc desc;
	146
	147	desc.low = QI_IEC_IIDEX(index) \| QI_IEC_TYPE \| QI_IEC_IM(mask)
	148	\| QI_IEC_SELECTIVE;
	149	desc.high = 0;
	150
704126ad	151	return qi_submit_sync(&desc, iommu);
b6fcb33a SS	152	}
	153
	154	int map_irq_to_irte_handle(int irq, u16 *sub_handle)
	155	{
d585d060	156	struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
4c5502b1	157	unsigned long flags;
d585d060	158	int index;
b6fcb33a	159
d585d060	160	if (!irq_iommu)
b6fcb33a	161	return -1;
b6fcb33a	162
d585d060	163	spin_lock_irqsave(&irq_2_ir_lock, flags);
e420dfb4 YL	164	*sub_handle = irq_iommu->sub_handle;
e420dfb4 YL	165	index = irq_iommu->irte_index;
4c5502b1	166	spin_unlock_irqrestore(&irq_2_ir_lock, flags);
b6fcb33a SS	167	return index;
	168	}
	169
	170	int set_irte_irq(int irq, struct intel_iommu *iommu, u16 index, u16 subhandle)
	171	{
d585d060	172	struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
4c5502b1	173	unsigned long flags;
e420dfb4	174
d585d060	175	if (!irq_iommu)
0b8f1efa	176	return -1;
d585d060 TG	177
d585d060 TG	178	spin_lock_irqsave(&irq_2_ir_lock, flags);
0b8f1efa	179
e420dfb4 YL	180	irq_iommu->iommu = iommu;
	181	irq_iommu->irte_index = index;
	182	irq_iommu->sub_handle = subhandle;
	183	irq_iommu->irte_mask = 0;
b6fcb33a	184
4c5502b1	185	spin_unlock_irqrestore(&irq_2_ir_lock, flags);
b6fcb33a SS	186
	187	return 0;
	188	}
	189
b6fcb33a SS	190	int modify_irte(int irq, struct irte *irte_modified)
b6fcb33a SS	191	{
d585d060	192	struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
b6fcb33a	193	struct intel_iommu *iommu;
4c5502b1	194	unsigned long flags;
d585d060 TG	195	struct irte *irte;
d585d060 TG	196	int rc, index;
b6fcb33a	197
d585d060	198	if (!irq_iommu)
b6fcb33a	199	return -1;
d585d060 TG	200
d585d060 TG	201	spin_lock_irqsave(&irq_2_ir_lock, flags);
b6fcb33a	202
e420dfb4	203	iommu = irq_iommu->iommu;
b6fcb33a	204
e420dfb4	205	index = irq_iommu->irte_index + irq_iommu->sub_handle;
b6fcb33a SS	206	irte = &iommu->ir_table->base[index];
b6fcb33a SS	207
c513b67e LT	208	set_64bit(&irte->low, irte_modified->low);
c513b67e LT	209	set_64bit(&irte->high, irte_modified->high);
b6fcb33a SS	210	__iommu_flush_cache(iommu, irte, sizeof(*irte));
b6fcb33a SS	211
704126ad	212	rc = qi_flush_iec(iommu, index, 0);
4c5502b1	213	spin_unlock_irqrestore(&irq_2_ir_lock, flags);
704126ad YZ	214
704126ad YZ	215	return rc;
b6fcb33a SS	216	}
b6fcb33a SS	217
20f3097b SS	218	struct intel_iommu *map_hpet_to_ir(u8 hpet_id)
	219	{
	220	int i;
	221
	222	for (i = 0; i < MAX_HPET_TBS; i++)
	223	if (ir_hpet[i].id == hpet_id)
	224	return ir_hpet[i].iommu;
	225	return NULL;
	226	}
	227
89027d35 SS	228	struct intel_iommu *map_ioapic_to_ir(int apic)
	229	{
	230	int i;
	231
	232	for (i = 0; i < MAX_IO_APICS; i++)
	233	if (ir_ioapic[i].id == apic)
	234	return ir_ioapic[i].iommu;
	235	return NULL;
	236	}
	237
75c46fa6 SS	238	struct intel_iommu map_dev_to_ir(struct pci_dev dev)
	239	{
	240	struct dmar_drhd_unit *drhd;
	241
	242	drhd = dmar_find_matched_drhd_unit(dev);
	243	if (!drhd)
	244	return NULL;
	245
	246	return drhd->iommu;
	247	}
	248
c4658b4e WH	249	static int clear_entries(struct irq_2_iommu *irq_iommu)
	250	{
	251	struct irte start, entry, *end;
	252	struct intel_iommu *iommu;
	253	int index;
	254
	255	if (irq_iommu->sub_handle)
	256	return 0;
	257
	258	iommu = irq_iommu->iommu;
	259	index = irq_iommu->irte_index + irq_iommu->sub_handle;
	260
	261	start = iommu->ir_table->base + index;
	262	end = start + (1 << irq_iommu->irte_mask);
	263
	264	for (entry = start; entry < end; entry++) {
c513b67e LT	265	set_64bit(&entry->low, 0);
c513b67e LT	266	set_64bit(&entry->high, 0);
c4658b4e WH	267	}
	268
	269	return qi_flush_iec(iommu, index, irq_iommu->irte_mask);
	270	}
	271
b6fcb33a SS	272	int free_irte(int irq)
b6fcb33a SS	273	{
d585d060	274	struct irq_2_iommu *irq_iommu = irq_2_iommu(irq);
4c5502b1	275	unsigned long flags;
d585d060	276	int rc;
b6fcb33a	277
d585d060	278	if (!irq_iommu)
b6fcb33a	279	return -1;
d585d060 TG	280
d585d060 TG	281	spin_lock_irqsave(&irq_2_ir_lock, flags);
b6fcb33a	282
c4658b4e	283	rc = clear_entries(irq_iommu);
b6fcb33a	284
e420dfb4 YL	285	irq_iommu->iommu = NULL;
	286	irq_iommu->irte_index = 0;
	287	irq_iommu->sub_handle = 0;
	288	irq_iommu->irte_mask = 0;
b6fcb33a	289
4c5502b1	290	spin_unlock_irqrestore(&irq_2_ir_lock, flags);
b6fcb33a	291
704126ad	292	return rc;
b6fcb33a SS	293	}
b6fcb33a SS	294
f007e99c WH	295	/*
	296	* source validation type
	297	*/
	298	#define SVT_NO_VERIFY 0x0 /* no verification is required */
	299	#define SVT_VERIFY_SID_SQ 0x1 /* verify using SID and SQ fiels */
	300	#define SVT_VERIFY_BUS 0x2 /* verify bus of request-id */
	301
	302	/*
	303	* source-id qualifier
	304	*/
	305	#define SQ_ALL_16 0x0 /* verify all 16 bits of request-id */
	306	#define SQ_13_IGNORE_1 0x1 /* verify most significant 13 bits, ignore
	307	* the third least significant bit
	308	*/
	309	#define SQ_13_IGNORE_2 0x2 /* verify most significant 13 bits, ignore
	310	* the second and third least significant bits
	311	*/
	312	#define SQ_13_IGNORE_3 0x3 /* verify most significant 13 bits, ignore
	313	* the least three significant bits
	314	*/
	315
	316	/*
	317	* set SVT, SQ and SID fields of irte to verify
	318	* source ids of interrupt requests
	319	*/
	320	static void set_irte_sid(struct irte *irte, unsigned int svt,
	321	unsigned int sq, unsigned int sid)
	322	{
d1423d56 CW	323	if (disable_sourceid_checking)
d1423d56 CW	324	svt = SVT_NO_VERIFY;
f007e99c WH	325	irte->svt = svt;
	326	irte->sq = sq;
	327	irte->sid = sid;
	328	}
	329
	330	int set_ioapic_sid(struct irte *irte, int apic)
	331	{
	332	int i;
	333	u16 sid = 0;
	334
	335	if (!irte)
	336	return -1;
	337
	338	for (i = 0; i < MAX_IO_APICS; i++) {
	339	if (ir_ioapic[i].id == apic) {
	340	sid = (ir_ioapic[i].bus << 8) \| ir_ioapic[i].devfn;
	341	break;
	342	}
	343	}
	344
	345	if (sid == 0) {
	346	pr_warning("Failed to set source-id of IOAPIC (%d)\n", apic);
	347	return -1;
	348	}
	349
	350	set_irte_sid(irte, 1, 0, sid);
	351
	352	return 0;
	353	}
	354
20f3097b SS	355	int set_hpet_sid(struct irte *irte, u8 id)
	356	{
	357	int i;
	358	u16 sid = 0;
	359
	360	if (!irte)
	361	return -1;
	362
	363	for (i = 0; i < MAX_HPET_TBS; i++) {
	364	if (ir_hpet[i].id == id) {
	365	sid = (ir_hpet[i].bus << 8) \| ir_hpet[i].devfn;
	366	break;
	367	}
	368	}
	369
	370	if (sid == 0) {
	371	pr_warning("Failed to set source-id of HPET block (%d)\n", id);
	372	return -1;
	373	}
	374
	375	/*
	376	* Should really use SQ_ALL_16. Some platforms are broken.
	377	* While we figure out the right quirks for these broken platforms, use
	378	* SQ_13_IGNORE_3 for now.
	379	*/
	380	set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_13_IGNORE_3, sid);
	381
	382	return 0;
	383	}
	384
f007e99c WH	385	int set_msi_sid(struct irte irte, struct pci_dev dev)
	386	{
	387	struct pci_dev *bridge;
	388
	389	if (!irte \|\| !dev)
	390	return -1;
	391
	392	/* PCIe device or Root Complex integrated PCI device */
5f4d91a1	393	if (pci_is_pcie(dev) \|\| !dev->bus->parent) {
f007e99c WH	394	set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16,
	395	(dev->bus->number << 8) \| dev->devfn);
	396	return 0;
	397	}
	398
	399	bridge = pci_find_upstream_pcie_bridge(dev);
	400	if (bridge) {
45e829ea	401	if (pci_is_pcie(bridge))/* this is a PCIe-to-PCI/PCIX bridge */
f007e99c WH	402	set_irte_sid(irte, SVT_VERIFY_BUS, SQ_ALL_16,
	403	(bridge->bus->number << 8) \| dev->bus->number);
	404	else /* this is a legacy PCI bridge */
	405	set_irte_sid(irte, SVT_VERIFY_SID_SQ, SQ_ALL_16,
	406	(bridge->bus->number << 8) \| bridge->devfn);
	407	}
	408
	409	return 0;
	410	}
	411
2ae21010 SS	412	static void iommu_set_intr_remapping(struct intel_iommu *iommu, int mode)
	413	{
	414	u64 addr;
c416daa9	415	u32 sts;
2ae21010 SS	416	unsigned long flags;
	417
	418	addr = virt_to_phys((void *)iommu->ir_table->base);
	419
	420	spin_lock_irqsave(&iommu->register_lock, flags);
	421
	422	dmar_writeq(iommu->reg + DMAR_IRTA_REG,
	423	(addr) \| IR_X2APIC_MODE(mode) \| INTR_REMAP_TABLE_REG_SIZE);
	424
	425	/* Set interrupt-remapping table pointer */
161fde08	426	iommu->gcmd \|= DMA_GCMD_SIRTP;
c416daa9	427	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
2ae21010 SS	428
	429	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
	430	readl, (sts & DMA_GSTS_IRTPS), sts);
	431	spin_unlock_irqrestore(&iommu->register_lock, flags);
	432
	433	/*
	434	* global invalidation of interrupt entry cache before enabling
	435	* interrupt-remapping.
	436	*/
	437	qi_global_iec(iommu);
	438
	439	spin_lock_irqsave(&iommu->register_lock, flags);
	440
	441	/* Enable interrupt-remapping */
2ae21010	442	iommu->gcmd \|= DMA_GCMD_IRE;
c416daa9	443	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
2ae21010 SS	444
	445	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
	446	readl, (sts & DMA_GSTS_IRES), sts);
	447
	448	spin_unlock_irqrestore(&iommu->register_lock, flags);
	449	}
	450
	451
	452	static int setup_intr_remapping(struct intel_iommu *iommu, int mode)
	453	{
	454	struct ir_table *ir_table;
	455	struct page *pages;
	456
	457	ir_table = iommu->ir_table = kzalloc(sizeof(struct ir_table),
fa4b57cc	458	GFP_ATOMIC);
2ae21010 SS	459
	460	if (!iommu->ir_table)
	461	return -ENOMEM;
	462
824cd75b SS	463	pages = alloc_pages_node(iommu->node, GFP_ATOMIC \| __GFP_ZERO,
824cd75b SS	464	INTR_REMAP_PAGE_ORDER);
2ae21010 SS	465
	466	if (!pages) {
	467	printk(KERN_ERR "failed to allocate pages of order %d\n",
	468	INTR_REMAP_PAGE_ORDER);
	469	kfree(iommu->ir_table);
	470	return -ENOMEM;
	471	}
	472
	473	ir_table->base = page_address(pages);
	474
	475	iommu_set_intr_remapping(iommu, mode);
	476	return 0;
	477	}
	478
eba67e5d SS	479	/*
	480	* Disable Interrupt Remapping.
	481	*/
b24696bc	482	static void iommu_disable_intr_remapping(struct intel_iommu *iommu)
eba67e5d SS	483	{
	484	unsigned long flags;
	485	u32 sts;
	486
	487	if (!ecap_ir_support(iommu->ecap))
	488	return;
	489
b24696bc FY	490	/*
	491	* global invalidation of interrupt entry cache before disabling
	492	* interrupt-remapping.
	493	*/
	494	qi_global_iec(iommu);
	495
eba67e5d SS	496	spin_lock_irqsave(&iommu->register_lock, flags);
	497
	498	sts = dmar_readq(iommu->reg + DMAR_GSTS_REG);
	499	if (!(sts & DMA_GSTS_IRES))
	500	goto end;
	501
	502	iommu->gcmd &= ~DMA_GCMD_IRE;
	503	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
	504
	505	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
	506	readl, !(sts & DMA_GSTS_IRES), sts);
	507
	508	end:
	509	spin_unlock_irqrestore(&iommu->register_lock, flags);
	510	}
	511
93758238 WH	512	int __init intr_remapping_supported(void)
	513	{
	514	struct dmar_drhd_unit *drhd;
	515
03ea8155 WH	516	if (disable_intremap)
	517	return 0;
	518
074835f0 YS	519	if (!dmar_ir_support())
	520	return 0;
	521
93758238 WH	522	for_each_drhd_unit(drhd) {
	523	struct intel_iommu *iommu = drhd->iommu;
	524
	525	if (!ecap_ir_support(iommu->ecap))
	526	return 0;
	527	}
	528
	529	return 1;
	530	}
	531
2ae21010 SS	532	int __init enable_intr_remapping(int eim)
	533	{
	534	struct dmar_drhd_unit *drhd;
	535	int setup = 0;
	536
e936d077 YS	537	if (parse_ioapics_under_ir() != 1) {
	538	printk(KERN_INFO "Not enable interrupt remapping\n");
	539	return -1;
	540	}
	541
1531a6a6 SS	542	for_each_drhd_unit(drhd) {
	543	struct intel_iommu *iommu = drhd->iommu;
	544
34aaaa94 HW	545	/*
	546	* If the queued invalidation is already initialized,
	547	* shouldn't disable it.
	548	*/
	549	if (iommu->qi)
	550	continue;
	551
1531a6a6 SS	552	/*
	553	* Clear previous faults.
	554	*/
	555	dmar_fault(-1, iommu);
	556
	557	/*
	558	* Disable intr remapping and queued invalidation, if already
	559	* enabled prior to OS handover.
	560	*/
b24696bc	561	iommu_disable_intr_remapping(iommu);
1531a6a6 SS	562
	563	dmar_disable_qi(iommu);
	564	}
	565
2ae21010 SS	566	/*
	567	* check for the Interrupt-remapping support
	568	*/
	569	for_each_drhd_unit(drhd) {
	570	struct intel_iommu *iommu = drhd->iommu;
	571
	572	if (!ecap_ir_support(iommu->ecap))
	573	continue;
	574
	575	if (eim && !ecap_eim_support(iommu->ecap)) {
	576	printk(KERN_INFO "DRHD %Lx: EIM not supported by DRHD, "
	577	" ecap %Lx\n", drhd->reg_base_addr, iommu->ecap);
	578	return -1;
	579	}
	580	}
	581
	582	/*
	583	* Enable queued invalidation for all the DRHD's.
	584	*/
	585	for_each_drhd_unit(drhd) {
	586	int ret;
	587	struct intel_iommu *iommu = drhd->iommu;
	588	ret = dmar_enable_qi(iommu);
	589
	590	if (ret) {
	591	printk(KERN_ERR "DRHD %Lx: failed to enable queued, "
	592	" invalidation, ecap %Lx, ret %d\n",
	593	drhd->reg_base_addr, iommu->ecap, ret);
	594	return -1;
	595	}
	596	}
	597
	598	/*
	599	* Setup Interrupt-remapping for all the DRHD's now.
	600	*/
	601	for_each_drhd_unit(drhd) {
	602	struct intel_iommu *iommu = drhd->iommu;
	603
	604	if (!ecap_ir_support(iommu->ecap))
	605	continue;
	606
	607	if (setup_intr_remapping(iommu, eim))
	608	goto error;
	609
	610	setup = 1;
	611	}
	612
	613	if (!setup)
	614	goto error;
	615
	616	intr_remapping_enabled = 1;
	617
	618	return 0;
	619
	620	error:
	621	/*
	622	* handle error condition gracefully here!
	623	*/
	624	return -1;
	625	}
ad3ad3f6	626
20f3097b SS	627	static void ir_parse_one_hpet_scope(struct acpi_dmar_device_scope *scope,
	628	struct intel_iommu *iommu)
	629	{
	630	struct acpi_dmar_pci_path *path;
	631	u8 bus;
	632	int count;
	633
	634	bus = scope->bus;
	635	path = (struct acpi_dmar_pci_path *)(scope + 1);
	636	count = (scope->length - sizeof(struct acpi_dmar_device_scope))
	637	/ sizeof(struct acpi_dmar_pci_path);
	638
	639	while (--count > 0) {
	640	/*
	641	* Access PCI directly due to the PCI
	642	* subsystem isn't initialized yet.
	643	*/
	644	bus = read_pci_config_byte(bus, path->dev, path->fn,
	645	PCI_SECONDARY_BUS);
	646	path++;
	647	}
	648	ir_hpet[ir_hpet_num].bus = bus;
	649	ir_hpet[ir_hpet_num].devfn = PCI_DEVFN(path->dev, path->fn);
	650	ir_hpet[ir_hpet_num].iommu = iommu;
	651	ir_hpet[ir_hpet_num].id = scope->enumeration_id;
	652	ir_hpet_num++;
	653	}
	654
f007e99c WH	655	static void ir_parse_one_ioapic_scope(struct acpi_dmar_device_scope *scope,
	656	struct intel_iommu *iommu)
	657	{
	658	struct acpi_dmar_pci_path *path;
	659	u8 bus;
	660	int count;
	661
	662	bus = scope->bus;
	663	path = (struct acpi_dmar_pci_path *)(scope + 1);
	664	count = (scope->length - sizeof(struct acpi_dmar_device_scope))
	665	/ sizeof(struct acpi_dmar_pci_path);
	666
	667	while (--count > 0) {
	668	/*
	669	* Access PCI directly due to the PCI
	670	* subsystem isn't initialized yet.
	671	*/
	672	bus = read_pci_config_byte(bus, path->dev, path->fn,
	673	PCI_SECONDARY_BUS);
	674	path++;
	675	}
	676
	677	ir_ioapic[ir_ioapic_num].bus = bus;
	678	ir_ioapic[ir_ioapic_num].devfn = PCI_DEVFN(path->dev, path->fn);
	679	ir_ioapic[ir_ioapic_num].iommu = iommu;
	680	ir_ioapic[ir_ioapic_num].id = scope->enumeration_id;
	681	ir_ioapic_num++;
	682	}
	683
20f3097b SS	684	static int ir_parse_ioapic_hpet_scope(struct acpi_dmar_header *header,
20f3097b SS	685	struct intel_iommu *iommu)
ad3ad3f6 SS	686	{
	687	struct acpi_dmar_hardware_unit *drhd;
	688	struct acpi_dmar_device_scope *scope;
	689	void start, end;
	690
	691	drhd = (struct acpi_dmar_hardware_unit *)header;
	692
	693	start = (void *)(drhd + 1);
	694	end = ((void *)drhd) + header->length;
	695
	696	while (start < end) {
	697	scope = start;
	698	if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_IOAPIC) {
	699	if (ir_ioapic_num == MAX_IO_APICS) {
	700	printk(KERN_WARNING "Exceeded Max IO APICS\n");
	701	return -1;
	702	}
	703
680a7524 YL	704	printk(KERN_INFO "IOAPIC id %d under DRHD base "
	705	" 0x%Lx IOMMU %d\n", scope->enumeration_id,
	706	drhd->address, iommu->seq_id);
ad3ad3f6	707
f007e99c	708	ir_parse_one_ioapic_scope(scope, iommu);
20f3097b SS	709	} else if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_HPET) {
	710	if (ir_hpet_num == MAX_HPET_TBS) {
	711	printk(KERN_WARNING "Exceeded Max HPET blocks\n");
	712	return -1;
	713	}
	714
	715	printk(KERN_INFO "HPET id %d under DRHD base"
	716	" 0x%Lx\n", scope->enumeration_id,
	717	drhd->address);
	718
	719	ir_parse_one_hpet_scope(scope, iommu);
ad3ad3f6 SS	720	}
	721	start += scope->length;
	722	}
	723
	724	return 0;
	725	}
	726
	727	/*
	728	* Finds the assocaition between IOAPIC's and its Interrupt-remapping
	729	* hardware unit.
	730	*/
	731	int __init parse_ioapics_under_ir(void)
	732	{
	733	struct dmar_drhd_unit *drhd;
	734	int ir_supported = 0;
	735
	736	for_each_drhd_unit(drhd) {
	737	struct intel_iommu *iommu = drhd->iommu;
	738
	739	if (ecap_ir_support(iommu->ecap)) {
20f3097b	740	if (ir_parse_ioapic_hpet_scope(drhd->hdr, iommu))
ad3ad3f6 SS	741	return -1;
	742
	743	ir_supported = 1;
	744	}
	745	}
	746
	747	if (ir_supported && ir_ioapic_num != nr_ioapics) {
	748	printk(KERN_WARNING
	749	"Not all IO-APIC's listed under remapping hardware\n");
	750	return -1;
	751	}
	752
	753	return ir_supported;
	754	}
b24696bc FY	755
	756	void disable_intr_remapping(void)
	757	{
	758	struct dmar_drhd_unit *drhd;
	759	struct intel_iommu *iommu = NULL;
	760
	761	/*
	762	* Disable Interrupt-remapping for all the DRHD's now.
	763	*/
	764	for_each_iommu(iommu, drhd) {
	765	if (!ecap_ir_support(iommu->ecap))
	766	continue;
	767
	768	iommu_disable_intr_remapping(iommu);
	769	}
	770	}
	771
	772	int reenable_intr_remapping(int eim)
	773	{
	774	struct dmar_drhd_unit *drhd;
	775	int setup = 0;
	776	struct intel_iommu *iommu = NULL;
	777
	778	for_each_iommu(iommu, drhd)
	779	if (iommu->qi)
	780	dmar_reenable_qi(iommu);
	781
	782	/*
	783	* Setup Interrupt-remapping for all the DRHD's now.
	784	*/
	785	for_each_iommu(iommu, drhd) {
	786	if (!ecap_ir_support(iommu->ecap))
	787	continue;
	788
	789	/* Set up interrupt remapping for iommu.*/
	790	iommu_set_intr_remapping(iommu, eim);
	791	setup = 1;
	792	}
	793
	794	if (!setup)
	795	goto error;
	796
	797	return 0;
	798
	799	error:
	800	/*
	801	* handle error condition gracefully here!
	802	*/
	803	return -1;
	804	}
	805