]> bbs.cooldavid.org Git - net-next-2.6.git/blame - drivers/pci/intel-iommu.c
git://bbs.cooldavid.org / net-next-2.6.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/bp/bp
[net-next-2.6.git] / drivers / pci / intel-iommu.c
CommitLineData
ba395927
KA		 1/*
2 * Copyright (c) 2006, Intel Corporation.
3 *
4 * This program is free software; you can redistribute it and/or modify it
5 * under the terms and conditions of the GNU General Public License,
6 * version 2, as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope it will be useful, but WITHOUT
9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
11 * more details.
12 *
13 * You should have received a copy of the GNU General Public License along with
14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
15 * Place - Suite 330, Boston, MA 02111-1307 USA.
16 *
98bcef56 17 * Copyright (C) 2006-2008 Intel Corporation
18 * Author: Ashok Raj <ashok.raj@intel.com>
19 * Author: Shaohua Li <shaohua.li@intel.com>
20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
5b6985ce 21 * Author: Fenghua Yu <fenghua.yu@intel.com>
ba395927
KA		 22 */
23
24#include <linux/init.h>
25#include <linux/bitmap.h>
5e0d2a6f 26#include <linux/debugfs.h>
ba395927
KA		 27#include <linux/slab.h>
28#include <linux/irq.h>
29#include <linux/interrupt.h>
ba395927
KA		 30#include <linux/spinlock.h>
31#include <linux/pci.h>
32#include <linux/dmar.h>
33#include <linux/dma-mapping.h>
34#include <linux/mempool.h>
5e0d2a6f 35#include <linux/timer.h>
38717946 36#include <linux/iova.h>
5d450806 37#include <linux/iommu.h>
38717946 38#include <linux/intel-iommu.h>
f59c7b69 39#include <linux/sysdev.h>
69575d38 40#include <linux/tboot.h>
adb2fe02 41#include <linux/dmi.h>
ba395927 42#include <asm/cacheflush.h>
46a7fa27 43#include <asm/iommu.h>
ba395927
KA		 44#include "pci.h"
45
5b6985ce
FY		 46#define ROOT_SIZE VTD_PAGE_SIZE
47#define CONTEXT_SIZE VTD_PAGE_SIZE
48
ba395927
KA		 49#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
50#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
e0fc7e0b 51#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
ba395927
KA		 52
53#define IOAPIC_RANGE_START (0xfee00000)
54#define IOAPIC_RANGE_END (0xfeefffff)
55#define IOVA_START_ADDR (0x1000)
56
57#define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
58
4ed0d3e6
FY		 59#define MAX_AGAW_WIDTH 64
60
2ebe3151
DW		 61#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << (gaw-VTD_PAGE_SHIFT)) - 1)
62#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << gaw) - 1)
63
64/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
65 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
66#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
67 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
68#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
ba395927 69
f27be03b 70#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
284901a9 71#define DMA_32BIT_PFN IOVA_PFN(DMA_BIT_MASK(32))
6a35528a 72#define DMA_64BIT_PFN IOVA_PFN(DMA_BIT_MASK(64))
5e0d2a6f 73
fd18de50 74
dd4e8319
DW		 75/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
76 are never going to work. */
77static inline unsigned long dma_to_mm_pfn(unsigned long dma_pfn)
78{
79 return dma_pfn >> (PAGE_SHIFT - VTD_PAGE_SHIFT);
80}
81
82static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
83{
84 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
85}
86static inline unsigned long page_to_dma_pfn(struct page *pg)
87{
88 return mm_to_dma_pfn(page_to_pfn(pg));
89}
90static inline unsigned long virt_to_dma_pfn(void *p)
91{
92 return page_to_dma_pfn(virt_to_page(p));
93}
94
d9630fe9
WH		 95/* global iommu list, set NULL for ignored DMAR units */
96static struct intel_iommu **g_iommus;
97
e0fc7e0b 98static void __init check_tylersburg_isoch(void);
9af88143
DW		 99static int rwbf_quirk;
100
46b08e1a
MM		 101/*
102 * 0: Present
103 * 1-11: Reserved
104 * 12-63: Context Ptr (12 - (haw-1))
105 * 64-127: Reserved
106 */
107struct root_entry {
108 u64 val;
109 u64 rsvd1;
110};
111#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
112static inline bool root_present(struct root_entry *root)
113{
114 return (root->val & 1);
115}
116static inline void set_root_present(struct root_entry *root)
117{
118 root->val |= 1;
119}
120static inline void set_root_value(struct root_entry *root, unsigned long value)
121{
122 root->val |= value & VTD_PAGE_MASK;
123}
124
125static inline struct context_entry *
126get_context_addr_from_root(struct root_entry *root)
127{
128 return (struct context_entry *)
129 (root_present(root)?phys_to_virt(
130 root->val & VTD_PAGE_MASK) :
131 NULL);
132}
133
7a8fc25e
MM		 134/*
135 * low 64 bits:
136 * 0: present
137 * 1: fault processing disable
138 * 2-3: translation type
139 * 12-63: address space root
140 * high 64 bits:
141 * 0-2: address width
142 * 3-6: aval
143 * 8-23: domain id
144 */
145struct context_entry {
146 u64 lo;
147 u64 hi;
148};
c07e7d21
MM		 149
150static inline bool context_present(struct context_entry *context)
151{
152 return (context->lo & 1);
153}
154static inline void context_set_present(struct context_entry *context)
155{
156 context->lo |= 1;
157}
158
159static inline void context_set_fault_enable(struct context_entry *context)
160{
161 context->lo &= (((u64)-1) << 2) | 1;
162}
163
c07e7d21
MM		 164static inline void context_set_translation_type(struct context_entry *context,
165 unsigned long value)
166{
167 context->lo &= (((u64)-1) << 4) | 3;
168 context->lo |= (value & 3) << 2;
169}
170
171static inline void context_set_address_root(struct context_entry *context,
172 unsigned long value)
173{
174 context->lo |= value & VTD_PAGE_MASK;
175}
176
177static inline void context_set_address_width(struct context_entry *context,
178 unsigned long value)
179{
180 context->hi |= value & 7;
181}
182
183static inline void context_set_domain_id(struct context_entry *context,
184 unsigned long value)
185{
186 context->hi |= (value & ((1 << 16) - 1)) << 8;
187}
188
189static inline void context_clear_entry(struct context_entry *context)
190{
191 context->lo = 0;
192 context->hi = 0;
193}
7a8fc25e 194
622ba12a
MM		 195/*
196 * 0: readable
197 * 1: writable
198 * 2-6: reserved
199 * 7: super page
9cf06697
SY		 200 * 8-10: available
201 * 11: snoop behavior
622ba12a
MM		 202 * 12-63: Host physcial address
203 */
204struct dma_pte {
205 u64 val;
206};
622ba12a 207
19c239ce
MM		 208static inline void dma_clear_pte(struct dma_pte *pte)
209{
210 pte->val = 0;
211}
212
213static inline void dma_set_pte_readable(struct dma_pte *pte)
214{
215 pte->val |= DMA_PTE_READ;
216}
217
218static inline void dma_set_pte_writable(struct dma_pte *pte)
219{
220 pte->val |= DMA_PTE_WRITE;
221}
222
9cf06697
SY		 223static inline void dma_set_pte_snp(struct dma_pte *pte)
224{
225 pte->val |= DMA_PTE_SNP;
226}
227
19c239ce
MM		 228static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
229{
230 pte->val = (pte->val & ~3) | (prot & 3);
231}
232
233static inline u64 dma_pte_addr(struct dma_pte *pte)
234{
c85994e4
DW		 235#ifdef CONFIG_64BIT
236 return pte->val & VTD_PAGE_MASK;
237#else
238 /* Must have a full atomic 64-bit read */
239 return __cmpxchg64(pte, 0ULL, 0ULL) & VTD_PAGE_MASK;
240#endif
19c239ce
MM		 241}
242
dd4e8319 243static inline void dma_set_pte_pfn(struct dma_pte *pte, unsigned long pfn)
19c239ce 244{
dd4e8319 245 pte->val |= (uint64_t)pfn << VTD_PAGE_SHIFT;
19c239ce
MM		 246}
247
248static inline bool dma_pte_present(struct dma_pte *pte)
249{
250 return (pte->val & 3) != 0;
251}
622ba12a 252
75e6bf96
DW		 253static inline int first_pte_in_page(struct dma_pte *pte)
254{
255 return !((unsigned long)pte & ~VTD_PAGE_MASK);
256}
257
2c2e2c38
FY		 258/*
259 * This domain is a statically identity mapping domain.
260 * 1. This domain creats a static 1:1 mapping to all usable memory.
261 * 2. It maps to each iommu if successful.
262 * 3. Each iommu mapps to this domain if successful.
263 */
19943b0e
DW		 264static struct dmar_domain *si_domain;
265static int hw_pass_through = 1;
2c2e2c38 266
3b5410e7 267/* devices under the same p2p bridge are owned in one domain */
cdc7b837 268#define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
3b5410e7 269
1ce28feb
WH		 270/* domain represents a virtual machine, more than one devices
271 * across iommus may be owned in one domain, e.g. kvm guest.
272 */
273#define DOMAIN_FLAG_VIRTUAL_MACHINE (1 << 1)
274
2c2e2c38
FY		 275/* si_domain contains mulitple devices */
276#define DOMAIN_FLAG_STATIC_IDENTITY (1 << 2)
277
99126f7c
MM		 278struct dmar_domain {
279 int id; /* domain id */
4c923d47 280 int nid; /* node id */
8c11e798 281 unsigned long iommu_bmp; /* bitmap of iommus this domain uses*/
99126f7c
MM		 282
283 struct list_head devices; /* all devices' list */
284 struct iova_domain iovad; /* iova's that belong to this domain */
285
286 struct dma_pte *pgd; /* virtual address */
99126f7c
MM		 287 int gaw; /* max guest address width */
288
289 /* adjusted guest address width, 0 is level 2 30-bit */
290 int agaw;
291
3b5410e7 292 int flags; /* flags to find out type of domain */
8e604097
WH		 293
294 int iommu_coherency;/* indicate coherency of iommu access */
58c610bd 295 int iommu_snooping; /* indicate snooping control feature*/
c7151a8d
WH		 296 int iommu_count; /* reference count of iommu */
297 spinlock_t iommu_lock; /* protect iommu set in domain */
fe40f1e0 298 u64 max_addr; /* maximum mapped address */
99126f7c
MM		 299};
300
a647dacb
MM		 301/* PCI domain-device relationship */
302struct device_domain_info {
303 struct list_head link; /* link to domain siblings */
304 struct list_head global; /* link to global list */
276dbf99
DW		 305 int segment; /* PCI domain */
306 u8 bus; /* PCI bus number */
a647dacb 307 u8 devfn; /* PCI devfn number */
45e829ea 308 struct pci_dev *dev; /* it's NULL for PCIe-to-PCI bridge */
93a23a72 309 struct intel_iommu *iommu; /* IOMMU used by this device */
a647dacb
MM		 310 struct dmar_domain *domain; /* pointer to domain */
311};
312
5e0d2a6f 313static void flush_unmaps_timeout(unsigned long data);
314
315DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0);
316
80b20dd8 317#define HIGH_WATER_MARK 250
318struct deferred_flush_tables {
319 int next;
320 struct iova *iova[HIGH_WATER_MARK];
321 struct dmar_domain *domain[HIGH_WATER_MARK];
322};
323
324static struct deferred_flush_tables *deferred_flush;
325
5e0d2a6f 326/* bitmap for indexing intel_iommus */
5e0d2a6f 327static int g_num_of_iommus;
328
329static DEFINE_SPINLOCK(async_umap_flush_lock);
330static LIST_HEAD(unmaps_to_do);
331
332static int timer_on;
333static long list_size;
5e0d2a6f 334
ba395927
KA		 335static void domain_remove_dev_info(struct dmar_domain *domain);
336
0cd5c3c8
KM		 337#ifdef CONFIG_DMAR_DEFAULT_ON
338int dmar_disabled = 0;
339#else
340int dmar_disabled = 1;
341#endif /*CONFIG_DMAR_DEFAULT_ON*/
342
2d9e667e 343static int dmar_map_gfx = 1;
7d3b03ce 344static int dmar_forcedac;
5e0d2a6f 345static int intel_iommu_strict;
ba395927
KA		 346
347#define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
348static DEFINE_SPINLOCK(device_domain_lock);
349static LIST_HEAD(device_domain_list);
350
a8bcbb0d
JR		 351static struct iommu_ops intel_iommu_ops;
352
ba395927
KA		 353static int __init intel_iommu_setup(char *str)
354{
355 if (!str)
356 return -EINVAL;
357 while (*str) {
0cd5c3c8
KM		 358 if (!strncmp(str, "on", 2)) {
359 dmar_disabled = 0;
360 printk(KERN_INFO "Intel-IOMMU: enabled\n");
361 } else if (!strncmp(str, "off", 3)) {
ba395927 362 dmar_disabled = 1;
0cd5c3c8 363 printk(KERN_INFO "Intel-IOMMU: disabled\n");
ba395927
KA		 364 } else if (!strncmp(str, "igfx_off", 8)) {
365 dmar_map_gfx = 0;
366 printk(KERN_INFO
367 "Intel-IOMMU: disable GFX device mapping\n");
7d3b03ce 368 } else if (!strncmp(str, "forcedac", 8)) {
5e0d2a6f 369 printk(KERN_INFO
7d3b03ce
KA		 370 "Intel-IOMMU: Forcing DAC for PCI devices\n");
371 dmar_forcedac = 1;
5e0d2a6f 372 } else if (!strncmp(str, "strict", 6)) {
373 printk(KERN_INFO
374 "Intel-IOMMU: disable batched IOTLB flush\n");
375 intel_iommu_strict = 1;
ba395927
KA		 376 }
377
378 str += strcspn(str, ",");
379 while (*str == ',')
380 str++;
381 }
382 return 0;
383}
384__setup("intel_iommu=", intel_iommu_setup);
385
386static struct kmem_cache *iommu_domain_cache;
387static struct kmem_cache *iommu_devinfo_cache;
388static struct kmem_cache *iommu_iova_cache;
389
4c923d47 390static inline void *alloc_pgtable_page(int node)
eb3fa7cb 391{
4c923d47
SS		 392 struct page *page;
393 void *vaddr = NULL;
eb3fa7cb 394
4c923d47
SS		 395 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
396 if (page)
397 vaddr = page_address(page);
eb3fa7cb 398 return vaddr;
ba395927
KA		 399}
400
401static inline void free_pgtable_page(void *vaddr)
402{
403 free_page((unsigned long)vaddr);
404}
405
406static inline void *alloc_domain_mem(void)
407{
354bb65e 408 return kmem_cache_alloc(iommu_domain_cache, GFP_ATOMIC);
ba395927
KA		 409}
410
38717946 411static void free_domain_mem(void *vaddr)
ba395927
KA		 412{
413 kmem_cache_free(iommu_domain_cache, vaddr);
414}
415
416static inline void * alloc_devinfo_mem(void)
417{
354bb65e 418 return kmem_cache_alloc(iommu_devinfo_cache, GFP_ATOMIC);
ba395927
KA		 419}
420
421static inline void free_devinfo_mem(void *vaddr)
422{
423 kmem_cache_free(iommu_devinfo_cache, vaddr);
424}
425
426struct iova *alloc_iova_mem(void)
427{
354bb65e 428 return kmem_cache_alloc(iommu_iova_cache, GFP_ATOMIC);
ba395927
KA		 429}
430
431void free_iova_mem(struct iova *iova)
432{
433 kmem_cache_free(iommu_iova_cache, iova);
434}
435
1b573683
WH		 436
437static inline int width_to_agaw(int width);
438
4ed0d3e6 439static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
1b573683
WH		 440{
441 unsigned long sagaw;
442 int agaw = -1;
443
444 sagaw = cap_sagaw(iommu->cap);
4ed0d3e6 445 for (agaw = width_to_agaw(max_gaw);
1b573683
WH		 446 agaw >= 0; agaw--) {
447 if (test_bit(agaw, &sagaw))
448 break;
449 }
450
451 return agaw;
452}
453
4ed0d3e6
FY		 454/*
455 * Calculate max SAGAW for each iommu.
456 */
457int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
458{
459 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
460}
461
462/*
463 * calculate agaw for each iommu.
464 * "SAGAW" may be different across iommus, use a default agaw, and
465 * get a supported less agaw for iommus that don't support the default agaw.
466 */
467int iommu_calculate_agaw(struct intel_iommu *iommu)
468{
469 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
470}
471
2c2e2c38 472/* This functionin only returns single iommu in a domain */
8c11e798
WH		 473static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
474{
475 int iommu_id;
476
2c2e2c38 477 /* si_domain and vm domain should not get here. */
1ce28feb 478 BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
2c2e2c38 479 BUG_ON(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY);
1ce28feb 480
8c11e798
WH		 481 iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
482 if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
483 return NULL;
484
485 return g_iommus[iommu_id];
486}
487
8e604097
WH		 488static void domain_update_iommu_coherency(struct dmar_domain *domain)
489{
490 int i;
491
492 domain->iommu_coherency = 1;
493
a45946ab 494 for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
8e604097
WH		 495 if (!ecap_coherent(g_iommus[i]->ecap)) {
496 domain->iommu_coherency = 0;
497 break;
498 }
8e604097
WH		 499 }
500}
501
58c610bd
SY		 502static void domain_update_iommu_snooping(struct dmar_domain *domain)
503{
504 int i;
505
506 domain->iommu_snooping = 1;
507
a45946ab 508 for_each_set_bit(i, &domain->iommu_bmp, g_num_of_iommus) {
58c610bd
SY		 509 if (!ecap_sc_support(g_iommus[i]->ecap)) {
510 domain->iommu_snooping = 0;
511 break;
512 }
58c610bd
SY		 513 }
514}
515
516/* Some capabilities may be different across iommus */
517static void domain_update_iommu_cap(struct dmar_domain *domain)
518{
519 domain_update_iommu_coherency(domain);
520 domain_update_iommu_snooping(domain);
521}
522
276dbf99 523static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn)
c7151a8d
WH		 524{
525 struct dmar_drhd_unit *drhd = NULL;
526 int i;
527
528 for_each_drhd_unit(drhd) {
529 if (drhd->ignored)
530 continue;
276dbf99
DW		 531 if (segment != drhd->segment)
532 continue;
c7151a8d 533
924b6231 534 for (i = 0; i < drhd->devices_cnt; i++) {
288e4877
DH		 535 if (drhd->devices[i] &&
536 drhd->devices[i]->bus->number == bus &&
c7151a8d
WH		 537 drhd->devices[i]->devfn == devfn)
538 return drhd->iommu;
4958c5dc
DW		 539 if (drhd->devices[i] &&
540 drhd->devices[i]->subordinate &&
924b6231
DW		 541 drhd->devices[i]->subordinate->number <= bus &&
542 drhd->devices[i]->subordinate->subordinate >= bus)
543 return drhd->iommu;
544 }
c7151a8d
WH		 545
546 if (drhd->include_all)
547 return drhd->iommu;
548 }
549
550 return NULL;
551}
552
5331fe6f
WH		 553static void domain_flush_cache(struct dmar_domain *domain,
554 void *addr, int size)
555{
556 if (!domain->iommu_coherency)
557 clflush_cache_range(addr, size);
558}
559
ba395927
KA		 560/* Gets context entry for a given bus and devfn */
561static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
562 u8 bus, u8 devfn)
563{
564 struct root_entry *root;
565 struct context_entry *context;
566 unsigned long phy_addr;
567 unsigned long flags;
568
569 spin_lock_irqsave(&iommu->lock, flags);
570 root = &iommu->root_entry[bus];
571 context = get_context_addr_from_root(root);
572 if (!context) {
4c923d47
SS		 573 context = (struct context_entry *)
574 alloc_pgtable_page(iommu->node);
ba395927
KA		 575 if (!context) {
576 spin_unlock_irqrestore(&iommu->lock, flags);
577 return NULL;
578 }
5b6985ce 579 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
ba395927
KA		 580 phy_addr = virt_to_phys((void *)context);
581 set_root_value(root, phy_addr);
582 set_root_present(root);
583 __iommu_flush_cache(iommu, root, sizeof(*root));
584 }
585 spin_unlock_irqrestore(&iommu->lock, flags);
586 return &context[devfn];
587}
588
589static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
590{
591 struct root_entry *root;
592 struct context_entry *context;
593 int ret;
594 unsigned long flags;
595
596 spin_lock_irqsave(&iommu->lock, flags);
597 root = &iommu->root_entry[bus];
598 context = get_context_addr_from_root(root);
599 if (!context) {
600 ret = 0;
601 goto out;
602 }
c07e7d21 603 ret = context_present(&context[devfn]);
ba395927
KA		 604out:
605 spin_unlock_irqrestore(&iommu->lock, flags);
606 return ret;
607}
608
609static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
610{
611 struct root_entry *root;
612 struct context_entry *context;
613 unsigned long flags;
614
615 spin_lock_irqsave(&iommu->lock, flags);
616 root = &iommu->root_entry[bus];
617 context = get_context_addr_from_root(root);
618 if (context) {
c07e7d21 619 context_clear_entry(&context[devfn]);
ba395927
KA		 620 __iommu_flush_cache(iommu, &context[devfn], \
621 sizeof(*context));
622 }
623 spin_unlock_irqrestore(&iommu->lock, flags);
624}
625
626static void free_context_table(struct intel_iommu *iommu)
627{
628 struct root_entry *root;
629 int i;
630 unsigned long flags;
631 struct context_entry *context;
632
633 spin_lock_irqsave(&iommu->lock, flags);
634 if (!iommu->root_entry) {
635 goto out;
636 }
637 for (i = 0; i < ROOT_ENTRY_NR; i++) {
638 root = &iommu->root_entry[i];
639 context = get_context_addr_from_root(root);
640 if (context)
641 free_pgtable_page(context);
642 }
643 free_pgtable_page(iommu->root_entry);
644 iommu->root_entry = NULL;
645out:
646 spin_unlock_irqrestore(&iommu->lock, flags);
647}
648
649/* page table handling */
650#define LEVEL_STRIDE (9)
651#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
652
653static inline int agaw_to_level(int agaw)
654{
655 return agaw + 2;
656}
657
658static inline int agaw_to_width(int agaw)
659{
660 return 30 + agaw * LEVEL_STRIDE;
661
662}
663
664static inline int width_to_agaw(int width)
665{
666 return (width - 30) / LEVEL_STRIDE;
667}
668
669static inline unsigned int level_to_offset_bits(int level)
670{
6660c63a 671 return (level - 1) * LEVEL_STRIDE;
ba395927
KA		 672}
673
77dfa56c 674static inline int pfn_level_offset(unsigned long pfn, int level)
ba395927 675{
6660c63a 676 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
ba395927
KA		 677}
678
6660c63a 679static inline unsigned long level_mask(int level)
ba395927 680{
6660c63a 681 return -1UL << level_to_offset_bits(level);
ba395927
KA		 682}
683
6660c63a 684static inline unsigned long level_size(int level)
ba395927 685{
6660c63a 686 return 1UL << level_to_offset_bits(level);
ba395927
KA		 687}
688
6660c63a 689static inline unsigned long align_to_level(unsigned long pfn, int level)
ba395927 690{
6660c63a 691 return (pfn + level_size(level) - 1) & level_mask(level);
ba395927
KA		 692}
693
b026fd28
DW		 694static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
695 unsigned long pfn)
ba395927 696{
b026fd28 697 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
ba395927
KA		 698 struct dma_pte *parent, *pte = NULL;
699 int level = agaw_to_level(domain->agaw);
700 int offset;
ba395927
KA		 701
702 BUG_ON(!domain->pgd);
b026fd28 703 BUG_ON(addr_width < BITS_PER_LONG && pfn >> addr_width);
ba395927
KA		 704 parent = domain->pgd;
705
ba395927
KA		 706 while (level > 0) {
707 void *tmp_page;
708
b026fd28 709 offset = pfn_level_offset(pfn, level);
ba395927
KA		 710 pte = &parent[offset];
711 if (level == 1)
712 break;
713
19c239ce 714 if (!dma_pte_present(pte)) {
c85994e4
DW		 715 uint64_t pteval;
716
4c923d47 717 tmp_page = alloc_pgtable_page(domain->nid);
ba395927 718
206a73c1 719 if (!tmp_page)
ba395927 720 return NULL;
206a73c1 721
c85994e4 722 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
64de5af0 723 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
c85994e4
DW		 724 if (cmpxchg64(&pte->val, 0ULL, pteval)) {
725 /* Someone else set it while we were thinking; use theirs. */
726 free_pgtable_page(tmp_page);
727 } else {
728 dma_pte_addr(pte);
729 domain_flush_cache(domain, pte, sizeof(*pte));
730 }
ba395927 731 }
19c239ce 732 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 733 level--;
734 }
735
ba395927
KA		 736 return pte;
737}
738
739/* return address's pte at specific level */
90dcfb5e
DW		 740static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
741 unsigned long pfn,
742 int level)
ba395927
KA		 743{
744 struct dma_pte *parent, *pte = NULL;
745 int total = agaw_to_level(domain->agaw);
746 int offset;
747
748 parent = domain->pgd;
749 while (level <= total) {
90dcfb5e 750 offset = pfn_level_offset(pfn, total);
ba395927
KA		 751 pte = &parent[offset];
752 if (level == total)
753 return pte;
754
19c239ce 755 if (!dma_pte_present(pte))
ba395927 756 break;
19c239ce 757 parent = phys_to_virt(dma_pte_addr(pte));
ba395927
KA		 758 total--;
759 }
760 return NULL;
761}
762
ba395927 763/* clear last level pte, a tlb flush should be followed */
595badf5
DW		 764static void dma_pte_clear_range(struct dmar_domain *domain,
765 unsigned long start_pfn,
766 unsigned long last_pfn)
ba395927 767{
04b18e65 768 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
310a5ab9 769 struct dma_pte *first_pte, *pte;
66eae846 770
04b18e65 771 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
595badf5 772 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 773 BUG_ON(start_pfn > last_pfn);
ba395927 774
04b18e65 775 /* we don't need lock here; nobody else touches the iova range */
59c36286 776 do {
310a5ab9
DW		 777 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1);
778 if (!pte) {
779 start_pfn = align_to_level(start_pfn + 1, 2);
780 continue;
781 }
75e6bf96 782 do {
310a5ab9
DW		 783 dma_clear_pte(pte);
784 start_pfn++;
785 pte++;
75e6bf96
DW		 786 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
787
310a5ab9
DW		 788 domain_flush_cache(domain, first_pte,
789 (void *)pte - (void *)first_pte);
59c36286
DW		 790
791 } while (start_pfn && start_pfn <= last_pfn);
ba395927
KA		 792}
793
794/* free page table pages. last level pte should already be cleared */
795static void dma_pte_free_pagetable(struct dmar_domain *domain,
d794dc9b
DW		 796 unsigned long start_pfn,
797 unsigned long last_pfn)
ba395927 798{
6660c63a 799 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
f3a0a52f 800 struct dma_pte *first_pte, *pte;
ba395927
KA		 801 int total = agaw_to_level(domain->agaw);
802 int level;
6660c63a 803 unsigned long tmp;
ba395927 804
6660c63a
DW		 805 BUG_ON(addr_width < BITS_PER_LONG && start_pfn >> addr_width);
806 BUG_ON(addr_width < BITS_PER_LONG && last_pfn >> addr_width);
59c36286 807 BUG_ON(start_pfn > last_pfn);
ba395927 808
f3a0a52f 809 /* We don't need lock here; nobody else touches the iova range */
ba395927
KA		 810 level = 2;
811 while (level <= total) {
6660c63a
DW		 812 tmp = align_to_level(start_pfn, level);
813
f3a0a52f 814 /* If we can't even clear one PTE at this level, we're done */
6660c63a 815 if (tmp + level_size(level) - 1 > last_pfn)
ba395927
KA		 816 return;
817
59c36286 818 do {
f3a0a52f
DW		 819 first_pte = pte = dma_pfn_level_pte(domain, tmp, level);
820 if (!pte) {
821 tmp = align_to_level(tmp + 1, level + 1);
822 continue;
823 }
75e6bf96 824 do {
6a43e574
DW		 825 if (dma_pte_present(pte)) {
826 free_pgtable_page(phys_to_virt(dma_pte_addr(pte)));
827 dma_clear_pte(pte);
828 }
f3a0a52f
DW		 829 pte++;
830 tmp += level_size(level);
75e6bf96
DW		 831 } while (!first_pte_in_page(pte) &&
832 tmp + level_size(level) - 1 <= last_pfn);
833
f3a0a52f
DW		 834 domain_flush_cache(domain, first_pte,
835 (void *)pte - (void *)first_pte);
836
59c36286 837 } while (tmp && tmp + level_size(level) - 1 <= last_pfn);
ba395927
KA		 838 level++;
839 }
840 /* free pgd */
d794dc9b 841 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
ba395927
KA		 842 free_pgtable_page(domain->pgd);
843 domain->pgd = NULL;
844 }
845}
846
847/* iommu handling */
848static int iommu_alloc_root_entry(struct intel_iommu *iommu)
849{
850 struct root_entry *root;
851 unsigned long flags;
852
4c923d47 853 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
ba395927
KA		 854 if (!root)
855 return -ENOMEM;
856
5b6985ce 857 __iommu_flush_cache(iommu, root, ROOT_SIZE);
ba395927
KA		 858
859 spin_lock_irqsave(&iommu->lock, flags);
860 iommu->root_entry = root;
861 spin_unlock_irqrestore(&iommu->lock, flags);
862
863 return 0;
864}
865
ba395927
KA		 866static void iommu_set_root_entry(struct intel_iommu *iommu)
867{
868 void *addr;
c416daa9 869 u32 sts;
ba395927
KA		 870 unsigned long flag;
871
872 addr = iommu->root_entry;
873
874 spin_lock_irqsave(&iommu->register_lock, flag);
875 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
876
c416daa9 877 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 878
879 /* Make sure hardware complete it */
880 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 881 readl, (sts & DMA_GSTS_RTPS), sts);
ba395927
KA		 882
883 spin_unlock_irqrestore(&iommu->register_lock, flag);
884}
885
886static void iommu_flush_write_buffer(struct intel_iommu *iommu)
887{
888 u32 val;
889 unsigned long flag;
890
9af88143 891 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
ba395927 892 return;
ba395927
KA		 893
894 spin_lock_irqsave(&iommu->register_lock, flag);
462b60f6 895 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 896
897 /* Make sure hardware complete it */
898 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 899 readl, (!(val & DMA_GSTS_WBFS)), val);
ba395927
KA		 900
901 spin_unlock_irqrestore(&iommu->register_lock, flag);
902}
903
904/* return value determine if we need a write buffer flush */
4c25a2c1
DW		 905static void __iommu_flush_context(struct intel_iommu *iommu,
906 u16 did, u16 source_id, u8 function_mask,
907 u64 type)
ba395927
KA		 908{
909 u64 val = 0;
910 unsigned long flag;
911
ba395927
KA		 912 switch (type) {
913 case DMA_CCMD_GLOBAL_INVL:
914 val = DMA_CCMD_GLOBAL_INVL;
915 break;
916 case DMA_CCMD_DOMAIN_INVL:
917 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
918 break;
919 case DMA_CCMD_DEVICE_INVL:
920 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
921 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
922 break;
923 default:
924 BUG();
925 }
926 val |= DMA_CCMD_ICC;
927
928 spin_lock_irqsave(&iommu->register_lock, flag);
929 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
930
931 /* Make sure hardware complete it */
932 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
933 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
934
935 spin_unlock_irqrestore(&iommu->register_lock, flag);
ba395927
KA		 936}
937
ba395927 938/* return value determine if we need a write buffer flush */
1f0ef2aa
DW		 939static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
940 u64 addr, unsigned int size_order, u64 type)
ba395927
KA		 941{
942 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
943 u64 val = 0, val_iva = 0;
944 unsigned long flag;
945
ba395927
KA		 946 switch (type) {
947 case DMA_TLB_GLOBAL_FLUSH:
948 /* global flush doesn't need set IVA_REG */
949 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
950 break;
951 case DMA_TLB_DSI_FLUSH:
952 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
953 break;
954 case DMA_TLB_PSI_FLUSH:
955 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
956 /* Note: always flush non-leaf currently */
957 val_iva = size_order | addr;
958 break;
959 default:
960 BUG();
961 }
962 /* Note: set drain read/write */
963#if 0
964 /*
965 * This is probably to be super secure.. Looks like we can
966 * ignore it without any impact.
967 */
968 if (cap_read_drain(iommu->cap))
969 val |= DMA_TLB_READ_DRAIN;
970#endif
971 if (cap_write_drain(iommu->cap))
972 val |= DMA_TLB_WRITE_DRAIN;
973
974 spin_lock_irqsave(&iommu->register_lock, flag);
975 /* Note: Only uses first TLB reg currently */
976 if (val_iva)
977 dmar_writeq(iommu->reg + tlb_offset, val_iva);
978 dmar_writeq(iommu->reg + tlb_offset + 8, val);
979
980 /* Make sure hardware complete it */
981 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
982 dmar_readq, (!(val & DMA_TLB_IVT)), val);
983
984 spin_unlock_irqrestore(&iommu->register_lock, flag);
985
986 /* check IOTLB invalidation granularity */
987 if (DMA_TLB_IAIG(val) == 0)
988 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
989 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
990 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
5b6985ce
FY		 991 (unsigned long long)DMA_TLB_IIRG(type),
992 (unsigned long long)DMA_TLB_IAIG(val));
ba395927
KA		 993}
994
93a23a72
YZ		 995static struct device_domain_info *iommu_support_dev_iotlb(
996 struct dmar_domain *domain, int segment, u8 bus, u8 devfn)
997{
998 int found = 0;
999 unsigned long flags;
1000 struct device_domain_info *info;
1001 struct intel_iommu *iommu = device_to_iommu(segment, bus, devfn);
1002
1003 if (!ecap_dev_iotlb_support(iommu->ecap))
1004 return NULL;
1005
1006 if (!iommu->qi)
1007 return NULL;
1008
1009 spin_lock_irqsave(&device_domain_lock, flags);
1010 list_for_each_entry(info, &domain->devices, link)
1011 if (info->bus == bus && info->devfn == devfn) {
1012 found = 1;
1013 break;
1014 }
1015 spin_unlock_irqrestore(&device_domain_lock, flags);
1016
1017 if (!found || !info->dev)
1018 return NULL;
1019
1020 if (!pci_find_ext_capability(info->dev, PCI_EXT_CAP_ID_ATS))
1021 return NULL;
1022
1023 if (!dmar_find_matched_atsr_unit(info->dev))
1024 return NULL;
1025
1026 info->iommu = iommu;
1027
1028 return info;
1029}
1030
1031static void iommu_enable_dev_iotlb(struct device_domain_info *info)
ba395927 1032{
93a23a72
YZ		 1033 if (!info)
1034 return;
1035
1036 pci_enable_ats(info->dev, VTD_PAGE_SHIFT);
1037}
1038
1039static void iommu_disable_dev_iotlb(struct device_domain_info *info)
1040{
1041 if (!info->dev || !pci_ats_enabled(info->dev))
1042 return;
1043
1044 pci_disable_ats(info->dev);
1045}
1046
1047static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1048 u64 addr, unsigned mask)
1049{
1050 u16 sid, qdep;
1051 unsigned long flags;
1052 struct device_domain_info *info;
1053
1054 spin_lock_irqsave(&device_domain_lock, flags);
1055 list_for_each_entry(info, &domain->devices, link) {
1056 if (!info->dev || !pci_ats_enabled(info->dev))
1057 continue;
1058
1059 sid = info->bus << 8 | info->devfn;
1060 qdep = pci_ats_queue_depth(info->dev);
1061 qi_flush_dev_iotlb(info->iommu, sid, qdep, addr, mask);
1062 }
1063 spin_unlock_irqrestore(&device_domain_lock, flags);
1064}
1065
1f0ef2aa 1066static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
82653633 1067 unsigned long pfn, unsigned int pages, int map)
ba395927 1068{
9dd2fe89 1069 unsigned int mask = ilog2(__roundup_pow_of_two(pages));
03d6a246 1070 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
ba395927 1071
ba395927
KA		 1072 BUG_ON(pages == 0);
1073
ba395927 1074 /*
9dd2fe89
YZ		 1075 * Fallback to domain selective flush if no PSI support or the size is
1076 * too big.
ba395927
KA		 1077 * PSI requires page size to be 2 ^ x, and the base address is naturally
1078 * aligned to the size
1079 */
9dd2fe89
YZ		 1080 if (!cap_pgsel_inv(iommu->cap) || mask > cap_max_amask_val(iommu->cap))
1081 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1f0ef2aa 1082 DMA_TLB_DSI_FLUSH);
9dd2fe89
YZ		 1083 else
1084 iommu->flush.flush_iotlb(iommu, did, addr, mask,
1085 DMA_TLB_PSI_FLUSH);
bf92df30
YZ		 1086
1087 /*
82653633
NA		 1088 * In caching mode, changes of pages from non-present to present require
1089 * flush. However, device IOTLB doesn't need to be flushed in this case.
bf92df30 1090 */
82653633 1091 if (!cap_caching_mode(iommu->cap) || !map)
93a23a72 1092 iommu_flush_dev_iotlb(iommu->domains[did], addr, mask);
ba395927
KA		 1093}
1094
f8bab735 1095static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1096{
1097 u32 pmen;
1098 unsigned long flags;
1099
1100 spin_lock_irqsave(&iommu->register_lock, flags);
1101 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1102 pmen &= ~DMA_PMEN_EPM;
1103 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1104
1105 /* wait for the protected region status bit to clear */
1106 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1107 readl, !(pmen & DMA_PMEN_PRS), pmen);
1108
1109 spin_unlock_irqrestore(&iommu->register_lock, flags);
1110}
1111
ba395927
KA		 1112static int iommu_enable_translation(struct intel_iommu *iommu)
1113{
1114 u32 sts;
1115 unsigned long flags;
1116
1117 spin_lock_irqsave(&iommu->register_lock, flags);
c416daa9
DW		 1118 iommu->gcmd |= DMA_GCMD_TE;
1119 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
ba395927
KA		 1120
1121 /* Make sure hardware complete it */
1122 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1123 readl, (sts & DMA_GSTS_TES), sts);
ba395927 1124
ba395927
KA		 1125 spin_unlock_irqrestore(&iommu->register_lock, flags);
1126 return 0;
1127}
1128
1129static int iommu_disable_translation(struct intel_iommu *iommu)
1130{
1131 u32 sts;
1132 unsigned long flag;
1133
1134 spin_lock_irqsave(&iommu->register_lock, flag);
1135 iommu->gcmd &= ~DMA_GCMD_TE;
1136 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1137
1138 /* Make sure hardware complete it */
1139 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
c416daa9 1140 readl, (!(sts & DMA_GSTS_TES)), sts);
ba395927
KA		 1141
1142 spin_unlock_irqrestore(&iommu->register_lock, flag);
1143 return 0;
1144}
1145
3460a6d9 1146
ba395927
KA		 1147static int iommu_init_domains(struct intel_iommu *iommu)
1148{
1149 unsigned long ndomains;
1150 unsigned long nlongs;
1151
1152 ndomains = cap_ndoms(iommu->cap);
680a7524
YL		 1153 pr_debug("IOMMU %d: Number of Domains supportd <%ld>\n", iommu->seq_id,
1154 ndomains);
ba395927
KA		 1155 nlongs = BITS_TO_LONGS(ndomains);
1156
94a91b50
DD		 1157 spin_lock_init(&iommu->lock);
1158
ba395927
KA		 1159 /* TBD: there might be 64K domains,
1160 * consider other allocation for future chip
1161 */
1162 iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1163 if (!iommu->domain_ids) {
1164 printk(KERN_ERR "Allocating domain id array failed\n");
1165 return -ENOMEM;
1166 }
1167 iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1168 GFP_KERNEL);
1169 if (!iommu->domains) {
1170 printk(KERN_ERR "Allocating domain array failed\n");
ba395927
KA		 1171 return -ENOMEM;
1172 }
1173
1174 /*
1175 * if Caching mode is set, then invalid translations are tagged
1176 * with domainid 0. Hence we need to pre-allocate it.
1177 */
1178 if (cap_caching_mode(iommu->cap))
1179 set_bit(0, iommu->domain_ids);
1180 return 0;
1181}
ba395927 1182
ba395927
KA		 1183
1184static void domain_exit(struct dmar_domain *domain);
5e98c4b1 1185static void vm_domain_exit(struct dmar_domain *domain);
e61d98d8
SS		 1186
1187void free_dmar_iommu(struct intel_iommu *iommu)
ba395927
KA		 1188{
1189 struct dmar_domain *domain;
1190 int i;
c7151a8d 1191 unsigned long flags;
ba395927 1192
94a91b50 1193 if ((iommu->domains) && (iommu->domain_ids)) {
a45946ab 1194 for_each_set_bit(i, iommu->domain_ids, cap_ndoms(iommu->cap)) {
94a91b50
DD		 1195 domain = iommu->domains[i];
1196 clear_bit(i, iommu->domain_ids);
1197
1198 spin_lock_irqsave(&domain->iommu_lock, flags);
1199 if (--domain->iommu_count == 0) {
1200 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1201 vm_domain_exit(domain);
1202 else
1203 domain_exit(domain);
1204 }
1205 spin_unlock_irqrestore(&domain->iommu_lock, flags);
5e98c4b1 1206 }
ba395927
KA		 1207 }
1208
1209 if (iommu->gcmd & DMA_GCMD_TE)
1210 iommu_disable_translation(iommu);
1211
1212 if (iommu->irq) {
1213 set_irq_data(iommu->irq, NULL);
1214 /* This will mask the irq */
1215 free_irq(iommu->irq, iommu);
1216 destroy_irq(iommu->irq);
1217 }
1218
1219 kfree(iommu->domains);
1220 kfree(iommu->domain_ids);
1221
d9630fe9
WH		 1222 g_iommus[iommu->seq_id] = NULL;
1223
1224 /* if all iommus are freed, free g_iommus */
1225 for (i = 0; i < g_num_of_iommus; i++) {
1226 if (g_iommus[i])
1227 break;
1228 }
1229
1230 if (i == g_num_of_iommus)
1231 kfree(g_iommus);
1232
ba395927
KA		 1233 /* free context mapping */
1234 free_context_table(iommu);
ba395927
KA		 1235}
1236
2c2e2c38 1237static struct dmar_domain *alloc_domain(void)
ba395927 1238{
ba395927 1239 struct dmar_domain *domain;
ba395927
KA		 1240
1241 domain = alloc_domain_mem();
1242 if (!domain)
1243 return NULL;
1244
4c923d47 1245 domain->nid = -1;
2c2e2c38
FY		 1246 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1247 domain->flags = 0;
1248
1249 return domain;
1250}
1251
1252static int iommu_attach_domain(struct dmar_domain *domain,
1253 struct intel_iommu *iommu)
1254{
1255 int num;
1256 unsigned long ndomains;
1257 unsigned long flags;
1258
ba395927
KA		 1259 ndomains = cap_ndoms(iommu->cap);
1260
1261 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38 1262
ba395927
KA		 1263 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1264 if (num >= ndomains) {
1265 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927 1266 printk(KERN_ERR "IOMMU: no free domain ids\n");
2c2e2c38 1267 return -ENOMEM;
ba395927
KA		 1268 }
1269
ba395927 1270 domain->id = num;
2c2e2c38 1271 set_bit(num, iommu->domain_ids);
8c11e798 1272 set_bit(iommu->seq_id, &domain->iommu_bmp);
ba395927
KA		 1273 iommu->domains[num] = domain;
1274 spin_unlock_irqrestore(&iommu->lock, flags);
1275
2c2e2c38 1276 return 0;
ba395927
KA		 1277}
1278
2c2e2c38
FY		 1279static void iommu_detach_domain(struct dmar_domain *domain,
1280 struct intel_iommu *iommu)
ba395927
KA		 1281{
1282 unsigned long flags;
2c2e2c38
FY		 1283 int num, ndomains;
1284 int found = 0;
ba395927 1285
8c11e798 1286 spin_lock_irqsave(&iommu->lock, flags);
2c2e2c38 1287 ndomains = cap_ndoms(iommu->cap);
a45946ab 1288 for_each_set_bit(num, iommu->domain_ids, ndomains) {
2c2e2c38
FY		 1289 if (iommu->domains[num] == domain) {
1290 found = 1;
1291 break;
1292 }
2c2e2c38
FY		 1293 }
1294
1295 if (found) {
1296 clear_bit(num, iommu->domain_ids);
1297 clear_bit(iommu->seq_id, &domain->iommu_bmp);
1298 iommu->domains[num] = NULL;
1299 }
8c11e798 1300 spin_unlock_irqrestore(&iommu->lock, flags);
ba395927
KA		 1301}
1302
1303static struct iova_domain reserved_iova_list;
8a443df4 1304static struct lock_class_key reserved_rbtree_key;
ba395927
KA		 1305
1306static void dmar_init_reserved_ranges(void)
1307{
1308 struct pci_dev *pdev = NULL;
1309 struct iova *iova;
1310 int i;
ba395927 1311
f661197e 1312 init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
ba395927 1313
8a443df4
MG		 1314 lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1315 &reserved_rbtree_key);
1316
ba395927
KA		 1317 /* IOAPIC ranges shouldn't be accessed by DMA */
1318 iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1319 IOVA_PFN(IOAPIC_RANGE_END));
1320 if (!iova)
1321 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1322
1323 /* Reserve all PCI MMIO to avoid peer-to-peer access */
1324 for_each_pci_dev(pdev) {
1325 struct resource *r;
1326
1327 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1328 r = &pdev->resource[i];
1329 if (!r->flags || !(r->flags & IORESOURCE_MEM))
1330 continue;
1a4a4551
DW		 1331 iova = reserve_iova(&reserved_iova_list,
1332 IOVA_PFN(r->start),
1333 IOVA_PFN(r->end));
ba395927
KA		 1334 if (!iova)
1335 printk(KERN_ERR "Reserve iova failed\n");
1336 }
1337 }
1338
1339}
1340
1341static void domain_reserve_special_ranges(struct dmar_domain *domain)
1342{
1343 copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1344}
1345
1346static inline int guestwidth_to_adjustwidth(int gaw)
1347{
1348 int agaw;
1349 int r = (gaw - 12) % 9;
1350
1351 if (r == 0)
1352 agaw = gaw;
1353 else
1354 agaw = gaw + 9 - r;
1355 if (agaw > 64)
1356 agaw = 64;
1357 return agaw;
1358}
1359
1360static int domain_init(struct dmar_domain *domain, int guest_width)
1361{
1362 struct intel_iommu *iommu;
1363 int adjust_width, agaw;
1364 unsigned long sagaw;
1365
f661197e 1366 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
c7151a8d 1367 spin_lock_init(&domain->iommu_lock);
ba395927
KA		 1368
1369 domain_reserve_special_ranges(domain);
1370
1371 /* calculate AGAW */
8c11e798 1372 iommu = domain_get_iommu(domain);
ba395927
KA		 1373 if (guest_width > cap_mgaw(iommu->cap))
1374 guest_width = cap_mgaw(iommu->cap);
1375 domain->gaw = guest_width;
1376 adjust_width = guestwidth_to_adjustwidth(guest_width);
1377 agaw = width_to_agaw(adjust_width);
1378 sagaw = cap_sagaw(iommu->cap);
1379 if (!test_bit(agaw, &sagaw)) {
1380 /* hardware doesn't support it, choose a bigger one */
1381 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1382 agaw = find_next_bit(&sagaw, 5, agaw);
1383 if (agaw >= 5)
1384 return -ENODEV;
1385 }
1386 domain->agaw = agaw;
1387 INIT_LIST_HEAD(&domain->devices);
1388
8e604097
WH		 1389 if (ecap_coherent(iommu->ecap))
1390 domain->iommu_coherency = 1;
1391 else
1392 domain->iommu_coherency = 0;
1393
58c610bd
SY		 1394 if (ecap_sc_support(iommu->ecap))
1395 domain->iommu_snooping = 1;
1396 else
1397 domain->iommu_snooping = 0;
1398
c7151a8d 1399 domain->iommu_count = 1;
4c923d47 1400 domain->nid = iommu->node;
c7151a8d 1401
ba395927 1402 /* always allocate the top pgd */
4c923d47 1403 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
ba395927
KA		 1404 if (!domain->pgd)
1405 return -ENOMEM;
5b6985ce 1406 __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
ba395927
KA		 1407 return 0;
1408}
1409
1410static void domain_exit(struct dmar_domain *domain)
1411{
2c2e2c38
FY		 1412 struct dmar_drhd_unit *drhd;
1413 struct intel_iommu *iommu;
ba395927
KA		 1414
1415 /* Domain 0 is reserved, so dont process it */
1416 if (!domain)
1417 return;
1418
1419 domain_remove_dev_info(domain);
1420 /* destroy iovas */
1421 put_iova_domain(&domain->iovad);
ba395927
KA		 1422
1423 /* clear ptes */
595badf5 1424 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927
KA		 1425
1426 /* free page tables */
d794dc9b 1427 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
ba395927 1428
2c2e2c38
FY		 1429 for_each_active_iommu(iommu, drhd)
1430 if (test_bit(iommu->seq_id, &domain->iommu_bmp))
1431 iommu_detach_domain(domain, iommu);
1432
ba395927
KA		 1433 free_domain_mem(domain);
1434}
1435
4ed0d3e6
FY		 1436static int domain_context_mapping_one(struct dmar_domain *domain, int segment,
1437 u8 bus, u8 devfn, int translation)
ba395927
KA		 1438{
1439 struct context_entry *context;
ba395927 1440 unsigned long flags;
5331fe6f 1441 struct intel_iommu *iommu;
ea6606b0
WH		 1442 struct dma_pte *pgd;
1443 unsigned long num;
1444 unsigned long ndomains;
1445 int id;
1446 int agaw;
93a23a72 1447 struct device_domain_info *info = NULL;
ba395927
KA		 1448
1449 pr_debug("Set context mapping for %02x:%02x.%d\n",
1450 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
4ed0d3e6 1451
ba395927 1452 BUG_ON(!domain->pgd);
4ed0d3e6
FY		 1453 BUG_ON(translation != CONTEXT_TT_PASS_THROUGH &&
1454 translation != CONTEXT_TT_MULTI_LEVEL);
5331fe6f 1455
276dbf99 1456 iommu = device_to_iommu(segment, bus, devfn);
5331fe6f
WH		 1457 if (!iommu)
1458 return -ENODEV;
1459
ba395927
KA		 1460 context = device_to_context_entry(iommu, bus, devfn);
1461 if (!context)
1462 return -ENOMEM;
1463 spin_lock_irqsave(&iommu->lock, flags);
c07e7d21 1464 if (context_present(context)) {
ba395927
KA		 1465 spin_unlock_irqrestore(&iommu->lock, flags);
1466 return 0;
1467 }
1468
ea6606b0
WH		 1469 id = domain->id;
1470 pgd = domain->pgd;
1471
2c2e2c38
FY		 1472 if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
1473 domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) {
ea6606b0
WH		 1474 int found = 0;
1475
1476 /* find an available domain id for this device in iommu */
1477 ndomains = cap_ndoms(iommu->cap);
a45946ab 1478 for_each_set_bit(num, iommu->domain_ids, ndomains) {
ea6606b0
WH		 1479 if (iommu->domains[num] == domain) {
1480 id = num;
1481 found = 1;
1482 break;
1483 }
ea6606b0
WH		 1484 }
1485
1486 if (found == 0) {
1487 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1488 if (num >= ndomains) {
1489 spin_unlock_irqrestore(&iommu->lock, flags);
1490 printk(KERN_ERR "IOMMU: no free domain ids\n");
1491 return -EFAULT;
1492 }
1493
1494 set_bit(num, iommu->domain_ids);
1495 iommu->domains[num] = domain;
1496 id = num;
1497 }
1498
1499 /* Skip top levels of page tables for
1500 * iommu which has less agaw than default.
1672af11 1501 * Unnecessary for PT mode.
ea6606b0 1502 */
1672af11
CW		 1503 if (translation != CONTEXT_TT_PASS_THROUGH) {
1504 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1505 pgd = phys_to_virt(dma_pte_addr(pgd));
1506 if (!dma_pte_present(pgd)) {
1507 spin_unlock_irqrestore(&iommu->lock, flags);
1508 return -ENOMEM;
1509 }
ea6606b0
WH		 1510 }
1511 }
1512 }
1513
1514 context_set_domain_id(context, id);
4ed0d3e6 1515
93a23a72
YZ		 1516 if (translation != CONTEXT_TT_PASS_THROUGH) {
1517 info = iommu_support_dev_iotlb(domain, segment, bus, devfn);
1518 translation = info ? CONTEXT_TT_DEV_IOTLB :
1519 CONTEXT_TT_MULTI_LEVEL;
1520 }
4ed0d3e6
FY		 1521 /*
1522 * In pass through mode, AW must be programmed to indicate the largest
1523 * AGAW value supported by hardware. And ASR is ignored by hardware.
1524 */
93a23a72 1525 if (unlikely(translation == CONTEXT_TT_PASS_THROUGH))
4ed0d3e6 1526 context_set_address_width(context, iommu->msagaw);
93a23a72
YZ		 1527 else {
1528 context_set_address_root(context, virt_to_phys(pgd));
1529 context_set_address_width(context, iommu->agaw);
1530 }
4ed0d3e6
FY		 1531
1532 context_set_translation_type(context, translation);
c07e7d21
MM		 1533 context_set_fault_enable(context);
1534 context_set_present(context);
5331fe6f 1535 domain_flush_cache(domain, context, sizeof(*context));
ba395927 1536
4c25a2c1
DW		 1537 /*
1538 * It's a non-present to present mapping. If hardware doesn't cache
1539 * non-present entry we only need to flush the write-buffer. If the
1540 * _does_ cache non-present entries, then it does so in the special
1541 * domain #0, which we have to flush:
1542 */
1543 if (cap_caching_mode(iommu->cap)) {
1544 iommu->flush.flush_context(iommu, 0,
1545 (((u16)bus) << 8) | devfn,
1546 DMA_CCMD_MASK_NOBIT,
1547 DMA_CCMD_DEVICE_INVL);
82653633 1548 iommu->flush.flush_iotlb(iommu, domain->id, 0, 0, DMA_TLB_DSI_FLUSH);
4c25a2c1 1549 } else {
ba395927 1550 iommu_flush_write_buffer(iommu);
4c25a2c1 1551 }
93a23a72 1552 iommu_enable_dev_iotlb(info);
ba395927 1553 spin_unlock_irqrestore(&iommu->lock, flags);
c7151a8d
WH		 1554
1555 spin_lock_irqsave(&domain->iommu_lock, flags);
1556 if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1557 domain->iommu_count++;
4c923d47
SS		 1558 if (domain->iommu_count == 1)
1559 domain->nid = iommu->node;
58c610bd 1560 domain_update_iommu_cap(domain);
c7151a8d
WH		 1561 }
1562 spin_unlock_irqrestore(&domain->iommu_lock, flags);
ba395927
KA		 1563 return 0;
1564}
1565
1566static int
4ed0d3e6
FY		 1567domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev,
1568 int translation)
ba395927
KA		 1569{
1570 int ret;
1571 struct pci_dev *tmp, *parent;
1572
276dbf99 1573 ret = domain_context_mapping_one(domain, pci_domain_nr(pdev->bus),
4ed0d3e6
FY		 1574 pdev->bus->number, pdev->devfn,
1575 translation);
ba395927
KA		 1576 if (ret)
1577 return ret;
1578
1579 /* dependent device mapping */
1580 tmp = pci_find_upstream_pcie_bridge(pdev);
1581 if (!tmp)
1582 return 0;
1583 /* Secondary interface's bus number and devfn 0 */
1584 parent = pdev->bus->self;
1585 while (parent != tmp) {
276dbf99
DW		 1586 ret = domain_context_mapping_one(domain,
1587 pci_domain_nr(parent->bus),
1588 parent->bus->number,
4ed0d3e6 1589 parent->devfn, translation);
ba395927
KA		 1590 if (ret)
1591 return ret;
1592 parent = parent->bus->self;
1593 }
45e829ea 1594 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
ba395927 1595 return domain_context_mapping_one(domain,
276dbf99 1596 pci_domain_nr(tmp->subordinate),
4ed0d3e6
FY		 1597 tmp->subordinate->number, 0,
1598 translation);
ba395927
KA		 1599 else /* this is a legacy PCI bridge */
1600 return domain_context_mapping_one(domain,
276dbf99
DW		 1601 pci_domain_nr(tmp->bus),
1602 tmp->bus->number,
4ed0d3e6
FY		 1603 tmp->devfn,
1604 translation);
ba395927
KA		 1605}
1606
5331fe6f 1607static int domain_context_mapped(struct pci_dev *pdev)
ba395927
KA		 1608{
1609 int ret;
1610 struct pci_dev *tmp, *parent;
5331fe6f
WH		 1611 struct intel_iommu *iommu;
1612
276dbf99
DW		 1613 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
1614 pdev->devfn);
5331fe6f
WH		 1615 if (!iommu)
1616 return -ENODEV;
ba395927 1617
276dbf99 1618 ret = device_context_mapped(iommu, pdev->bus->number, pdev->devfn);
ba395927
KA		 1619 if (!ret)
1620 return ret;
1621 /* dependent device mapping */
1622 tmp = pci_find_upstream_pcie_bridge(pdev);
1623 if (!tmp)
1624 return ret;
1625 /* Secondary interface's bus number and devfn 0 */
1626 parent = pdev->bus->self;
1627 while (parent != tmp) {
8c11e798 1628 ret = device_context_mapped(iommu, parent->bus->number,
276dbf99 1629 parent->devfn);
ba395927
KA		 1630 if (!ret)
1631 return ret;
1632 parent = parent->bus->self;
1633 }
5f4d91a1 1634 if (pci_is_pcie(tmp))
276dbf99
DW		 1635 return device_context_mapped(iommu, tmp->subordinate->number,
1636 0);
ba395927 1637 else
276dbf99
DW		 1638 return device_context_mapped(iommu, tmp->bus->number,
1639 tmp->devfn);
ba395927
KA		 1640}
1641
f532959b
FY		 1642/* Returns a number of VTD pages, but aligned to MM page size */
1643static inline unsigned long aligned_nrpages(unsigned long host_addr,
1644 size_t size)
1645{
1646 host_addr &= ~PAGE_MASK;
1647 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
1648}
1649
9051aa02
DW		 1650static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1651 struct scatterlist *sg, unsigned long phys_pfn,
1652 unsigned long nr_pages, int prot)
e1605495
DW		 1653{
1654 struct dma_pte *first_pte = NULL, *pte = NULL;
9051aa02 1655 phys_addr_t uninitialized_var(pteval);
e1605495 1656 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
9051aa02 1657 unsigned long sg_res;
e1605495
DW		 1658
1659 BUG_ON(addr_width < BITS_PER_LONG && (iov_pfn + nr_pages - 1) >> addr_width);
1660
1661 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1662 return -EINVAL;
1663
1664 prot &= DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP;
1665
9051aa02
DW		 1666 if (sg)
1667 sg_res = 0;
1668 else {
1669 sg_res = nr_pages + 1;
1670 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | prot;
1671 }
1672
e1605495 1673 while (nr_pages--) {
c85994e4
DW		 1674 uint64_t tmp;
1675
e1605495 1676 if (!sg_res) {
f532959b 1677 sg_res = aligned_nrpages(sg->offset, sg->length);
e1605495
DW		 1678 sg->dma_address = ((dma_addr_t)iov_pfn << VTD_PAGE_SHIFT) + sg->offset;
1679 sg->dma_length = sg->length;
1680 pteval = page_to_phys(sg_page(sg)) | prot;
1681 }
1682 if (!pte) {
1683 first_pte = pte = pfn_to_dma_pte(domain, iov_pfn);
1684 if (!pte)
1685 return -ENOMEM;
1686 }
1687 /* We don't need lock here, nobody else
1688 * touches the iova range
1689 */
7766a3fb 1690 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
c85994e4 1691 if (tmp) {
1bf20f0d 1692 static int dumps = 5;
c85994e4
DW		 1693 printk(KERN_CRIT "ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1694 iov_pfn, tmp, (unsigned long long)pteval);
1bf20f0d
DW		 1695 if (dumps) {
1696 dumps--;
1697 debug_dma_dump_mappings(NULL);
1698 }
1699 WARN_ON(1);
1700 }
e1605495 1701 pte++;
75e6bf96 1702 if (!nr_pages || first_pte_in_page(pte)) {
e1605495
DW		 1703 domain_flush_cache(domain, first_pte,
1704 (void *)pte - (void *)first_pte);
1705 pte = NULL;
1706 }
1707 iov_pfn++;
1708 pteval += VTD_PAGE_SIZE;
1709 sg_res--;
1710 if (!sg_res)
1711 sg = sg_next(sg);
1712 }
1713 return 0;
1714}
1715
9051aa02
DW		 1716static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1717 struct scatterlist *sg, unsigned long nr_pages,
1718 int prot)
ba395927 1719{
9051aa02
DW		 1720 return __domain_mapping(domain, iov_pfn, sg, 0, nr_pages, prot);
1721}
6f6a00e4 1722
9051aa02
DW		 1723static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1724 unsigned long phys_pfn, unsigned long nr_pages,
1725 int prot)
1726{
1727 return __domain_mapping(domain, iov_pfn, NULL, phys_pfn, nr_pages, prot);
ba395927
KA		 1728}
1729
c7151a8d 1730static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
ba395927 1731{
c7151a8d
WH		 1732 if (!iommu)
1733 return;
8c11e798
WH		 1734
1735 clear_context_table(iommu, bus, devfn);
1736 iommu->flush.flush_context(iommu, 0, 0, 0,
4c25a2c1 1737 DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 1738 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
ba395927
KA		 1739}
1740
1741static void domain_remove_dev_info(struct dmar_domain *domain)
1742{
1743 struct device_domain_info *info;
1744 unsigned long flags;
c7151a8d 1745 struct intel_iommu *iommu;
ba395927
KA		 1746
1747 spin_lock_irqsave(&device_domain_lock, flags);
1748 while (!list_empty(&domain->devices)) {
1749 info = list_entry(domain->devices.next,
1750 struct device_domain_info, link);
1751 list_del(&info->link);
1752 list_del(&info->global);
1753 if (info->dev)
358dd8ac 1754 info->dev->dev.archdata.iommu = NULL;
ba395927
KA		 1755 spin_unlock_irqrestore(&device_domain_lock, flags);
1756
93a23a72 1757 iommu_disable_dev_iotlb(info);
276dbf99 1758 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 1759 iommu_detach_dev(iommu, info->bus, info->devfn);
ba395927
KA		 1760 free_devinfo_mem(info);
1761
1762 spin_lock_irqsave(&device_domain_lock, flags);
1763 }
1764 spin_unlock_irqrestore(&device_domain_lock, flags);
1765}
1766
1767/*
1768 * find_domain
358dd8ac 1769 * Note: we use struct pci_dev->dev.archdata.iommu stores the info
ba395927 1770 */
38717946 1771static struct dmar_domain *
ba395927
KA		 1772find_domain(struct pci_dev *pdev)
1773{
1774 struct device_domain_info *info;
1775
1776 /* No lock here, assumes no domain exit in normal case */
358dd8ac 1777 info = pdev->dev.archdata.iommu;
ba395927
KA		 1778 if (info)
1779 return info->domain;
1780 return NULL;
1781}
1782
ba395927
KA		 1783/* domain is initialized */
1784static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1785{
1786 struct dmar_domain *domain, *found = NULL;
1787 struct intel_iommu *iommu;
1788 struct dmar_drhd_unit *drhd;
1789 struct device_domain_info *info, *tmp;
1790 struct pci_dev *dev_tmp;
1791 unsigned long flags;
1792 int bus = 0, devfn = 0;
276dbf99 1793 int segment;
2c2e2c38 1794 int ret;
ba395927
KA		 1795
1796 domain = find_domain(pdev);
1797 if (domain)
1798 return domain;
1799
276dbf99
DW		 1800 segment = pci_domain_nr(pdev->bus);
1801
ba395927
KA		 1802 dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1803 if (dev_tmp) {
5f4d91a1 1804 if (pci_is_pcie(dev_tmp)) {
ba395927
KA		 1805 bus = dev_tmp->subordinate->number;
1806 devfn = 0;
1807 } else {
1808 bus = dev_tmp->bus->number;
1809 devfn = dev_tmp->devfn;
1810 }
1811 spin_lock_irqsave(&device_domain_lock, flags);
1812 list_for_each_entry(info, &device_domain_list, global) {
276dbf99
DW		 1813 if (info->segment == segment &&
1814 info->bus == bus && info->devfn == devfn) {
ba395927
KA		 1815 found = info->domain;
1816 break;
1817 }
1818 }
1819 spin_unlock_irqrestore(&device_domain_lock, flags);
1820 /* pcie-pci bridge already has a domain, uses it */
1821 if (found) {
1822 domain = found;
1823 goto found_domain;
1824 }
1825 }
1826
2c2e2c38
FY		 1827 domain = alloc_domain();
1828 if (!domain)
1829 goto error;
1830
ba395927
KA		 1831 /* Allocate new domain for the device */
1832 drhd = dmar_find_matched_drhd_unit(pdev);
1833 if (!drhd) {
1834 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1835 pci_name(pdev));
1836 return NULL;
1837 }
1838 iommu = drhd->iommu;
1839
2c2e2c38
FY		 1840 ret = iommu_attach_domain(domain, iommu);
1841 if (ret) {
1842 domain_exit(domain);
ba395927 1843 goto error;
2c2e2c38 1844 }
ba395927
KA		 1845
1846 if (domain_init(domain, gaw)) {
1847 domain_exit(domain);
1848 goto error;
1849 }
1850
1851 /* register pcie-to-pci device */
1852 if (dev_tmp) {
1853 info = alloc_devinfo_mem();
1854 if (!info) {
1855 domain_exit(domain);
1856 goto error;
1857 }
276dbf99 1858 info->segment = segment;
ba395927
KA		 1859 info->bus = bus;
1860 info->devfn = devfn;
1861 info->dev = NULL;
1862 info->domain = domain;
1863 /* This domain is shared by devices under p2p bridge */
3b5410e7 1864 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
ba395927
KA		 1865
1866 /* pcie-to-pci bridge already has a domain, uses it */
1867 found = NULL;
1868 spin_lock_irqsave(&device_domain_lock, flags);
1869 list_for_each_entry(tmp, &device_domain_list, global) {
276dbf99
DW		 1870 if (tmp->segment == segment &&
1871 tmp->bus == bus && tmp->devfn == devfn) {
ba395927
KA		 1872 found = tmp->domain;
1873 break;
1874 }
1875 }
1876 if (found) {
00dfff77 1877 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927
KA		 1878 free_devinfo_mem(info);
1879 domain_exit(domain);
1880 domain = found;
1881 } else {
1882 list_add(&info->link, &domain->devices);
1883 list_add(&info->global, &device_domain_list);
00dfff77 1884 spin_unlock_irqrestore(&device_domain_lock, flags);
ba395927 1885 }
ba395927
KA		 1886 }
1887
1888found_domain:
1889 info = alloc_devinfo_mem();
1890 if (!info)
1891 goto error;
276dbf99 1892 info->segment = segment;
ba395927
KA		 1893 info->bus = pdev->bus->number;
1894 info->devfn = pdev->devfn;
1895 info->dev = pdev;
1896 info->domain = domain;
1897 spin_lock_irqsave(&device_domain_lock, flags);
1898 /* somebody is fast */
1899 found = find_domain(pdev);
1900 if (found != NULL) {
1901 spin_unlock_irqrestore(&device_domain_lock, flags);
1902 if (found != domain) {
1903 domain_exit(domain);
1904 domain = found;
1905 }
1906 free_devinfo_mem(info);
1907 return domain;
1908 }
1909 list_add(&info->link, &domain->devices);
1910 list_add(&info->global, &device_domain_list);
358dd8ac 1911 pdev->dev.archdata.iommu = info;
ba395927
KA		 1912 spin_unlock_irqrestore(&device_domain_lock, flags);
1913 return domain;
1914error:
1915 /* recheck it here, maybe others set it */
1916 return find_domain(pdev);
1917}
1918
2c2e2c38 1919static int iommu_identity_mapping;
e0fc7e0b
DW		 1920#define IDENTMAP_ALL 1
1921#define IDENTMAP_GFX 2
1922#define IDENTMAP_AZALIA 4
2c2e2c38 1923
b213203e
DW		 1924static int iommu_domain_identity_map(struct dmar_domain *domain,
1925 unsigned long long start,
1926 unsigned long long end)
ba395927 1927{
c5395d5c
DW		 1928 unsigned long first_vpfn = start >> VTD_PAGE_SHIFT;
1929 unsigned long last_vpfn = end >> VTD_PAGE_SHIFT;
1930
1931 if (!reserve_iova(&domain->iovad, dma_to_mm_pfn(first_vpfn),
1932 dma_to_mm_pfn(last_vpfn))) {
ba395927 1933 printk(KERN_ERR "IOMMU: reserve iova failed\n");
b213203e 1934 return -ENOMEM;
ba395927
KA		 1935 }
1936
c5395d5c
DW		 1937 pr_debug("Mapping reserved region %llx-%llx for domain %d\n",
1938 start, end, domain->id);
ba395927
KA		 1939 /*
1940 * RMRR range might have overlap with physical memory range,
1941 * clear it first
1942 */
c5395d5c 1943 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
ba395927 1944
c5395d5c
DW		 1945 return domain_pfn_mapping(domain, first_vpfn, first_vpfn,
1946 last_vpfn - first_vpfn + 1,
61df7443 1947 DMA_PTE_READ|DMA_PTE_WRITE);
b213203e
DW		 1948}
1949
1950static int iommu_prepare_identity_map(struct pci_dev *pdev,
1951 unsigned long long start,
1952 unsigned long long end)
1953{
1954 struct dmar_domain *domain;
1955 int ret;
1956
c7ab48d2 1957 domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
b213203e
DW		 1958 if (!domain)
1959 return -ENOMEM;
1960
19943b0e
DW		 1961 /* For _hardware_ passthrough, don't bother. But for software
1962 passthrough, we do it anyway -- it may indicate a memory
1963 range which is reserved in E820, so which didn't get set
1964 up to start with in si_domain */
1965 if (domain == si_domain && hw_pass_through) {
1966 printk("Ignoring identity map for HW passthrough device %s [0x%Lx - 0x%Lx]\n",
1967 pci_name(pdev), start, end);
1968 return 0;
1969 }
1970
1971 printk(KERN_INFO
1972 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1973 pci_name(pdev), start, end);
2ff729f5 1974
5595b528
DW		 1975 if (end < start) {
1976 WARN(1, "Your BIOS is broken; RMRR ends before it starts!\n"
1977 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
1978 dmi_get_system_info(DMI_BIOS_VENDOR),
1979 dmi_get_system_info(DMI_BIOS_VERSION),
1980 dmi_get_system_info(DMI_PRODUCT_VERSION));
1981 ret = -EIO;
1982 goto error;
1983 }
1984
2ff729f5
DW		 1985 if (end >> agaw_to_width(domain->agaw)) {
1986 WARN(1, "Your BIOS is broken; RMRR exceeds permitted address width (%d bits)\n"
1987 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
1988 agaw_to_width(domain->agaw),
1989 dmi_get_system_info(DMI_BIOS_VENDOR),
1990 dmi_get_system_info(DMI_BIOS_VERSION),
1991 dmi_get_system_info(DMI_PRODUCT_VERSION));
1992 ret = -EIO;
1993 goto error;
1994 }
19943b0e 1995
b213203e 1996 ret = iommu_domain_identity_map(domain, start, end);
ba395927
KA		 1997 if (ret)
1998 goto error;
1999
2000 /* context entry init */
4ed0d3e6 2001 ret = domain_context_mapping(domain, pdev, CONTEXT_TT_MULTI_LEVEL);
b213203e
DW		 2002 if (ret)
2003 goto error;
2004
2005 return 0;
2006
2007 error:
ba395927
KA		 2008 domain_exit(domain);
2009 return ret;
ba395927
KA		 2010}
2011
2012static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
2013 struct pci_dev *pdev)
2014{
358dd8ac 2015 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
ba395927
KA		 2016 return 0;
2017 return iommu_prepare_identity_map(pdev, rmrr->base_address,
2018 rmrr->end_address + 1);
2019}
2020
49a0429e
KA		 2021#ifdef CONFIG_DMAR_FLOPPY_WA
2022static inline void iommu_prepare_isa(void)
2023{
2024 struct pci_dev *pdev;
2025 int ret;
2026
2027 pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
2028 if (!pdev)
2029 return;
2030
c7ab48d2 2031 printk(KERN_INFO "IOMMU: Prepare 0-16MiB unity mapping for LPC\n");
49a0429e
KA		 2032 ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
2033
2034 if (ret)
c7ab48d2
DW		 2035 printk(KERN_ERR "IOMMU: Failed to create 0-16MiB identity map; "
2036 "floppy might not work\n");
49a0429e
KA		 2037
2038}
2039#else
2040static inline void iommu_prepare_isa(void)
2041{
2042 return;
2043}
2044#endif /* !CONFIG_DMAR_FLPY_WA */
2045
2c2e2c38 2046static int md_domain_init(struct dmar_domain *domain, int guest_width);
c7ab48d2
DW		 2047
2048static int __init si_domain_work_fn(unsigned long start_pfn,
2049 unsigned long end_pfn, void *datax)
2050{
2051 int *ret = datax;
2052
2053 *ret = iommu_domain_identity_map(si_domain,
2054 (uint64_t)start_pfn << PAGE_SHIFT,
2055 (uint64_t)end_pfn << PAGE_SHIFT);
2056 return *ret;
2057
2058}
2059
071e1374 2060static int __init si_domain_init(int hw)
2c2e2c38
FY		 2061{
2062 struct dmar_drhd_unit *drhd;
2063 struct intel_iommu *iommu;
c7ab48d2 2064 int nid, ret = 0;
2c2e2c38
FY		 2065
2066 si_domain = alloc_domain();
2067 if (!si_domain)
2068 return -EFAULT;
2069
c7ab48d2 2070 pr_debug("Identity mapping domain is domain %d\n", si_domain->id);
2c2e2c38
FY		 2071
2072 for_each_active_iommu(iommu, drhd) {
2073 ret = iommu_attach_domain(si_domain, iommu);
2074 if (ret) {
2075 domain_exit(si_domain);
2076 return -EFAULT;
2077 }
2078 }
2079
2080 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2081 domain_exit(si_domain);
2082 return -EFAULT;
2083 }
2084
2085 si_domain->flags = DOMAIN_FLAG_STATIC_IDENTITY;
2086
19943b0e
DW		 2087 if (hw)
2088 return 0;
2089
c7ab48d2
DW		 2090 for_each_online_node(nid) {
2091 work_with_active_regions(nid, si_domain_work_fn, &ret);
2092 if (ret)
2093 return ret;
2094 }
2095
2c2e2c38
FY		 2096 return 0;
2097}
2098
2099static void domain_remove_one_dev_info(struct dmar_domain *domain,
2100 struct pci_dev *pdev);
2101static int identity_mapping(struct pci_dev *pdev)
2102{
2103 struct device_domain_info *info;
2104
2105 if (likely(!iommu_identity_mapping))
2106 return 0;
2107
2108
2109 list_for_each_entry(info, &si_domain->devices, link)
2110 if (info->dev == pdev)
2111 return 1;
2112 return 0;
2113}
2114
2115static int domain_add_dev_info(struct dmar_domain *domain,
5fe60f4e
DW		 2116 struct pci_dev *pdev,
2117 int translation)
2c2e2c38
FY		 2118{
2119 struct device_domain_info *info;
2120 unsigned long flags;
5fe60f4e 2121 int ret;
2c2e2c38
FY		 2122
2123 info = alloc_devinfo_mem();
2124 if (!info)
2125 return -ENOMEM;
2126
5fe60f4e
DW		 2127 ret = domain_context_mapping(domain, pdev, translation);
2128 if (ret) {
2129 free_devinfo_mem(info);
2130 return ret;
2131 }
2132
2c2e2c38
FY		 2133 info->segment = pci_domain_nr(pdev->bus);
2134 info->bus = pdev->bus->number;
2135 info->devfn = pdev->devfn;
2136 info->dev = pdev;
2137 info->domain = domain;
2138
2139 spin_lock_irqsave(&device_domain_lock, flags);
2140 list_add(&info->link, &domain->devices);
2141 list_add(&info->global, &device_domain_list);
2142 pdev->dev.archdata.iommu = info;
2143 spin_unlock_irqrestore(&device_domain_lock, flags);
2144
2145 return 0;
2146}
2147
6941af28
DW		 2148static int iommu_should_identity_map(struct pci_dev *pdev, int startup)
2149{
e0fc7e0b
DW		 2150 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2151 return 1;
2152
2153 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2154 return 1;
2155
2156 if (!(iommu_identity_mapping & IDENTMAP_ALL))
2157 return 0;
6941af28 2158
3dfc813d
DW		 2159 /*
2160 * We want to start off with all devices in the 1:1 domain, and
2161 * take them out later if we find they can't access all of memory.
2162 *
2163 * However, we can't do this for PCI devices behind bridges,
2164 * because all PCI devices behind the same bridge will end up
2165 * with the same source-id on their transactions.
2166 *
2167 * Practically speaking, we can't change things around for these
2168 * devices at run-time, because we can't be sure there'll be no
2169 * DMA transactions in flight for any of their siblings.
2170 *
2171 * So PCI devices (unless they're on the root bus) as well as
2172 * their parent PCI-PCI or PCIe-PCI bridges must be left _out_ of
2173 * the 1:1 domain, just in _case_ one of their siblings turns out
2174 * not to be able to map all of memory.
2175 */
5f4d91a1 2176 if (!pci_is_pcie(pdev)) {
3dfc813d
DW		 2177 if (!pci_is_root_bus(pdev->bus))
2178 return 0;
2179 if (pdev->class >> 8 == PCI_CLASS_BRIDGE_PCI)
2180 return 0;
2181 } else if (pdev->pcie_type == PCI_EXP_TYPE_PCI_BRIDGE)
2182 return 0;
2183
2184 /*
2185 * At boot time, we don't yet know if devices will be 64-bit capable.
2186 * Assume that they will -- if they turn out not to be, then we can
2187 * take them out of the 1:1 domain later.
2188 */
6941af28
DW		 2189 if (!startup)
2190 return pdev->dma_mask > DMA_BIT_MASK(32);
2191
2192 return 1;
2193}
2194
071e1374 2195static int __init iommu_prepare_static_identity_mapping(int hw)
2c2e2c38 2196{
2c2e2c38
FY		 2197 struct pci_dev *pdev = NULL;
2198 int ret;
2199
19943b0e 2200 ret = si_domain_init(hw);
2c2e2c38
FY		 2201 if (ret)
2202 return -EFAULT;
2203
2c2e2c38 2204 for_each_pci_dev(pdev) {
6941af28 2205 if (iommu_should_identity_map(pdev, 1)) {
19943b0e
DW		 2206 printk(KERN_INFO "IOMMU: %s identity mapping for device %s\n",
2207 hw ? "hardware" : "software", pci_name(pdev));
62edf5dc 2208
5fe60f4e 2209 ret = domain_add_dev_info(si_domain, pdev,
19943b0e 2210 hw ? CONTEXT_TT_PASS_THROUGH :
62edf5dc
DW		 2211 CONTEXT_TT_MULTI_LEVEL);
2212 if (ret)
2213 return ret;
62edf5dc 2214 }
2c2e2c38
FY		 2215 }
2216
2217 return 0;
2218}
2219
2220int __init init_dmars(void)
ba395927
KA		 2221{
2222 struct dmar_drhd_unit *drhd;
2223 struct dmar_rmrr_unit *rmrr;
2224 struct pci_dev *pdev;
2225 struct intel_iommu *iommu;
9d783ba0 2226 int i, ret;
2c2e2c38 2227
ba395927
KA		 2228 /*
2229 * for each drhd
2230 * allocate root
2231 * initialize and program root entry to not present
2232 * endfor
2233 */
2234 for_each_drhd_unit(drhd) {
5e0d2a6f 2235 g_num_of_iommus++;
2236 /*
2237 * lock not needed as this is only incremented in the single
2238 * threaded kernel __init code path all other access are read
2239 * only
2240 */
2241 }
2242
d9630fe9
WH		 2243 g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
2244 GFP_KERNEL);
2245 if (!g_iommus) {
2246 printk(KERN_ERR "Allocating global iommu array failed\n");
2247 ret = -ENOMEM;
2248 goto error;
2249 }
2250
80b20dd8 2251 deferred_flush = kzalloc(g_num_of_iommus *
2252 sizeof(struct deferred_flush_tables), GFP_KERNEL);
2253 if (!deferred_flush) {
5e0d2a6f 2254 ret = -ENOMEM;
2255 goto error;
2256 }
2257
5e0d2a6f 2258 for_each_drhd_unit(drhd) {
2259 if (drhd->ignored)
2260 continue;
1886e8a9
SS		 2261
2262 iommu = drhd->iommu;
d9630fe9 2263 g_iommus[iommu->seq_id] = iommu;
ba395927 2264
e61d98d8
SS		 2265 ret = iommu_init_domains(iommu);
2266 if (ret)
2267 goto error;
2268
ba395927
KA		 2269 /*
2270 * TBD:
2271 * we could share the same root & context tables
2272 * amoung all IOMMU's. Need to Split it later.
2273 */
2274 ret = iommu_alloc_root_entry(iommu);
2275 if (ret) {
2276 printk(KERN_ERR "IOMMU: allocate root entry failed\n");
2277 goto error;
2278 }
4ed0d3e6 2279 if (!ecap_pass_through(iommu->ecap))
19943b0e 2280 hw_pass_through = 0;
ba395927
KA		 2281 }
2282
1531a6a6
SS		 2283 /*
2284 * Start from the sane iommu hardware state.
2285 */
a77b67d4
YS		 2286 for_each_drhd_unit(drhd) {
2287 if (drhd->ignored)
2288 continue;
2289
2290 iommu = drhd->iommu;
1531a6a6
SS		 2291
2292 /*
2293 * If the queued invalidation is already initialized by us
2294 * (for example, while enabling interrupt-remapping) then
2295 * we got the things already rolling from a sane state.
2296 */
2297 if (iommu->qi)
2298 continue;
2299
2300 /*
2301 * Clear any previous faults.
2302 */
2303 dmar_fault(-1, iommu);
2304 /*
2305 * Disable queued invalidation if supported and already enabled
2306 * before OS handover.
2307 */
2308 dmar_disable_qi(iommu);
2309 }
2310
2311 for_each_drhd_unit(drhd) {
2312 if (drhd->ignored)
2313 continue;
2314
2315 iommu = drhd->iommu;
2316
a77b67d4
YS		 2317 if (dmar_enable_qi(iommu)) {
2318 /*
2319 * Queued Invalidate not enabled, use Register Based
2320 * Invalidate
2321 */
2322 iommu->flush.flush_context = __iommu_flush_context;
2323 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
680a7524 2324 printk(KERN_INFO "IOMMU %d 0x%Lx: using Register based "
b4e0f9eb 2325 "invalidation\n",
680a7524 2326 iommu->seq_id,
b4e0f9eb 2327 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2328 } else {
2329 iommu->flush.flush_context = qi_flush_context;
2330 iommu->flush.flush_iotlb = qi_flush_iotlb;
680a7524 2331 printk(KERN_INFO "IOMMU %d 0x%Lx: using Queued "
b4e0f9eb 2332 "invalidation\n",
680a7524 2333 iommu->seq_id,
b4e0f9eb 2334 (unsigned long long)drhd->reg_base_addr);
a77b67d4
YS		 2335 }
2336 }
2337
19943b0e 2338 if (iommu_pass_through)
e0fc7e0b
DW		 2339 iommu_identity_mapping |= IDENTMAP_ALL;
2340
19943b0e 2341#ifdef CONFIG_DMAR_BROKEN_GFX_WA
e0fc7e0b 2342 iommu_identity_mapping |= IDENTMAP_GFX;
19943b0e 2343#endif
e0fc7e0b
DW		 2344
2345 check_tylersburg_isoch();
2346
ba395927 2347 /*
19943b0e
DW		 2348 * If pass through is not set or not enabled, setup context entries for
2349 * identity mappings for rmrr, gfx, and isa and may fall back to static
2350 * identity mapping if iommu_identity_mapping is set.
ba395927 2351 */
19943b0e
DW		 2352 if (iommu_identity_mapping) {
2353 ret = iommu_prepare_static_identity_mapping(hw_pass_through);
4ed0d3e6 2354 if (ret) {
19943b0e
DW		 2355 printk(KERN_CRIT "Failed to setup IOMMU pass-through\n");
2356 goto error;
ba395927
KA		 2357 }
2358 }
ba395927 2359 /*
19943b0e
DW		 2360 * For each rmrr
2361 * for each dev attached to rmrr
2362 * do
2363 * locate drhd for dev, alloc domain for dev
2364 * allocate free domain
2365 * allocate page table entries for rmrr
2366 * if context not allocated for bus
2367 * allocate and init context
2368 * set present in root table for this bus
2369 * init context with domain, translation etc
2370 * endfor
2371 * endfor
ba395927 2372 */
19943b0e
DW		 2373 printk(KERN_INFO "IOMMU: Setting RMRR:\n");
2374 for_each_rmrr_units(rmrr) {
2375 for (i = 0; i < rmrr->devices_cnt; i++) {
2376 pdev = rmrr->devices[i];
2377 /*
2378 * some BIOS lists non-exist devices in DMAR
2379 * table.
2380 */
2381 if (!pdev)
2382 continue;
2383 ret = iommu_prepare_rmrr_dev(rmrr, pdev);
2384 if (ret)
2385 printk(KERN_ERR
2386 "IOMMU: mapping reserved region failed\n");
ba395927 2387 }
4ed0d3e6 2388 }
49a0429e 2389
19943b0e
DW		 2390 iommu_prepare_isa();
2391
ba395927
KA		 2392 /*
2393 * for each drhd
2394 * enable fault log
2395 * global invalidate context cache
2396 * global invalidate iotlb
2397 * enable translation
2398 */
2399 for_each_drhd_unit(drhd) {
2400 if (drhd->ignored)
2401 continue;
2402 iommu = drhd->iommu;
ba395927
KA		 2403
2404 iommu_flush_write_buffer(iommu);
2405
3460a6d9
KA		 2406 ret = dmar_set_interrupt(iommu);
2407 if (ret)
2408 goto error;
2409
ba395927
KA		 2410 iommu_set_root_entry(iommu);
2411
4c25a2c1 2412 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1f0ef2aa 2413 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
f8bab735 2414
ba395927
KA		 2415 ret = iommu_enable_translation(iommu);
2416 if (ret)
2417 goto error;
b94996c9
DW		 2418
2419 iommu_disable_protect_mem_regions(iommu);
ba395927
KA		 2420 }
2421
2422 return 0;
2423error:
2424 for_each_drhd_unit(drhd) {
2425 if (drhd->ignored)
2426 continue;
2427 iommu = drhd->iommu;
2428 free_iommu(iommu);
2429 }
d9630fe9 2430 kfree(g_iommus);
ba395927
KA		 2431 return ret;
2432}
2433
5a5e02a6 2434/* This takes a number of _MM_ pages, not VTD pages */
875764de
DW		 2435static struct iova *intel_alloc_iova(struct device *dev,
2436 struct dmar_domain *domain,
2437 unsigned long nrpages, uint64_t dma_mask)
ba395927 2438{
ba395927 2439 struct pci_dev *pdev = to_pci_dev(dev);
ba395927 2440 struct iova *iova = NULL;
ba395927 2441
875764de
DW		 2442 /* Restrict dma_mask to the width that the iommu can handle */
2443 dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw), dma_mask);
2444
2445 if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
ba395927
KA		 2446 /*
2447 * First try to allocate an io virtual address in
284901a9 2448 * DMA_BIT_MASK(32) and if that fails then try allocating
3609801e 2449 * from higher range
ba395927 2450 */
875764de
DW		 2451 iova = alloc_iova(&domain->iovad, nrpages,
2452 IOVA_PFN(DMA_BIT_MASK(32)), 1);
2453 if (iova)
2454 return iova;
2455 }
2456 iova = alloc_iova(&domain->iovad, nrpages, IOVA_PFN(dma_mask), 1);
2457 if (unlikely(!iova)) {
2458 printk(KERN_ERR "Allocating %ld-page iova for %s failed",
2459 nrpages, pci_name(pdev));
f76aec76
KA		 2460 return NULL;
2461 }
2462
2463 return iova;
2464}
2465
147202aa 2466static struct dmar_domain *__get_valid_domain_for_dev(struct pci_dev *pdev)
f76aec76
KA		 2467{
2468 struct dmar_domain *domain;
2469 int ret;
2470
2471 domain = get_domain_for_dev(pdev,
2472 DEFAULT_DOMAIN_ADDRESS_WIDTH);
2473 if (!domain) {
2474 printk(KERN_ERR
2475 "Allocating domain for %s failed", pci_name(pdev));
4fe05bbc 2476 return NULL;
ba395927
KA		 2477 }
2478
2479 /* make sure context mapping is ok */
5331fe6f 2480 if (unlikely(!domain_context_mapped(pdev))) {
4ed0d3e6
FY		 2481 ret = domain_context_mapping(domain, pdev,
2482 CONTEXT_TT_MULTI_LEVEL);
f76aec76
KA		 2483 if (ret) {
2484 printk(KERN_ERR
2485 "Domain context map for %s failed",
2486 pci_name(pdev));
4fe05bbc 2487 return NULL;
f76aec76 2488 }
ba395927
KA		 2489 }
2490
f76aec76
KA		 2491 return domain;
2492}
2493
147202aa
DW		 2494static inline struct dmar_domain *get_valid_domain_for_dev(struct pci_dev *dev)
2495{
2496 struct device_domain_info *info;
2497
2498 /* No lock here, assumes no domain exit in normal case */
2499 info = dev->dev.archdata.iommu;
2500 if (likely(info))
2501 return info->domain;
2502
2503 return __get_valid_domain_for_dev(dev);
2504}
2505
2c2e2c38
FY		 2506static int iommu_dummy(struct pci_dev *pdev)
2507{
2508 return pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO;
2509}
2510
2511/* Check if the pdev needs to go through non-identity map and unmap process.*/
73676832 2512static int iommu_no_mapping(struct device *dev)
2c2e2c38 2513{
73676832 2514 struct pci_dev *pdev;
2c2e2c38
FY		 2515 int found;
2516
73676832
DW		 2517 if (unlikely(dev->bus != &pci_bus_type))
2518 return 1;
2519
2520 pdev = to_pci_dev(dev);
1e4c64c4
DW		 2521 if (iommu_dummy(pdev))
2522 return 1;
2523
2c2e2c38 2524 if (!iommu_identity_mapping)
1e4c64c4 2525 return 0;
2c2e2c38
FY		 2526
2527 found = identity_mapping(pdev);
2528 if (found) {
6941af28 2529 if (iommu_should_identity_map(pdev, 0))
2c2e2c38
FY		 2530 return 1;
2531 else {
2532 /*
2533 * 32 bit DMA is removed from si_domain and fall back
2534 * to non-identity mapping.
2535 */
2536 domain_remove_one_dev_info(si_domain, pdev);
2537 printk(KERN_INFO "32bit %s uses non-identity mapping\n",
2538 pci_name(pdev));
2539 return 0;
2540 }
2541 } else {
2542 /*
2543 * In case of a detached 64 bit DMA device from vm, the device
2544 * is put into si_domain for identity mapping.
2545 */
6941af28 2546 if (iommu_should_identity_map(pdev, 0)) {
2c2e2c38 2547 int ret;
5fe60f4e
DW		 2548 ret = domain_add_dev_info(si_domain, pdev,
2549 hw_pass_through ?
2550 CONTEXT_TT_PASS_THROUGH :
2551 CONTEXT_TT_MULTI_LEVEL);
2c2e2c38
FY		 2552 if (!ret) {
2553 printk(KERN_INFO "64bit %s uses identity mapping\n",
2554 pci_name(pdev));
2555 return 1;
2556 }
2557 }
2558 }
2559
1e4c64c4 2560 return 0;
2c2e2c38
FY		 2561}
2562
bb9e6d65
FT		 2563static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2564 size_t size, int dir, u64 dma_mask)
f76aec76
KA		 2565{
2566 struct pci_dev *pdev = to_pci_dev(hwdev);
f76aec76 2567 struct dmar_domain *domain;
5b6985ce 2568 phys_addr_t start_paddr;
f76aec76
KA		 2569 struct iova *iova;
2570 int prot = 0;
6865f0d1 2571 int ret;
8c11e798 2572 struct intel_iommu *iommu;
33041ec0 2573 unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
f76aec76
KA		 2574
2575 BUG_ON(dir == DMA_NONE);
2c2e2c38 2576
73676832 2577 if (iommu_no_mapping(hwdev))
6865f0d1 2578 return paddr;
f76aec76
KA		 2579
2580 domain = get_valid_domain_for_dev(pdev);
2581 if (!domain)
2582 return 0;
2583
8c11e798 2584 iommu = domain_get_iommu(domain);
88cb6a74 2585 size = aligned_nrpages(paddr, size);
f76aec76 2586
5a5e02a6
DW		 2587 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
2588 pdev->dma_mask);
f76aec76
KA		 2589 if (!iova)
2590 goto error;
2591
ba395927
KA		 2592 /*
2593 * Check if DMAR supports zero-length reads on write only
2594 * mappings..
2595 */
2596 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 2597 !cap_zlr(iommu->cap))
ba395927
KA		 2598 prot |= DMA_PTE_READ;
2599 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2600 prot |= DMA_PTE_WRITE;
2601 /*
6865f0d1 2602 * paddr - (paddr + size) might be partial page, we should map the whole
ba395927 2603 * page. Note: if two part of one page are separately mapped, we
6865f0d1 2604 * might have two guest_addr mapping to the same host paddr, but this
ba395927
KA		 2605 * is not a big problem
2606 */
0ab36de2 2607 ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova->pfn_lo),
33041ec0 2608 mm_to_dma_pfn(paddr_pfn), size, prot);
ba395927
KA		 2609 if (ret)
2610 goto error;
2611
1f0ef2aa
DW		 2612 /* it's a non-present to present mapping. Only flush if caching mode */
2613 if (cap_caching_mode(iommu->cap))
82653633 2614 iommu_flush_iotlb_psi(iommu, domain->id, mm_to_dma_pfn(iova->pfn_lo), size, 1);
1f0ef2aa 2615 else
8c11e798 2616 iommu_flush_write_buffer(iommu);
f76aec76 2617
03d6a246
DW		 2618 start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2619 start_paddr += paddr & ~PAGE_MASK;
2620 return start_paddr;
ba395927 2621
ba395927 2622error:
f76aec76
KA		 2623 if (iova)
2624 __free_iova(&domain->iovad, iova);
4cf2e75d 2625 printk(KERN_ERR"Device %s request: %zx@%llx dir %d --- failed\n",
5b6985ce 2626 pci_name(pdev), size, (unsigned long long)paddr, dir);
ba395927
KA		 2627 return 0;
2628}
2629
ffbbef5c
FT		 2630static dma_addr_t intel_map_page(struct device *dev, struct page *page,
2631 unsigned long offset, size_t size,
2632 enum dma_data_direction dir,
2633 struct dma_attrs *attrs)
bb9e6d65 2634{
ffbbef5c
FT		 2635 return __intel_map_single(dev, page_to_phys(page) + offset, size,
2636 dir, to_pci_dev(dev)->dma_mask);
bb9e6d65
FT		 2637}
2638
5e0d2a6f 2639static void flush_unmaps(void)
2640{
80b20dd8 2641 int i, j;
5e0d2a6f 2642
5e0d2a6f 2643 timer_on = 0;
2644
2645 /* just flush them all */
2646 for (i = 0; i < g_num_of_iommus; i++) {
a2bb8459
WH		 2647 struct intel_iommu *iommu = g_iommus[i];
2648 if (!iommu)
2649 continue;
c42d9f32 2650
9dd2fe89
YZ		 2651 if (!deferred_flush[i].next)
2652 continue;
2653
78d5f0f5
NA		 2654 /* In caching mode, global flushes turn emulation expensive */
2655 if (!cap_caching_mode(iommu->cap))
2656 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
93a23a72 2657 DMA_TLB_GLOBAL_FLUSH);
9dd2fe89 2658 for (j = 0; j < deferred_flush[i].next; j++) {
93a23a72
YZ		 2659 unsigned long mask;
2660 struct iova *iova = deferred_flush[i].iova[j];
78d5f0f5
NA		 2661 struct dmar_domain *domain = deferred_flush[i].domain[j];
2662
2663 /* On real hardware multiple invalidations are expensive */
2664 if (cap_caching_mode(iommu->cap))
2665 iommu_flush_iotlb_psi(iommu, domain->id,
2666 iova->pfn_lo, iova->pfn_hi - iova->pfn_lo + 1, 0);
2667 else {
2668 mask = ilog2(mm_to_dma_pfn(iova->pfn_hi - iova->pfn_lo + 1));
2669 iommu_flush_dev_iotlb(deferred_flush[i].domain[j],
2670 (uint64_t)iova->pfn_lo << PAGE_SHIFT, mask);
2671 }
93a23a72 2672 __free_iova(&deferred_flush[i].domain[j]->iovad, iova);
80b20dd8 2673 }
9dd2fe89 2674 deferred_flush[i].next = 0;
5e0d2a6f 2675 }
2676
5e0d2a6f 2677 list_size = 0;
5e0d2a6f 2678}
2679
2680static void flush_unmaps_timeout(unsigned long data)
2681{
80b20dd8 2682 unsigned long flags;
2683
2684 spin_lock_irqsave(&async_umap_flush_lock, flags);
5e0d2a6f 2685 flush_unmaps();
80b20dd8 2686 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
5e0d2a6f 2687}
2688
2689static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2690{
2691 unsigned long flags;
80b20dd8 2692 int next, iommu_id;
8c11e798 2693 struct intel_iommu *iommu;
5e0d2a6f 2694
2695 spin_lock_irqsave(&async_umap_flush_lock, flags);
80b20dd8 2696 if (list_size == HIGH_WATER_MARK)
2697 flush_unmaps();
2698
8c11e798
WH		 2699 iommu = domain_get_iommu(dom);
2700 iommu_id = iommu->seq_id;
c42d9f32 2701
80b20dd8 2702 next = deferred_flush[iommu_id].next;
2703 deferred_flush[iommu_id].domain[next] = dom;
2704 deferred_flush[iommu_id].iova[next] = iova;
2705 deferred_flush[iommu_id].next++;
5e0d2a6f 2706
2707 if (!timer_on) {
2708 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2709 timer_on = 1;
2710 }
2711 list_size++;
2712 spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2713}
2714
ffbbef5c
FT		 2715static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
2716 size_t size, enum dma_data_direction dir,
2717 struct dma_attrs *attrs)
ba395927 2718{
ba395927 2719 struct pci_dev *pdev = to_pci_dev(dev);
f76aec76 2720 struct dmar_domain *domain;
d794dc9b 2721 unsigned long start_pfn, last_pfn;
ba395927 2722 struct iova *iova;
8c11e798 2723 struct intel_iommu *iommu;
ba395927 2724
73676832 2725 if (iommu_no_mapping(dev))
f76aec76 2726 return;
2c2e2c38 2727
ba395927
KA		 2728 domain = find_domain(pdev);
2729 BUG_ON(!domain);
2730
8c11e798
WH		 2731 iommu = domain_get_iommu(domain);
2732
ba395927 2733 iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
85b98276
DW		 2734 if (WARN_ONCE(!iova, "Driver unmaps unmatched page at PFN %llx\n",
2735 (unsigned long long)dev_addr))
ba395927 2736 return;
ba395927 2737
d794dc9b
DW		 2738 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2739 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
ba395927 2740
d794dc9b
DW		 2741 pr_debug("Device %s unmapping: pfn %lx-%lx\n",
2742 pci_name(pdev), start_pfn, last_pfn);
ba395927 2743
f76aec76 2744 /* clear the whole page */
d794dc9b
DW		 2745 dma_pte_clear_range(domain, start_pfn, last_pfn);
2746
f76aec76 2747 /* free page tables */
d794dc9b
DW		 2748 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
2749
5e0d2a6f 2750 if (intel_iommu_strict) {
03d6a246 2751 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
82653633 2752 last_pfn - start_pfn + 1, 0);
5e0d2a6f 2753 /* free iova */
2754 __free_iova(&domain->iovad, iova);
2755 } else {
2756 add_unmap(domain, iova);
2757 /*
2758 * queue up the release of the unmap to save the 1/6th of the
2759 * cpu used up by the iotlb flush operation...
2760 */
5e0d2a6f 2761 }
ba395927
KA		 2762}
2763
d7ab5c46
FT		 2764static void *intel_alloc_coherent(struct device *hwdev, size_t size,
2765 dma_addr_t *dma_handle, gfp_t flags)
ba395927
KA		 2766{
2767 void *vaddr;
2768 int order;
2769
5b6985ce 2770 size = PAGE_ALIGN(size);
ba395927 2771 order = get_order(size);
e8bb910d
AW		 2772
2773 if (!iommu_no_mapping(hwdev))
2774 flags &= ~(GFP_DMA | GFP_DMA32);
2775 else if (hwdev->coherent_dma_mask < dma_get_required_mask(hwdev)) {
2776 if (hwdev->coherent_dma_mask < DMA_BIT_MASK(32))
2777 flags |= GFP_DMA;
2778 else
2779 flags |= GFP_DMA32;
2780 }
ba395927
KA		 2781
2782 vaddr = (void *)__get_free_pages(flags, order);
2783 if (!vaddr)
2784 return NULL;
2785 memset(vaddr, 0, size);
2786
bb9e6d65
FT		 2787 *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2788 DMA_BIDIRECTIONAL,
2789 hwdev->coherent_dma_mask);
ba395927
KA		 2790 if (*dma_handle)
2791 return vaddr;
2792 free_pages((unsigned long)vaddr, order);
2793 return NULL;
2794}
2795
d7ab5c46
FT		 2796static void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2797 dma_addr_t dma_handle)
ba395927
KA		 2798{
2799 int order;
2800
5b6985ce 2801 size = PAGE_ALIGN(size);
ba395927
KA		 2802 order = get_order(size);
2803
0db9b7ae 2804 intel_unmap_page(hwdev, dma_handle, size, DMA_BIDIRECTIONAL, NULL);
ba395927
KA		 2805 free_pages((unsigned long)vaddr, order);
2806}
2807
d7ab5c46
FT		 2808static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2809 int nelems, enum dma_data_direction dir,
2810 struct dma_attrs *attrs)
ba395927 2811{
ba395927
KA		 2812 struct pci_dev *pdev = to_pci_dev(hwdev);
2813 struct dmar_domain *domain;
d794dc9b 2814 unsigned long start_pfn, last_pfn;
f76aec76 2815 struct iova *iova;
8c11e798 2816 struct intel_iommu *iommu;
ba395927 2817
73676832 2818 if (iommu_no_mapping(hwdev))
ba395927
KA		 2819 return;
2820
2821 domain = find_domain(pdev);
8c11e798
WH		 2822 BUG_ON(!domain);
2823
2824 iommu = domain_get_iommu(domain);
ba395927 2825
c03ab37c 2826 iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
85b98276
DW		 2827 if (WARN_ONCE(!iova, "Driver unmaps unmatched sglist at PFN %llx\n",
2828 (unsigned long long)sglist[0].dma_address))
f76aec76 2829 return;
f76aec76 2830
d794dc9b
DW		 2831 start_pfn = mm_to_dma_pfn(iova->pfn_lo);
2832 last_pfn = mm_to_dma_pfn(iova->pfn_hi + 1) - 1;
f76aec76
KA		 2833
2834 /* clear the whole page */
d794dc9b
DW		 2835 dma_pte_clear_range(domain, start_pfn, last_pfn);
2836
f76aec76 2837 /* free page tables */
d794dc9b 2838 dma_pte_free_pagetable(domain, start_pfn, last_pfn);
f76aec76 2839
acea0018
DW		 2840 if (intel_iommu_strict) {
2841 iommu_flush_iotlb_psi(iommu, domain->id, start_pfn,
82653633 2842 last_pfn - start_pfn + 1, 0);
acea0018
DW		 2843 /* free iova */
2844 __free_iova(&domain->iovad, iova);
2845 } else {
2846 add_unmap(domain, iova);
2847 /*
2848 * queue up the release of the unmap to save the 1/6th of the
2849 * cpu used up by the iotlb flush operation...
2850 */
2851 }
ba395927
KA		 2852}
2853
ba395927 2854static int intel_nontranslate_map_sg(struct device *hddev,
c03ab37c 2855 struct scatterlist *sglist, int nelems, int dir)
ba395927
KA		 2856{
2857 int i;
c03ab37c 2858 struct scatterlist *sg;
ba395927 2859
c03ab37c 2860 for_each_sg(sglist, sg, nelems, i) {
12d4d40e 2861 BUG_ON(!sg_page(sg));
4cf2e75d 2862 sg->dma_address = page_to_phys(sg_page(sg)) + sg->offset;
c03ab37c 2863 sg->dma_length = sg->length;
ba395927
KA		 2864 }
2865 return nelems;
2866}
2867
d7ab5c46
FT		 2868static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2869 enum dma_data_direction dir, struct dma_attrs *attrs)
ba395927 2870{
ba395927 2871 int i;
ba395927
KA		 2872 struct pci_dev *pdev = to_pci_dev(hwdev);
2873 struct dmar_domain *domain;
f76aec76
KA		 2874 size_t size = 0;
2875 int prot = 0;
f76aec76
KA		 2876 struct iova *iova = NULL;
2877 int ret;
c03ab37c 2878 struct scatterlist *sg;
b536d24d 2879 unsigned long start_vpfn;
8c11e798 2880 struct intel_iommu *iommu;
ba395927
KA		 2881
2882 BUG_ON(dir == DMA_NONE);
73676832 2883 if (iommu_no_mapping(hwdev))
c03ab37c 2884 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
ba395927 2885
f76aec76
KA		 2886 domain = get_valid_domain_for_dev(pdev);
2887 if (!domain)
2888 return 0;
2889
8c11e798
WH		 2890 iommu = domain_get_iommu(domain);
2891
b536d24d 2892 for_each_sg(sglist, sg, nelems, i)
88cb6a74 2893 size += aligned_nrpages(sg->offset, sg->length);
f76aec76 2894
5a5e02a6
DW		 2895 iova = intel_alloc_iova(hwdev, domain, dma_to_mm_pfn(size),
2896 pdev->dma_mask);
f76aec76 2897 if (!iova) {
c03ab37c 2898 sglist->dma_length = 0;
f76aec76
KA		 2899 return 0;
2900 }
2901
2902 /*
2903 * Check if DMAR supports zero-length reads on write only
2904 * mappings..
2905 */
2906 if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
8c11e798 2907 !cap_zlr(iommu->cap))
f76aec76
KA		 2908 prot |= DMA_PTE_READ;
2909 if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2910 prot |= DMA_PTE_WRITE;
2911
b536d24d 2912 start_vpfn = mm_to_dma_pfn(iova->pfn_lo);
e1605495 2913
f532959b 2914 ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
e1605495
DW		 2915 if (unlikely(ret)) {
2916 /* clear the page */
2917 dma_pte_clear_range(domain, start_vpfn,
2918 start_vpfn + size - 1);
2919 /* free page tables */
2920 dma_pte_free_pagetable(domain, start_vpfn,
2921 start_vpfn + size - 1);
2922 /* free iova */
2923 __free_iova(&domain->iovad, iova);
2924 return 0;
ba395927
KA		 2925 }
2926
1f0ef2aa
DW		 2927 /* it's a non-present to present mapping. Only flush if caching mode */
2928 if (cap_caching_mode(iommu->cap))
82653633 2929 iommu_flush_iotlb_psi(iommu, domain->id, start_vpfn, size, 1);
1f0ef2aa 2930 else
8c11e798 2931 iommu_flush_write_buffer(iommu);
1f0ef2aa 2932
ba395927
KA		 2933 return nelems;
2934}
2935
dfb805e8
FT		 2936static int intel_mapping_error(struct device *dev, dma_addr_t dma_addr)
2937{
2938 return !dma_addr;
2939}
2940
160c1d8e 2941struct dma_map_ops intel_dma_ops = {
ba395927
KA		 2942 .alloc_coherent = intel_alloc_coherent,
2943 .free_coherent = intel_free_coherent,
ba395927
KA		 2944 .map_sg = intel_map_sg,
2945 .unmap_sg = intel_unmap_sg,
ffbbef5c
FT		 2946 .map_page = intel_map_page,
2947 .unmap_page = intel_unmap_page,
dfb805e8 2948 .mapping_error = intel_mapping_error,
ba395927
KA		 2949};
2950
2951static inline int iommu_domain_cache_init(void)
2952{
2953 int ret = 0;
2954
2955 iommu_domain_cache = kmem_cache_create("iommu_domain",
2956 sizeof(struct dmar_domain),
2957 0,
2958 SLAB_HWCACHE_ALIGN,
2959
2960 NULL);
2961 if (!iommu_domain_cache) {
2962 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2963 ret = -ENOMEM;
2964 }
2965
2966 return ret;
2967}
2968
2969static inline int iommu_devinfo_cache_init(void)
2970{
2971 int ret = 0;
2972
2973 iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2974 sizeof(struct device_domain_info),
2975 0,
2976 SLAB_HWCACHE_ALIGN,
ba395927
KA		 2977 NULL);
2978 if (!iommu_devinfo_cache) {
2979 printk(KERN_ERR "Couldn't create devinfo cache\n");
2980 ret = -ENOMEM;
2981 }
2982
2983 return ret;
2984}
2985
2986static inline int iommu_iova_cache_init(void)
2987{
2988 int ret = 0;
2989
2990 iommu_iova_cache = kmem_cache_create("iommu_iova",
2991 sizeof(struct iova),
2992 0,
2993 SLAB_HWCACHE_ALIGN,
ba395927
KA		 2994 NULL);
2995 if (!iommu_iova_cache) {
2996 printk(KERN_ERR "Couldn't create iova cache\n");
2997 ret = -ENOMEM;
2998 }
2999
3000 return ret;
3001}
3002
3003static int __init iommu_init_mempool(void)
3004{
3005 int ret;
3006 ret = iommu_iova_cache_init();
3007 if (ret)
3008 return ret;
3009
3010 ret = iommu_domain_cache_init();
3011 if (ret)
3012 goto domain_error;
3013
3014 ret = iommu_devinfo_cache_init();
3015 if (!ret)
3016 return ret;
3017
3018 kmem_cache_destroy(iommu_domain_cache);
3019domain_error:
3020 kmem_cache_destroy(iommu_iova_cache);
3021
3022 return -ENOMEM;
3023}
3024
3025static void __init iommu_exit_mempool(void)
3026{
3027 kmem_cache_destroy(iommu_devinfo_cache);
3028 kmem_cache_destroy(iommu_domain_cache);
3029 kmem_cache_destroy(iommu_iova_cache);
3030
3031}
3032
ba395927
KA		 3033static void __init init_no_remapping_devices(void)
3034{
3035 struct dmar_drhd_unit *drhd;
3036
3037 for_each_drhd_unit(drhd) {
3038 if (!drhd->include_all) {
3039 int i;
3040 for (i = 0; i < drhd->devices_cnt; i++)
3041 if (drhd->devices[i] != NULL)
3042 break;
3043 /* ignore DMAR unit if no pci devices exist */
3044 if (i == drhd->devices_cnt)
3045 drhd->ignored = 1;
3046 }
3047 }
3048
3049 if (dmar_map_gfx)
3050 return;
3051
3052 for_each_drhd_unit(drhd) {
3053 int i;
3054 if (drhd->ignored || drhd->include_all)
3055 continue;
3056
3057 for (i = 0; i < drhd->devices_cnt; i++)
3058 if (drhd->devices[i] &&
3059 !IS_GFX_DEVICE(drhd->devices[i]))
3060 break;
3061
3062 if (i < drhd->devices_cnt)
3063 continue;
3064
3065 /* bypass IOMMU if it is just for gfx devices */
3066 drhd->ignored = 1;
3067 for (i = 0; i < drhd->devices_cnt; i++) {
3068 if (!drhd->devices[i])
3069 continue;
358dd8ac 3070 drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
ba395927
KA		 3071 }
3072 }
3073}
3074
f59c7b69
FY		 3075#ifdef CONFIG_SUSPEND
3076static int init_iommu_hw(void)
3077{
3078 struct dmar_drhd_unit *drhd;
3079 struct intel_iommu *iommu = NULL;
3080
3081 for_each_active_iommu(iommu, drhd)
3082 if (iommu->qi)
3083 dmar_reenable_qi(iommu);
3084
3085 for_each_active_iommu(iommu, drhd) {
3086 iommu_flush_write_buffer(iommu);
3087
3088 iommu_set_root_entry(iommu);
3089
3090 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3091 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3092 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3093 DMA_TLB_GLOBAL_FLUSH);
f59c7b69 3094 iommu_enable_translation(iommu);
b94996c9 3095 iommu_disable_protect_mem_regions(iommu);
f59c7b69
FY		 3096 }
3097
3098 return 0;
3099}
3100
3101static void iommu_flush_all(void)
3102{
3103 struct dmar_drhd_unit *drhd;
3104 struct intel_iommu *iommu;
3105
3106 for_each_active_iommu(iommu, drhd) {
3107 iommu->flush.flush_context(iommu, 0, 0, 0,
1f0ef2aa 3108 DMA_CCMD_GLOBAL_INVL);
f59c7b69 3109 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1f0ef2aa 3110 DMA_TLB_GLOBAL_FLUSH);
f59c7b69
FY		 3111 }
3112}
3113
3114static int iommu_suspend(struct sys_device *dev, pm_message_t state)
3115{
3116 struct dmar_drhd_unit *drhd;
3117 struct intel_iommu *iommu = NULL;
3118 unsigned long flag;
3119
3120 for_each_active_iommu(iommu, drhd) {
3121 iommu->iommu_state = kzalloc(sizeof(u32) * MAX_SR_DMAR_REGS,
3122 GFP_ATOMIC);
3123 if (!iommu->iommu_state)
3124 goto nomem;
3125 }
3126
3127 iommu_flush_all();
3128
3129 for_each_active_iommu(iommu, drhd) {
3130 iommu_disable_translation(iommu);
3131
3132 spin_lock_irqsave(&iommu->register_lock, flag);
3133
3134 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3135 readl(iommu->reg + DMAR_FECTL_REG);
3136 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3137 readl(iommu->reg + DMAR_FEDATA_REG);
3138 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3139 readl(iommu->reg + DMAR_FEADDR_REG);
3140 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3141 readl(iommu->reg + DMAR_FEUADDR_REG);
3142
3143 spin_unlock_irqrestore(&iommu->register_lock, flag);
3144 }
3145 return 0;
3146
3147nomem:
3148 for_each_active_iommu(iommu, drhd)
3149 kfree(iommu->iommu_state);
3150
3151 return -ENOMEM;
3152}
3153
3154static int iommu_resume(struct sys_device *dev)
3155{
3156 struct dmar_drhd_unit *drhd;
3157 struct intel_iommu *iommu = NULL;
3158 unsigned long flag;
3159
3160 if (init_iommu_hw()) {
3161 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3162 return -EIO;
3163 }
3164
3165 for_each_active_iommu(iommu, drhd) {
3166
3167 spin_lock_irqsave(&iommu->register_lock, flag);
3168
3169 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3170 iommu->reg + DMAR_FECTL_REG);
3171 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3172 iommu->reg + DMAR_FEDATA_REG);
3173 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3174 iommu->reg + DMAR_FEADDR_REG);
3175 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3176 iommu->reg + DMAR_FEUADDR_REG);
3177
3178 spin_unlock_irqrestore(&iommu->register_lock, flag);
3179 }
3180
3181 for_each_active_iommu(iommu, drhd)
3182 kfree(iommu->iommu_state);
3183
3184 return 0;
3185}
3186
3187static struct sysdev_class iommu_sysclass = {
3188 .name = "iommu",
3189 .resume = iommu_resume,
3190 .suspend = iommu_suspend,
3191};
3192
3193static struct sys_device device_iommu = {
3194 .cls = &iommu_sysclass,
3195};
3196
3197static int __init init_iommu_sysfs(void)
3198{
3199 int error;
3200
3201 error = sysdev_class_register(&iommu_sysclass);
3202 if (error)
3203 return error;
3204
3205 error = sysdev_register(&device_iommu);
3206 if (error)
3207 sysdev_class_unregister(&iommu_sysclass);
3208
3209 return error;
3210}
3211
3212#else
3213static int __init init_iommu_sysfs(void)
3214{
3215 return 0;
3216}
3217#endif /* CONFIG_PM */
3218
99dcaded
FY		 3219/*
3220 * Here we only respond to action of unbound device from driver.
3221 *
3222 * Added device is not attached to its DMAR domain here yet. That will happen
3223 * when mapping the device to iova.
3224 */
3225static int device_notifier(struct notifier_block *nb,
3226 unsigned long action, void *data)
3227{
3228 struct device *dev = data;
3229 struct pci_dev *pdev = to_pci_dev(dev);
3230 struct dmar_domain *domain;
3231
44cd613c
DW		 3232 if (iommu_no_mapping(dev))
3233 return 0;
3234
99dcaded
FY		 3235 domain = find_domain(pdev);
3236 if (!domain)
3237 return 0;
3238
3239 if (action == BUS_NOTIFY_UNBOUND_DRIVER && !iommu_pass_through)
3240 domain_remove_one_dev_info(domain, pdev);
3241
3242 return 0;
3243}
3244
3245static struct notifier_block device_nb = {
3246 .notifier_call = device_notifier,
3247};
3248
ba395927
KA		 3249int __init intel_iommu_init(void)
3250{
3251 int ret = 0;
a59b50e9 3252 int force_on = 0;
ba395927 3253
a59b50e9
JC		 3254 /* VT-d is required for a TXT/tboot launch, so enforce that */
3255 force_on = tboot_force_iommu();
3256
3257 if (dmar_table_init()) {
3258 if (force_on)
3259 panic("tboot: Failed to initialize DMAR table\n");
ba395927 3260 return -ENODEV;
a59b50e9 3261 }
ba395927 3262
a59b50e9
JC		 3263 if (dmar_dev_scope_init()) {
3264 if (force_on)
3265 panic("tboot: Failed to initialize DMAR device scope\n");
1886e8a9 3266 return -ENODEV;
a59b50e9 3267 }
1886e8a9 3268
2ae21010
SS		 3269 /*
3270 * Check the need for DMA-remapping initialization now.
3271 * Above initialization will also be used by Interrupt-remapping.
3272 */
75f1cdf1 3273 if (no_iommu || dmar_disabled)
2ae21010
SS		 3274 return -ENODEV;
3275
ba395927
KA		 3276 iommu_init_mempool();
3277 dmar_init_reserved_ranges();
3278
3279 init_no_remapping_devices();
3280
3281 ret = init_dmars();
3282 if (ret) {
a59b50e9
JC		 3283 if (force_on)
3284 panic("tboot: Failed to initialize DMARs\n");
ba395927
KA		 3285 printk(KERN_ERR "IOMMU: dmar init failed\n");
3286 put_iova_domain(&reserved_iova_list);
3287 iommu_exit_mempool();
3288 return ret;
3289 }
3290 printk(KERN_INFO
3291 "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
3292
5e0d2a6f 3293 init_timer(&unmap_timer);
75f1cdf1
FT		 3294#ifdef CONFIG_SWIOTLB
3295 swiotlb = 0;
3296#endif
19943b0e 3297 dma_ops = &intel_dma_ops;
4ed0d3e6 3298
f59c7b69 3299 init_iommu_sysfs();
a8bcbb0d
JR		 3300
3301 register_iommu(&intel_iommu_ops);
3302
99dcaded
FY		 3303 bus_register_notifier(&pci_bus_type, &device_nb);
3304
ba395927
KA		 3305 return 0;
3306}
e820482c 3307
3199aa6b
HW		 3308static void iommu_detach_dependent_devices(struct intel_iommu *iommu,
3309 struct pci_dev *pdev)
3310{
3311 struct pci_dev *tmp, *parent;
3312
3313 if (!iommu || !pdev)
3314 return;
3315
3316 /* dependent device detach */
3317 tmp = pci_find_upstream_pcie_bridge(pdev);
3318 /* Secondary interface's bus number and devfn 0 */
3319 if (tmp) {
3320 parent = pdev->bus->self;
3321 while (parent != tmp) {
3322 iommu_detach_dev(iommu, parent->bus->number,
276dbf99 3323 parent->devfn);
3199aa6b
HW		 3324 parent = parent->bus->self;
3325 }
45e829ea 3326 if (pci_is_pcie(tmp)) /* this is a PCIe-to-PCI bridge */
3199aa6b
HW		 3327 iommu_detach_dev(iommu,
3328 tmp->subordinate->number, 0);
3329 else /* this is a legacy PCI bridge */
276dbf99
DW		 3330 iommu_detach_dev(iommu, tmp->bus->number,
3331 tmp->devfn);
3199aa6b
HW		 3332 }
3333}
3334
2c2e2c38 3335static void domain_remove_one_dev_info(struct dmar_domain *domain,
c7151a8d
WH		 3336 struct pci_dev *pdev)
3337{
3338 struct device_domain_info *info;
3339 struct intel_iommu *iommu;
3340 unsigned long flags;
3341 int found = 0;
3342 struct list_head *entry, *tmp;
3343
276dbf99
DW		 3344 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3345 pdev->devfn);
c7151a8d
WH		 3346 if (!iommu)
3347 return;
3348
3349 spin_lock_irqsave(&device_domain_lock, flags);
3350 list_for_each_safe(entry, tmp, &domain->devices) {
3351 info = list_entry(entry, struct device_domain_info, link);
276dbf99 3352 /* No need to compare PCI domain; it has to be the same */
c7151a8d
WH		 3353 if (info->bus == pdev->bus->number &&
3354 info->devfn == pdev->devfn) {
3355 list_del(&info->link);
3356 list_del(&info->global);
3357 if (info->dev)
3358 info->dev->dev.archdata.iommu = NULL;
3359 spin_unlock_irqrestore(&device_domain_lock, flags);
3360
93a23a72 3361 iommu_disable_dev_iotlb(info);
c7151a8d 3362 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 3363 iommu_detach_dependent_devices(iommu, pdev);
c7151a8d
WH		 3364 free_devinfo_mem(info);
3365
3366 spin_lock_irqsave(&device_domain_lock, flags);
3367
3368 if (found)
3369 break;
3370 else
3371 continue;
3372 }
3373
3374 /* if there is no other devices under the same iommu
3375 * owned by this domain, clear this iommu in iommu_bmp
3376 * update iommu count and coherency
3377 */
276dbf99
DW		 3378 if (iommu == device_to_iommu(info->segment, info->bus,
3379 info->devfn))
c7151a8d
WH		 3380 found = 1;
3381 }
3382
3383 if (found == 0) {
3384 unsigned long tmp_flags;
3385 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
3386 clear_bit(iommu->seq_id, &domain->iommu_bmp);
3387 domain->iommu_count--;
58c610bd 3388 domain_update_iommu_cap(domain);
c7151a8d
WH		 3389 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
3390 }
3391
3392 spin_unlock_irqrestore(&device_domain_lock, flags);
3393}
3394
3395static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
3396{
3397 struct device_domain_info *info;
3398 struct intel_iommu *iommu;
3399 unsigned long flags1, flags2;
3400
3401 spin_lock_irqsave(&device_domain_lock, flags1);
3402 while (!list_empty(&domain->devices)) {
3403 info = list_entry(domain->devices.next,
3404 struct device_domain_info, link);
3405 list_del(&info->link);
3406 list_del(&info->global);
3407 if (info->dev)
3408 info->dev->dev.archdata.iommu = NULL;
3409
3410 spin_unlock_irqrestore(&device_domain_lock, flags1);
3411
93a23a72 3412 iommu_disable_dev_iotlb(info);
276dbf99 3413 iommu = device_to_iommu(info->segment, info->bus, info->devfn);
c7151a8d 3414 iommu_detach_dev(iommu, info->bus, info->devfn);
3199aa6b 3415 iommu_detach_dependent_devices(iommu, info->dev);
c7151a8d
WH		 3416
3417 /* clear this iommu in iommu_bmp, update iommu count
58c610bd 3418 * and capabilities
c7151a8d
WH		 3419 */
3420 spin_lock_irqsave(&domain->iommu_lock, flags2);
3421 if (test_and_clear_bit(iommu->seq_id,
3422 &domain->iommu_bmp)) {
3423 domain->iommu_count--;
58c610bd 3424 domain_update_iommu_cap(domain);
c7151a8d
WH		 3425 }
3426 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
3427
3428 free_devinfo_mem(info);
3429 spin_lock_irqsave(&device_domain_lock, flags1);
3430 }
3431 spin_unlock_irqrestore(&device_domain_lock, flags1);
3432}
3433
5e98c4b1
WH		 3434/* domain id for virtual machine, it won't be set in context */
3435static unsigned long vm_domid;
3436
3437static struct dmar_domain *iommu_alloc_vm_domain(void)
3438{
3439 struct dmar_domain *domain;
3440
3441 domain = alloc_domain_mem();
3442 if (!domain)
3443 return NULL;
3444
3445 domain->id = vm_domid++;
4c923d47 3446 domain->nid = -1;
5e98c4b1
WH		 3447 memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
3448 domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
3449
3450 return domain;
3451}
3452
2c2e2c38 3453static int md_domain_init(struct dmar_domain *domain, int guest_width)
5e98c4b1
WH		 3454{
3455 int adjust_width;
3456
3457 init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
5e98c4b1
WH		 3458 spin_lock_init(&domain->iommu_lock);
3459
3460 domain_reserve_special_ranges(domain);
3461
3462 /* calculate AGAW */
3463 domain->gaw = guest_width;
3464 adjust_width = guestwidth_to_adjustwidth(guest_width);
3465 domain->agaw = width_to_agaw(adjust_width);
3466
3467 INIT_LIST_HEAD(&domain->devices);
3468
3469 domain->iommu_count = 0;
3470 domain->iommu_coherency = 0;
c5b15255 3471 domain->iommu_snooping = 0;
fe40f1e0 3472 domain->max_addr = 0;
4c923d47 3473 domain->nid = -1;
5e98c4b1
WH		 3474
3475 /* always allocate the top pgd */
4c923d47 3476 domain->pgd = (struct dma_pte *)alloc_pgtable_page(domain->nid);
5e98c4b1
WH		 3477 if (!domain->pgd)
3478 return -ENOMEM;
3479 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3480 return 0;
3481}
3482
3483static void iommu_free_vm_domain(struct dmar_domain *domain)
3484{
3485 unsigned long flags;
3486 struct dmar_drhd_unit *drhd;
3487 struct intel_iommu *iommu;
3488 unsigned long i;
3489 unsigned long ndomains;
3490
3491 for_each_drhd_unit(drhd) {
3492 if (drhd->ignored)
3493 continue;
3494 iommu = drhd->iommu;
3495
3496 ndomains = cap_ndoms(iommu->cap);
a45946ab 3497 for_each_set_bit(i, iommu->domain_ids, ndomains) {
5e98c4b1
WH		 3498 if (iommu->domains[i] == domain) {
3499 spin_lock_irqsave(&iommu->lock, flags);
3500 clear_bit(i, iommu->domain_ids);
3501 iommu->domains[i] = NULL;
3502 spin_unlock_irqrestore(&iommu->lock, flags);
3503 break;
3504 }
5e98c4b1
WH		 3505 }
3506 }
3507}
3508
3509static void vm_domain_exit(struct dmar_domain *domain)
3510{
5e98c4b1
WH		 3511 /* Domain 0 is reserved, so dont process it */
3512 if (!domain)
3513 return;
3514
3515 vm_domain_remove_all_dev_info(domain);
3516 /* destroy iovas */
3517 put_iova_domain(&domain->iovad);
5e98c4b1
WH		 3518
3519 /* clear ptes */
595badf5 3520 dma_pte_clear_range(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
5e98c4b1
WH		 3521
3522 /* free page tables */
d794dc9b 3523 dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw));
5e98c4b1
WH		 3524
3525 iommu_free_vm_domain(domain);
3526 free_domain_mem(domain);
3527}
3528
5d450806 3529static int intel_iommu_domain_init(struct iommu_domain *domain)
38717946 3530{
5d450806 3531 struct dmar_domain *dmar_domain;
38717946 3532
5d450806
JR		 3533 dmar_domain = iommu_alloc_vm_domain();
3534 if (!dmar_domain) {
38717946 3535 printk(KERN_ERR
5d450806
JR		 3536 "intel_iommu_domain_init: dmar_domain == NULL\n");
3537 return -ENOMEM;
38717946 3538 }
2c2e2c38 3539 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
38717946 3540 printk(KERN_ERR
5d450806
JR		 3541 "intel_iommu_domain_init() failed\n");
3542 vm_domain_exit(dmar_domain);
3543 return -ENOMEM;
38717946 3544 }
5d450806 3545 domain->priv = dmar_domain;
faa3d6f5 3546
5d450806 3547 return 0;
38717946 3548}
38717946 3549
5d450806 3550static void intel_iommu_domain_destroy(struct iommu_domain *domain)
38717946 3551{
5d450806
JR		 3552 struct dmar_domain *dmar_domain = domain->priv;
3553
3554 domain->priv = NULL;
3555 vm_domain_exit(dmar_domain);
38717946 3556}
38717946 3557
4c5478c9
JR		 3558static int intel_iommu_attach_device(struct iommu_domain *domain,
3559 struct device *dev)
38717946 3560{
4c5478c9
JR		 3561 struct dmar_domain *dmar_domain = domain->priv;
3562 struct pci_dev *pdev = to_pci_dev(dev);
fe40f1e0
WH		 3563 struct intel_iommu *iommu;
3564 int addr_width;
faa3d6f5
WH		 3565
3566 /* normally pdev is not mapped */
3567 if (unlikely(domain_context_mapped(pdev))) {
3568 struct dmar_domain *old_domain;
3569
3570 old_domain = find_domain(pdev);
3571 if (old_domain) {
2c2e2c38
FY		 3572 if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE ||
3573 dmar_domain->flags & DOMAIN_FLAG_STATIC_IDENTITY)
3574 domain_remove_one_dev_info(old_domain, pdev);
faa3d6f5
WH		 3575 else
3576 domain_remove_dev_info(old_domain);
3577 }
3578 }
3579
276dbf99
DW		 3580 iommu = device_to_iommu(pci_domain_nr(pdev->bus), pdev->bus->number,
3581 pdev->devfn);
fe40f1e0
WH		 3582 if (!iommu)
3583 return -ENODEV;
3584
3585 /* check if this iommu agaw is sufficient for max mapped address */
3586 addr_width = agaw_to_width(iommu->agaw);
a99c47a2
TL		 3587 if (addr_width > cap_mgaw(iommu->cap))
3588 addr_width = cap_mgaw(iommu->cap);
3589
3590 if (dmar_domain->max_addr > (1LL << addr_width)) {
3591 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 3592 "sufficient for the mapped address (%llx)\n",
a99c47a2 3593 __func__, addr_width, dmar_domain->max_addr);
fe40f1e0
WH		 3594 return -EFAULT;
3595 }
a99c47a2
TL		 3596 dmar_domain->gaw = addr_width;
3597
3598 /*
3599 * Knock out extra levels of page tables if necessary
3600 */
3601 while (iommu->agaw < dmar_domain->agaw) {
3602 struct dma_pte *pte;
3603
3604 pte = dmar_domain->pgd;
3605 if (dma_pte_present(pte)) {
3606 free_pgtable_page(dmar_domain->pgd);
25cbff16
SY		 3607 dmar_domain->pgd = (struct dma_pte *)
3608 phys_to_virt(dma_pte_addr(pte));
a99c47a2
TL		 3609 }
3610 dmar_domain->agaw--;
3611 }
fe40f1e0 3612
5fe60f4e 3613 return domain_add_dev_info(dmar_domain, pdev, CONTEXT_TT_MULTI_LEVEL);
38717946 3614}
38717946 3615
4c5478c9
JR		 3616static void intel_iommu_detach_device(struct iommu_domain *domain,
3617 struct device *dev)
38717946 3618{
4c5478c9
JR		 3619 struct dmar_domain *dmar_domain = domain->priv;
3620 struct pci_dev *pdev = to_pci_dev(dev);
3621
2c2e2c38 3622 domain_remove_one_dev_info(dmar_domain, pdev);
faa3d6f5 3623}
c7151a8d 3624
b146a1c9
JR		 3625static int intel_iommu_map(struct iommu_domain *domain,
3626 unsigned long iova, phys_addr_t hpa,
3627 int gfp_order, int iommu_prot)
faa3d6f5 3628{
dde57a21 3629 struct dmar_domain *dmar_domain = domain->priv;
fe40f1e0 3630 u64 max_addr;
dde57a21 3631 int prot = 0;
b146a1c9 3632 size_t size;
faa3d6f5 3633 int ret;
fe40f1e0 3634
dde57a21
JR		 3635 if (iommu_prot & IOMMU_READ)
3636 prot |= DMA_PTE_READ;
3637 if (iommu_prot & IOMMU_WRITE)
3638 prot |= DMA_PTE_WRITE;
9cf06697
SY		 3639 if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)
3640 prot |= DMA_PTE_SNP;
dde57a21 3641
b146a1c9 3642 size = PAGE_SIZE << gfp_order;
163cc52c 3643 max_addr = iova + size;
dde57a21 3644 if (dmar_domain->max_addr < max_addr) {
fe40f1e0
WH		 3645 u64 end;
3646
3647 /* check if minimum agaw is sufficient for mapped address */
8954da1f 3648 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
fe40f1e0 3649 if (end < max_addr) {
8954da1f 3650 printk(KERN_ERR "%s: iommu width (%d) is not "
fe40f1e0 3651 "sufficient for the mapped address (%llx)\n",
8954da1f 3652 __func__, dmar_domain->gaw, max_addr);
fe40f1e0
WH		 3653 return -EFAULT;
3654 }
dde57a21 3655 dmar_domain->max_addr = max_addr;
fe40f1e0 3656 }
ad051221
DW		 3657 /* Round up size to next multiple of PAGE_SIZE, if it and
3658 the low bits of hpa would take us onto the next page */
88cb6a74 3659 size = aligned_nrpages(hpa, size);
ad051221
DW		 3660 ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
3661 hpa >> VTD_PAGE_SHIFT, size, prot);
faa3d6f5 3662 return ret;
38717946 3663}
38717946 3664
b146a1c9
JR		 3665static int intel_iommu_unmap(struct iommu_domain *domain,
3666 unsigned long iova, int gfp_order)
38717946 3667{
dde57a21 3668 struct dmar_domain *dmar_domain = domain->priv;
b146a1c9 3669 size_t size = PAGE_SIZE << gfp_order;
4b99d352 3670
163cc52c
DW		 3671 dma_pte_clear_range(dmar_domain, iova >> VTD_PAGE_SHIFT,
3672 (iova + size - 1) >> VTD_PAGE_SHIFT);
fe40f1e0 3673
163cc52c
DW		 3674 if (dmar_domain->max_addr == iova + size)
3675 dmar_domain->max_addr = iova;
b146a1c9
JR		 3676
3677 return gfp_order;
38717946 3678}
38717946 3679
d14d6577
JR		 3680static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
3681 unsigned long iova)
38717946 3682{
d14d6577 3683 struct dmar_domain *dmar_domain = domain->priv;
38717946 3684 struct dma_pte *pte;
faa3d6f5 3685 u64 phys = 0;
38717946 3686
b026fd28 3687 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT);
38717946 3688 if (pte)
faa3d6f5 3689 phys = dma_pte_addr(pte);
38717946 3690
faa3d6f5 3691 return phys;
38717946 3692}
a8bcbb0d 3693
dbb9fd86
SY		 3694static int intel_iommu_domain_has_cap(struct iommu_domain *domain,
3695 unsigned long cap)
3696{
3697 struct dmar_domain *dmar_domain = domain->priv;
3698
3699 if (cap == IOMMU_CAP_CACHE_COHERENCY)
3700 return dmar_domain->iommu_snooping;
3701
3702 return 0;
3703}
3704
a8bcbb0d
JR		 3705static struct iommu_ops intel_iommu_ops = {
3706 .domain_init = intel_iommu_domain_init,
3707 .domain_destroy = intel_iommu_domain_destroy,
3708 .attach_dev = intel_iommu_attach_device,
3709 .detach_dev = intel_iommu_detach_device,
b146a1c9
JR		 3710 .map = intel_iommu_map,
3711 .unmap = intel_iommu_unmap,
a8bcbb0d 3712 .iova_to_phys = intel_iommu_iova_to_phys,
dbb9fd86 3713 .domain_has_cap = intel_iommu_domain_has_cap,
a8bcbb0d 3714};
9af88143
DW		 3715
3716static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
3717{
3718 /*
3719 * Mobile 4 Series Chipset neglects to set RWBF capability,
3720 * but needs it:
3721 */
3722 printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
3723 rwbf_quirk = 1;
2d9e667e
DW		 3724
3725 /* https://bugzilla.redhat.com/show_bug.cgi?id=538163 */
3726 if (dev->revision == 0x07) {
3727 printk(KERN_INFO "DMAR: Disabling IOMMU for graphics on this chipset\n");
3728 dmar_map_gfx = 0;
3729 }
9af88143
DW		 3730}
3731
3732DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
e0fc7e0b
DW		 3733
3734/* On Tylersburg chipsets, some BIOSes have been known to enable the
3735 ISOCH DMAR unit for the Azalia sound device, but not give it any
3736 TLB entries, which causes it to deadlock. Check for that. We do
3737 this in a function called from init_dmars(), instead of in a PCI
3738 quirk, because we don't want to print the obnoxious "BIOS broken"
3739 message if VT-d is actually disabled.
3740*/
3741static void __init check_tylersburg_isoch(void)
3742{
3743 struct pci_dev *pdev;
3744 uint32_t vtisochctrl;
3745
3746 /* If there's no Azalia in the system anyway, forget it. */
3747 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
3748 if (!pdev)
3749 return;
3750 pci_dev_put(pdev);
3751
3752 /* System Management Registers. Might be hidden, in which case
3753 we can't do the sanity check. But that's OK, because the
3754 known-broken BIOSes _don't_ actually hide it, so far. */
3755 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
3756 if (!pdev)
3757 return;
3758
3759 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
3760 pci_dev_put(pdev);
3761 return;
3762 }
3763
3764 pci_dev_put(pdev);
3765
3766 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
3767 if (vtisochctrl & 1)
3768 return;
3769
3770 /* Drop all bits other than the number of TLB entries */
3771 vtisochctrl &= 0x1c;
3772
3773 /* If we have the recommended number of TLB entries (16), fine. */
3774 if (vtisochctrl == 0x10)
3775 return;
3776
3777 /* Zero TLB entries? You get to ride the short bus to school. */
3778 if (!vtisochctrl) {
3779 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
3780 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
3781 dmi_get_system_info(DMI_BIOS_VENDOR),
3782 dmi_get_system_info(DMI_BIOS_VERSION),
3783 dmi_get_system_info(DMI_PRODUCT_VERSION));
3784 iommu_identity_mapping |= IDENTMAP_AZALIA;
3785 return;
3786 }
3787
3788 printk(KERN_WARNING "DMAR: Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
3789 vtisochctrl);
3790}
Clone of davem's net-next-2.6 tree