drivers/pci/intel-iommu.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  * Author: Fenghua Yu <fenghua.yu@intel.com>
  22  */
  23
  24 #include <linux/init.h>
  25 #include <linux/bitmap.h>
  26 #include <linux/debugfs.h>
  27 #include <linux/slab.h>
  28 #include <linux/irq.h>
  29 #include <linux/interrupt.h>
  30 #include <linux/spinlock.h>
  31 #include <linux/pci.h>
  32 #include <linux/dmar.h>
  33 #include <linux/dma-mapping.h>
  34 #include <linux/mempool.h>
  35 #include <linux/timer.h>
  36 #include <linux/iova.h>
  37 #include <linux/iommu.h>
  38 #include <linux/intel-iommu.h>
  39 #include <asm/cacheflush.h>
  40 #include <asm/iommu.h>
  41 #include "pci.h"
  42
  43 #define ROOT_SIZE               VTD_PAGE_SIZE
  44 #define CONTEXT_SIZE            VTD_PAGE_SIZE
  45
  46 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  47 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  48
  49 #define IOAPIC_RANGE_START      (0xfee00000)
  50 #define IOAPIC_RANGE_END        (0xfeefffff)
  51 #define IOVA_START_ADDR         (0x1000)
  52
  53 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  54
  55 #define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1)
  56
  57 #define IOVA_PFN(addr)          ((addr) >> PAGE_SHIFT)
  58 #define DMA_32BIT_PFN           IOVA_PFN(DMA_32BIT_MASK)
  59 #define DMA_64BIT_PFN           IOVA_PFN(DMA_64BIT_MASK)
  60
  61 /* global iommu list, set NULL for ignored DMAR units */
  62 static struct intel_iommu **g_iommus;
  63
  64 static int rwbf_quirk;
  65
  66 /*
  67  * 0: Present
  68  * 1-11: Reserved
  69  * 12-63: Context Ptr (12 - (haw-1))
  70  * 64-127: Reserved
  71  */
  72 struct root_entry {
  73         u64     val;
  74         u64     rsvd1;
  75 };
  76 #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
  77 static inline bool root_present(struct root_entry *root)
  78 {
  79         return (root->val & 1);
  80 }
  81 static inline void set_root_present(struct root_entry *root)
  82 {
  83         root->val |= 1;
  84 }
  85 static inline void set_root_value(struct root_entry *root, unsigned long value)
  86 {
  87         root->val |= value & VTD_PAGE_MASK;
  88 }
  89
  90 static inline struct context_entry *
  91 get_context_addr_from_root(struct root_entry *root)
  92 {
  93         return (struct context_entry *)
  94                 (root_present(root)?phys_to_virt(
  95                 root->val & VTD_PAGE_MASK) :
  96                 NULL);
  97 }
  98
  99 /*
 100  * low 64 bits:
 101  * 0: present
 102  * 1: fault processing disable
 103  * 2-3: translation type
 104  * 12-63: address space root
 105  * high 64 bits:
 106  * 0-2: address width
 107  * 3-6: aval
 108  * 8-23: domain id
 109  */
 110 struct context_entry {
 111         u64 lo;
 112         u64 hi;
 113 };
 114
 115 static inline bool context_present(struct context_entry *context)
 116 {
 117         return (context->lo & 1);
 118 }
 119 static inline void context_set_present(struct context_entry *context)
 120 {
 121         context->lo |= 1;
 122 }
 123
 124 static inline void context_set_fault_enable(struct context_entry *context)
 125 {
 126         context->lo &= (((u64)-1) << 2) | 1;
 127 }
 128
 129 #define CONTEXT_TT_MULTI_LEVEL 0
 130
 131 static inline void context_set_translation_type(struct context_entry *context,
 132                                                 unsigned long value)
 133 {
 134         context->lo &= (((u64)-1) << 4) | 3;
 135         context->lo |= (value & 3) << 2;
 136 }
 137
 138 static inline void context_set_address_root(struct context_entry *context,
 139                                             unsigned long value)
 140 {
 141         context->lo |= value & VTD_PAGE_MASK;
 142 }
 143
 144 static inline void context_set_address_width(struct context_entry *context,
 145                                              unsigned long value)
 146 {
 147         context->hi |= value & 7;
 148 }
 149
 150 static inline void context_set_domain_id(struct context_entry *context,
 151                                          unsigned long value)
 152 {
 153         context->hi |= (value & ((1 << 16) - 1)) << 8;
 154 }
 155
 156 static inline void context_clear_entry(struct context_entry *context)
 157 {
 158         context->lo = 0;
 159         context->hi = 0;
 160 }
 161
 162 /*
 163  * 0: readable
 164  * 1: writable
 165  * 2-6: reserved
 166  * 7: super page
 167  * 8-11: available
 168  * 12-63: Host physcial address
 169  */
 170 struct dma_pte {
 171         u64 val;
 172 };
 173
 174 static inline void dma_clear_pte(struct dma_pte *pte)
 175 {
 176         pte->val = 0;
 177 }
 178
 179 static inline void dma_set_pte_readable(struct dma_pte *pte)
 180 {
 181         pte->val |= DMA_PTE_READ;
 182 }
 183
 184 static inline void dma_set_pte_writable(struct dma_pte *pte)
 185 {
 186         pte->val |= DMA_PTE_WRITE;
 187 }
 188
 189 static inline void dma_set_pte_prot(struct dma_pte *pte, unsigned long prot)
 190 {
 191         pte->val = (pte->val & ~3) | (prot & 3);
 192 }
 193
 194 static inline u64 dma_pte_addr(struct dma_pte *pte)
 195 {
 196         return (pte->val & VTD_PAGE_MASK);
 197 }
 198
 199 static inline void dma_set_pte_addr(struct dma_pte *pte, u64 addr)
 200 {
 201         pte->val |= (addr & VTD_PAGE_MASK);
 202 }
 203
 204 static inline bool dma_pte_present(struct dma_pte *pte)
 205 {
 206         return (pte->val & 3) != 0;
 207 }
 208
 209 /* devices under the same p2p bridge are owned in one domain */
 210 #define DOMAIN_FLAG_P2P_MULTIPLE_DEVICES (1 << 0)
 211
 212 /* domain represents a virtual machine, more than one devices
 213  * across iommus may be owned in one domain, e.g. kvm guest.
 214  */
 215 #define DOMAIN_FLAG_VIRTUAL_MACHINE     (1 << 1)
 216
 217 struct dmar_domain {
 218         int     id;                     /* domain id */
 219         unsigned long iommu_bmp;        /* bitmap of iommus this domain uses*/
 220
 221         struct list_head devices;       /* all devices' list */
 222         struct iova_domain iovad;       /* iova's that belong to this domain */
 223
 224         struct dma_pte  *pgd;           /* virtual address */
 225         spinlock_t      mapping_lock;   /* page table lock */
 226         int             gaw;            /* max guest address width */
 227
 228         /* adjusted guest address width, 0 is level 2 30-bit */
 229         int             agaw;
 230
 231         int             flags;          /* flags to find out type of domain */
 232
 233         int             iommu_coherency;/* indicate coherency of iommu access */
 234         int             iommu_count;    /* reference count of iommu */
 235         spinlock_t      iommu_lock;     /* protect iommu set in domain */
 236         u64             max_addr;       /* maximum mapped address */
 237 };
 238
 239 /* PCI domain-device relationship */
 240 struct device_domain_info {
 241         struct list_head link;  /* link to domain siblings */
 242         struct list_head global; /* link to global list */
 243         u8 bus;                 /* PCI bus numer */
 244         u8 devfn;               /* PCI devfn number */
 245         struct pci_dev *dev; /* it's NULL for PCIE-to-PCI bridge */
 246         struct dmar_domain *domain; /* pointer to domain */
 247 };
 248
 249 static void flush_unmaps_timeout(unsigned long data);
 250
 251 DEFINE_TIMER(unmap_timer,  flush_unmaps_timeout, 0, 0);
 252
 253 #define HIGH_WATER_MARK 250
 254 struct deferred_flush_tables {
 255         int next;
 256         struct iova *iova[HIGH_WATER_MARK];
 257         struct dmar_domain *domain[HIGH_WATER_MARK];
 258 };
 259
 260 static struct deferred_flush_tables *deferred_flush;
 261
 262 /* bitmap for indexing intel_iommus */
 263 static int g_num_of_iommus;
 264
 265 static DEFINE_SPINLOCK(async_umap_flush_lock);
 266 static LIST_HEAD(unmaps_to_do);
 267
 268 static int timer_on;
 269 static long list_size;
 270
 271 static void domain_remove_dev_info(struct dmar_domain *domain);
 272
 273 #ifdef CONFIG_DMAR_DEFAULT_ON
 274 int dmar_disabled = 0;
 275 #else
 276 int dmar_disabled = 1;
 277 #endif /*CONFIG_DMAR_DEFAULT_ON*/
 278
 279 static int __initdata dmar_map_gfx = 1;
 280 static int dmar_forcedac;
 281 static int intel_iommu_strict;
 282
 283 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
 284 static DEFINE_SPINLOCK(device_domain_lock);
 285 static LIST_HEAD(device_domain_list);
 286
 287 static struct iommu_ops intel_iommu_ops;
 288
 289 static int __init intel_iommu_setup(char *str)
 290 {
 291         if (!str)
 292                 return -EINVAL;
 293         while (*str) {
 294                 if (!strncmp(str, "on", 2)) {
 295                         dmar_disabled = 0;
 296                         printk(KERN_INFO "Intel-IOMMU: enabled\n");
 297                 } else if (!strncmp(str, "off", 3)) {
 298                         dmar_disabled = 1;
 299                         printk(KERN_INFO "Intel-IOMMU: disabled\n");
 300                 } else if (!strncmp(str, "igfx_off", 8)) {
 301                         dmar_map_gfx = 0;
 302                         printk(KERN_INFO
 303                                 "Intel-IOMMU: disable GFX device mapping\n");
 304                 } else if (!strncmp(str, "forcedac", 8)) {
 305                         printk(KERN_INFO
 306                                 "Intel-IOMMU: Forcing DAC for PCI devices\n");
 307                         dmar_forcedac = 1;
 308                 } else if (!strncmp(str, "strict", 6)) {
 309                         printk(KERN_INFO
 310                                 "Intel-IOMMU: disable batched IOTLB flush\n");
 311                         intel_iommu_strict = 1;
 312                 }
 313
 314                 str += strcspn(str, ",");
 315                 while (*str == ',')
 316                         str++;
 317         }
 318         return 0;
 319 }
 320 __setup("intel_iommu=", intel_iommu_setup);
 321
 322 static struct kmem_cache *iommu_domain_cache;
 323 static struct kmem_cache *iommu_devinfo_cache;
 324 static struct kmem_cache *iommu_iova_cache;
 325
 326 static inline void *iommu_kmem_cache_alloc(struct kmem_cache *cachep)
 327 {
 328         unsigned int flags;
 329         void *vaddr;
 330
 331         /* trying to avoid low memory issues */
 332         flags = current->flags & PF_MEMALLOC;
 333         current->flags |= PF_MEMALLOC;
 334         vaddr = kmem_cache_alloc(cachep, GFP_ATOMIC);
 335         current->flags &= (~PF_MEMALLOC | flags);
 336         return vaddr;
 337 }
 338
 339
 340 static inline void *alloc_pgtable_page(void)
 341 {
 342         unsigned int flags;
 343         void *vaddr;
 344
 345         /* trying to avoid low memory issues */
 346         flags = current->flags & PF_MEMALLOC;
 347         current->flags |= PF_MEMALLOC;
 348         vaddr = (void *)get_zeroed_page(GFP_ATOMIC);
 349         current->flags &= (~PF_MEMALLOC | flags);
 350         return vaddr;
 351 }
 352
 353 static inline void free_pgtable_page(void *vaddr)
 354 {
 355         free_page((unsigned long)vaddr);
 356 }
 357
 358 static inline void *alloc_domain_mem(void)
 359 {
 360         return iommu_kmem_cache_alloc(iommu_domain_cache);
 361 }
 362
 363 static void free_domain_mem(void *vaddr)
 364 {
 365         kmem_cache_free(iommu_domain_cache, vaddr);
 366 }
 367
 368 static inline void * alloc_devinfo_mem(void)
 369 {
 370         return iommu_kmem_cache_alloc(iommu_devinfo_cache);
 371 }
 372
 373 static inline void free_devinfo_mem(void *vaddr)
 374 {
 375         kmem_cache_free(iommu_devinfo_cache, vaddr);
 376 }
 377
 378 struct iova *alloc_iova_mem(void)
 379 {
 380         return iommu_kmem_cache_alloc(iommu_iova_cache);
 381 }
 382
 383 void free_iova_mem(struct iova *iova)
 384 {
 385         kmem_cache_free(iommu_iova_cache, iova);
 386 }
 387
 388
 389 static inline int width_to_agaw(int width);
 390
 391 /* calculate agaw for each iommu.
 392  * "SAGAW" may be different across iommus, use a default agaw, and
 393  * get a supported less agaw for iommus that don't support the default agaw.
 394  */
 395 int iommu_calculate_agaw(struct intel_iommu *iommu)
 396 {
 397         unsigned long sagaw;
 398         int agaw = -1;
 399
 400         sagaw = cap_sagaw(iommu->cap);
 401         for (agaw = width_to_agaw(DEFAULT_DOMAIN_ADDRESS_WIDTH);
 402              agaw >= 0; agaw--) {
 403                 if (test_bit(agaw, &sagaw))
 404                         break;
 405         }
 406
 407         return agaw;
 408 }
 409
 410 /* in native case, each domain is related to only one iommu */
 411 static struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
 412 {
 413         int iommu_id;
 414
 415         BUG_ON(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE);
 416
 417         iommu_id = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
 418         if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
 419                 return NULL;
 420
 421         return g_iommus[iommu_id];
 422 }
 423
 424 /* "Coherency" capability may be different across iommus */
 425 static void domain_update_iommu_coherency(struct dmar_domain *domain)
 426 {
 427         int i;
 428
 429         domain->iommu_coherency = 1;
 430
 431         i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
 432         for (; i < g_num_of_iommus; ) {
 433                 if (!ecap_coherent(g_iommus[i]->ecap)) {
 434                         domain->iommu_coherency = 0;
 435                         break;
 436                 }
 437                 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
 438         }
 439 }
 440
 441 static struct intel_iommu *device_to_iommu(u8 bus, u8 devfn)
 442 {
 443         struct dmar_drhd_unit *drhd = NULL;
 444         int i;
 445
 446         for_each_drhd_unit(drhd) {
 447                 if (drhd->ignored)
 448                         continue;
 449
 450                 for (i = 0; i < drhd->devices_cnt; i++)
 451                         if (drhd->devices[i] &&
 452                             drhd->devices[i]->bus->number == bus &&
 453                             drhd->devices[i]->devfn == devfn)
 454                                 return drhd->iommu;
 455
 456                 if (drhd->include_all)
 457                         return drhd->iommu;
 458         }
 459
 460         return NULL;
 461 }
 462
 463 static void domain_flush_cache(struct dmar_domain *domain,
 464                                void *addr, int size)
 465 {
 466         if (!domain->iommu_coherency)
 467                 clflush_cache_range(addr, size);
 468 }
 469
 470 /* Gets context entry for a given bus and devfn */
 471 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
 472                 u8 bus, u8 devfn)
 473 {
 474         struct root_entry *root;
 475         struct context_entry *context;
 476         unsigned long phy_addr;
 477         unsigned long flags;
 478
 479         spin_lock_irqsave(&iommu->lock, flags);
 480         root = &iommu->root_entry[bus];
 481         context = get_context_addr_from_root(root);
 482         if (!context) {
 483                 context = (struct context_entry *)alloc_pgtable_page();
 484                 if (!context) {
 485                         spin_unlock_irqrestore(&iommu->lock, flags);
 486                         return NULL;
 487                 }
 488                 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
 489                 phy_addr = virt_to_phys((void *)context);
 490                 set_root_value(root, phy_addr);
 491                 set_root_present(root);
 492                 __iommu_flush_cache(iommu, root, sizeof(*root));
 493         }
 494         spin_unlock_irqrestore(&iommu->lock, flags);
 495         return &context[devfn];
 496 }
 497
 498 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 499 {
 500         struct root_entry *root;
 501         struct context_entry *context;
 502         int ret;
 503         unsigned long flags;
 504
 505         spin_lock_irqsave(&iommu->lock, flags);
 506         root = &iommu->root_entry[bus];
 507         context = get_context_addr_from_root(root);
 508         if (!context) {
 509                 ret = 0;
 510                 goto out;
 511         }
 512         ret = context_present(&context[devfn]);
 513 out:
 514         spin_unlock_irqrestore(&iommu->lock, flags);
 515         return ret;
 516 }
 517
 518 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 519 {
 520         struct root_entry *root;
 521         struct context_entry *context;
 522         unsigned long flags;
 523
 524         spin_lock_irqsave(&iommu->lock, flags);
 525         root = &iommu->root_entry[bus];
 526         context = get_context_addr_from_root(root);
 527         if (context) {
 528                 context_clear_entry(&context[devfn]);
 529                 __iommu_flush_cache(iommu, &context[devfn], \
 530                         sizeof(*context));
 531         }
 532         spin_unlock_irqrestore(&iommu->lock, flags);
 533 }
 534
 535 static void free_context_table(struct intel_iommu *iommu)
 536 {
 537         struct root_entry *root;
 538         int i;
 539         unsigned long flags;
 540         struct context_entry *context;
 541
 542         spin_lock_irqsave(&iommu->lock, flags);
 543         if (!iommu->root_entry) {
 544                 goto out;
 545         }
 546         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 547                 root = &iommu->root_entry[i];
 548                 context = get_context_addr_from_root(root);
 549                 if (context)
 550                         free_pgtable_page(context);
 551         }
 552         free_pgtable_page(iommu->root_entry);
 553         iommu->root_entry = NULL;
 554 out:
 555         spin_unlock_irqrestore(&iommu->lock, flags);
 556 }
 557
 558 /* page table handling */
 559 #define LEVEL_STRIDE            (9)
 560 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
 561
 562 static inline int agaw_to_level(int agaw)
 563 {
 564         return agaw + 2;
 565 }
 566
 567 static inline int agaw_to_width(int agaw)
 568 {
 569         return 30 + agaw * LEVEL_STRIDE;
 570
 571 }
 572
 573 static inline int width_to_agaw(int width)
 574 {
 575         return (width - 30) / LEVEL_STRIDE;
 576 }
 577
 578 static inline unsigned int level_to_offset_bits(int level)
 579 {
 580         return (12 + (level - 1) * LEVEL_STRIDE);
 581 }
 582
 583 static inline int address_level_offset(u64 addr, int level)
 584 {
 585         return ((addr >> level_to_offset_bits(level)) & LEVEL_MASK);
 586 }
 587
 588 static inline u64 level_mask(int level)
 589 {
 590         return ((u64)-1 << level_to_offset_bits(level));
 591 }
 592
 593 static inline u64 level_size(int level)
 594 {
 595         return ((u64)1 << level_to_offset_bits(level));
 596 }
 597
 598 static inline u64 align_to_level(u64 addr, int level)
 599 {
 600         return ((addr + level_size(level) - 1) & level_mask(level));
 601 }
 602
 603 static struct dma_pte * addr_to_dma_pte(struct dmar_domain *domain, u64 addr)
 604 {
 605         int addr_width = agaw_to_width(domain->agaw);
 606         struct dma_pte *parent, *pte = NULL;
 607         int level = agaw_to_level(domain->agaw);
 608         int offset;
 609         unsigned long flags;
 610
 611         BUG_ON(!domain->pgd);
 612
 613         addr &= (((u64)1) << addr_width) - 1;
 614         parent = domain->pgd;
 615
 616         spin_lock_irqsave(&domain->mapping_lock, flags);
 617         while (level > 0) {
 618                 void *tmp_page;
 619
 620                 offset = address_level_offset(addr, level);
 621                 pte = &parent[offset];
 622                 if (level == 1)
 623                         break;
 624
 625                 if (!dma_pte_present(pte)) {
 626                         tmp_page = alloc_pgtable_page();
 627
 628                         if (!tmp_page) {
 629                                 spin_unlock_irqrestore(&domain->mapping_lock,
 630                                         flags);
 631                                 return NULL;
 632                         }
 633                         domain_flush_cache(domain, tmp_page, PAGE_SIZE);
 634                         dma_set_pte_addr(pte, virt_to_phys(tmp_page));
 635                         /*
 636                          * high level table always sets r/w, last level page
 637                          * table control read/write
 638                          */
 639                         dma_set_pte_readable(pte);
 640                         dma_set_pte_writable(pte);
 641                         domain_flush_cache(domain, pte, sizeof(*pte));
 642                 }
 643                 parent = phys_to_virt(dma_pte_addr(pte));
 644                 level--;
 645         }
 646
 647         spin_unlock_irqrestore(&domain->mapping_lock, flags);
 648         return pte;
 649 }
 650
 651 /* return address's pte at specific level */
 652 static struct dma_pte *dma_addr_level_pte(struct dmar_domain *domain, u64 addr,
 653                 int level)
 654 {
 655         struct dma_pte *parent, *pte = NULL;
 656         int total = agaw_to_level(domain->agaw);
 657         int offset;
 658
 659         parent = domain->pgd;
 660         while (level <= total) {
 661                 offset = address_level_offset(addr, total);
 662                 pte = &parent[offset];
 663                 if (level == total)
 664                         return pte;
 665
 666                 if (!dma_pte_present(pte))
 667                         break;
 668                 parent = phys_to_virt(dma_pte_addr(pte));
 669                 total--;
 670         }
 671         return NULL;
 672 }
 673
 674 /* clear one page's page table */
 675 static void dma_pte_clear_one(struct dmar_domain *domain, u64 addr)
 676 {
 677         struct dma_pte *pte = NULL;
 678
 679         /* get last level pte */
 680         pte = dma_addr_level_pte(domain, addr, 1);
 681
 682         if (pte) {
 683                 dma_clear_pte(pte);
 684                 domain_flush_cache(domain, pte, sizeof(*pte));
 685         }
 686 }
 687
 688 /* clear last level pte, a tlb flush should be followed */
 689 static void dma_pte_clear_range(struct dmar_domain *domain, u64 start, u64 end)
 690 {
 691         int addr_width = agaw_to_width(domain->agaw);
 692
 693         start &= (((u64)1) << addr_width) - 1;
 694         end &= (((u64)1) << addr_width) - 1;
 695         /* in case it's partial page */
 696         start = PAGE_ALIGN(start);
 697         end &= PAGE_MASK;
 698
 699         /* we don't need lock here, nobody else touches the iova range */
 700         while (start < end) {
 701                 dma_pte_clear_one(domain, start);
 702                 start += VTD_PAGE_SIZE;
 703         }
 704 }
 705
 706 /* free page table pages. last level pte should already be cleared */
 707 static void dma_pte_free_pagetable(struct dmar_domain *domain,
 708         u64 start, u64 end)
 709 {
 710         int addr_width = agaw_to_width(domain->agaw);
 711         struct dma_pte *pte;
 712         int total = agaw_to_level(domain->agaw);
 713         int level;
 714         u64 tmp;
 715
 716         start &= (((u64)1) << addr_width) - 1;
 717         end &= (((u64)1) << addr_width) - 1;
 718
 719         /* we don't need lock here, nobody else touches the iova range */
 720         level = 2;
 721         while (level <= total) {
 722                 tmp = align_to_level(start, level);
 723                 if (tmp >= end || (tmp + level_size(level) > end))
 724                         return;
 725
 726                 while (tmp < end) {
 727                         pte = dma_addr_level_pte(domain, tmp, level);
 728                         if (pte) {
 729                                 free_pgtable_page(
 730                                         phys_to_virt(dma_pte_addr(pte)));
 731                                 dma_clear_pte(pte);
 732                                 domain_flush_cache(domain, pte, sizeof(*pte));
 733                         }
 734                         tmp += level_size(level);
 735                 }
 736                 level++;
 737         }
 738         /* free pgd */
 739         if (start == 0 && end >= ((((u64)1) << addr_width) - 1)) {
 740                 free_pgtable_page(domain->pgd);
 741                 domain->pgd = NULL;
 742         }
 743 }
 744
 745 /* iommu handling */
 746 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
 747 {
 748         struct root_entry *root;
 749         unsigned long flags;
 750
 751         root = (struct root_entry *)alloc_pgtable_page();
 752         if (!root)
 753                 return -ENOMEM;
 754
 755         __iommu_flush_cache(iommu, root, ROOT_SIZE);
 756
 757         spin_lock_irqsave(&iommu->lock, flags);
 758         iommu->root_entry = root;
 759         spin_unlock_irqrestore(&iommu->lock, flags);
 760
 761         return 0;
 762 }
 763
 764 static void iommu_set_root_entry(struct intel_iommu *iommu)
 765 {
 766         void *addr;
 767         u32 cmd, sts;
 768         unsigned long flag;
 769
 770         addr = iommu->root_entry;
 771
 772         spin_lock_irqsave(&iommu->register_lock, flag);
 773         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
 774
 775         cmd = iommu->gcmd | DMA_GCMD_SRTP;
 776         writel(cmd, iommu->reg + DMAR_GCMD_REG);
 777
 778         /* Make sure hardware complete it */
 779         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 780                 readl, (sts & DMA_GSTS_RTPS), sts);
 781
 782         spin_unlock_irqrestore(&iommu->register_lock, flag);
 783 }
 784
 785 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
 786 {
 787         u32 val;
 788         unsigned long flag;
 789
 790         if (!rwbf_quirk && !cap_rwbf(iommu->cap))
 791                 return;
 792         val = iommu->gcmd | DMA_GCMD_WBF;
 793
 794         spin_lock_irqsave(&iommu->register_lock, flag);
 795         writel(val, iommu->reg + DMAR_GCMD_REG);
 796
 797         /* Make sure hardware complete it */
 798         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 799                         readl, (!(val & DMA_GSTS_WBFS)), val);
 800
 801         spin_unlock_irqrestore(&iommu->register_lock, flag);
 802 }
 803
 804 /* return value determine if we need a write buffer flush */
 805 static int __iommu_flush_context(struct intel_iommu *iommu,
 806         u16 did, u16 source_id, u8 function_mask, u64 type,
 807         int non_present_entry_flush)
 808 {
 809         u64 val = 0;
 810         unsigned long flag;
 811
 812         /*
 813          * In the non-present entry flush case, if hardware doesn't cache
 814          * non-present entry we do nothing and if hardware cache non-present
 815          * entry, we flush entries of domain 0 (the domain id is used to cache
 816          * any non-present entries)
 817          */
 818         if (non_present_entry_flush) {
 819                 if (!cap_caching_mode(iommu->cap))
 820                         return 1;
 821                 else
 822                         did = 0;
 823         }
 824
 825         switch (type) {
 826         case DMA_CCMD_GLOBAL_INVL:
 827                 val = DMA_CCMD_GLOBAL_INVL;
 828                 break;
 829         case DMA_CCMD_DOMAIN_INVL:
 830                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
 831                 break;
 832         case DMA_CCMD_DEVICE_INVL:
 833                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
 834                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
 835                 break;
 836         default:
 837                 BUG();
 838         }
 839         val |= DMA_CCMD_ICC;
 840
 841         spin_lock_irqsave(&iommu->register_lock, flag);
 842         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
 843
 844         /* Make sure hardware complete it */
 845         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
 846                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
 847
 848         spin_unlock_irqrestore(&iommu->register_lock, flag);
 849
 850         /* flush context entry will implicitly flush write buffer */
 851         return 0;
 852 }
 853
 854 /* return value determine if we need a write buffer flush */
 855 static int __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
 856         u64 addr, unsigned int size_order, u64 type,
 857         int non_present_entry_flush)
 858 {
 859         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
 860         u64 val = 0, val_iva = 0;
 861         unsigned long flag;
 862
 863         /*
 864          * In the non-present entry flush case, if hardware doesn't cache
 865          * non-present entry we do nothing and if hardware cache non-present
 866          * entry, we flush entries of domain 0 (the domain id is used to cache
 867          * any non-present entries)
 868          */
 869         if (non_present_entry_flush) {
 870                 if (!cap_caching_mode(iommu->cap))
 871                         return 1;
 872                 else
 873                         did = 0;
 874         }
 875
 876         switch (type) {
 877         case DMA_TLB_GLOBAL_FLUSH:
 878                 /* global flush doesn't need set IVA_REG */
 879                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
 880                 break;
 881         case DMA_TLB_DSI_FLUSH:
 882                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
 883                 break;
 884         case DMA_TLB_PSI_FLUSH:
 885                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
 886                 /* Note: always flush non-leaf currently */
 887                 val_iva = size_order | addr;
 888                 break;
 889         default:
 890                 BUG();
 891         }
 892         /* Note: set drain read/write */
 893 #if 0
 894         /*
 895          * This is probably to be super secure.. Looks like we can
 896          * ignore it without any impact.
 897          */
 898         if (cap_read_drain(iommu->cap))
 899                 val |= DMA_TLB_READ_DRAIN;
 900 #endif
 901         if (cap_write_drain(iommu->cap))
 902                 val |= DMA_TLB_WRITE_DRAIN;
 903
 904         spin_lock_irqsave(&iommu->register_lock, flag);
 905         /* Note: Only uses first TLB reg currently */
 906         if (val_iva)
 907                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
 908         dmar_writeq(iommu->reg + tlb_offset + 8, val);
 909
 910         /* Make sure hardware complete it */
 911         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
 912                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
 913
 914         spin_unlock_irqrestore(&iommu->register_lock, flag);
 915
 916         /* check IOTLB invalidation granularity */
 917         if (DMA_TLB_IAIG(val) == 0)
 918                 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
 919         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
 920                 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
 921                         (unsigned long long)DMA_TLB_IIRG(type),
 922                         (unsigned long long)DMA_TLB_IAIG(val));
 923         /* flush iotlb entry will implicitly flush write buffer */
 924         return 0;
 925 }
 926
 927 static int iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
 928         u64 addr, unsigned int pages, int non_present_entry_flush)
 929 {
 930         unsigned int mask;
 931
 932         BUG_ON(addr & (~VTD_PAGE_MASK));
 933         BUG_ON(pages == 0);
 934
 935         /* Fallback to domain selective flush if no PSI support */
 936         if (!cap_pgsel_inv(iommu->cap))
 937                 return iommu->flush.flush_iotlb(iommu, did, 0, 0,
 938                                                 DMA_TLB_DSI_FLUSH,
 939                                                 non_present_entry_flush);
 940
 941         /*
 942          * PSI requires page size to be 2 ^ x, and the base address is naturally
 943          * aligned to the size
 944          */
 945         mask = ilog2(__roundup_pow_of_two(pages));
 946         /* Fallback to domain selective flush if size is too big */
 947         if (mask > cap_max_amask_val(iommu->cap))
 948                 return iommu->flush.flush_iotlb(iommu, did, 0, 0,
 949                         DMA_TLB_DSI_FLUSH, non_present_entry_flush);
 950
 951         return iommu->flush.flush_iotlb(iommu, did, addr, mask,
 952                                         DMA_TLB_PSI_FLUSH,
 953                                         non_present_entry_flush);
 954 }
 955
 956 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
 957 {
 958         u32 pmen;
 959         unsigned long flags;
 960
 961         spin_lock_irqsave(&iommu->register_lock, flags);
 962         pmen = readl(iommu->reg + DMAR_PMEN_REG);
 963         pmen &= ~DMA_PMEN_EPM;
 964         writel(pmen, iommu->reg + DMAR_PMEN_REG);
 965
 966         /* wait for the protected region status bit to clear */
 967         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
 968                 readl, !(pmen & DMA_PMEN_PRS), pmen);
 969
 970         spin_unlock_irqrestore(&iommu->register_lock, flags);
 971 }
 972
 973 static int iommu_enable_translation(struct intel_iommu *iommu)
 974 {
 975         u32 sts;
 976         unsigned long flags;
 977
 978         spin_lock_irqsave(&iommu->register_lock, flags);
 979         writel(iommu->gcmd|DMA_GCMD_TE, iommu->reg + DMAR_GCMD_REG);
 980
 981         /* Make sure hardware complete it */
 982         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 983                 readl, (sts & DMA_GSTS_TES), sts);
 984
 985         iommu->gcmd |= DMA_GCMD_TE;
 986         spin_unlock_irqrestore(&iommu->register_lock, flags);
 987         return 0;
 988 }
 989
 990 static int iommu_disable_translation(struct intel_iommu *iommu)
 991 {
 992         u32 sts;
 993         unsigned long flag;
 994
 995         spin_lock_irqsave(&iommu->register_lock, flag);
 996         iommu->gcmd &= ~DMA_GCMD_TE;
 997         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
 998
 999         /* Make sure hardware complete it */
1000         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1001                 readl, (!(sts & DMA_GSTS_TES)), sts);
1002
1003         spin_unlock_irqrestore(&iommu->register_lock, flag);
1004         return 0;
1005 }
1006
1007
1008 static int iommu_init_domains(struct intel_iommu *iommu)
1009 {
1010         unsigned long ndomains;
1011         unsigned long nlongs;
1012
1013         ndomains = cap_ndoms(iommu->cap);
1014         pr_debug("Number of Domains supportd <%ld>\n", ndomains);
1015         nlongs = BITS_TO_LONGS(ndomains);
1016
1017         /* TBD: there might be 64K domains,
1018          * consider other allocation for future chip
1019          */
1020         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
1021         if (!iommu->domain_ids) {
1022                 printk(KERN_ERR "Allocating domain id array failed\n");
1023                 return -ENOMEM;
1024         }
1025         iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
1026                         GFP_KERNEL);
1027         if (!iommu->domains) {
1028                 printk(KERN_ERR "Allocating domain array failed\n");
1029                 kfree(iommu->domain_ids);
1030                 return -ENOMEM;
1031         }
1032
1033         spin_lock_init(&iommu->lock);
1034
1035         /*
1036          * if Caching mode is set, then invalid translations are tagged
1037          * with domainid 0. Hence we need to pre-allocate it.
1038          */
1039         if (cap_caching_mode(iommu->cap))
1040                 set_bit(0, iommu->domain_ids);
1041         return 0;
1042 }
1043
1044
1045 static void domain_exit(struct dmar_domain *domain);
1046 static void vm_domain_exit(struct dmar_domain *domain);
1047
1048 void free_dmar_iommu(struct intel_iommu *iommu)
1049 {
1050         struct dmar_domain *domain;
1051         int i;
1052         unsigned long flags;
1053
1054         i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
1055         for (; i < cap_ndoms(iommu->cap); ) {
1056                 domain = iommu->domains[i];
1057                 clear_bit(i, iommu->domain_ids);
1058
1059                 spin_lock_irqsave(&domain->iommu_lock, flags);
1060                 if (--domain->iommu_count == 0) {
1061                         if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
1062                                 vm_domain_exit(domain);
1063                         else
1064                                 domain_exit(domain);
1065                 }
1066                 spin_unlock_irqrestore(&domain->iommu_lock, flags);
1067
1068                 i = find_next_bit(iommu->domain_ids,
1069                         cap_ndoms(iommu->cap), i+1);
1070         }
1071
1072         if (iommu->gcmd & DMA_GCMD_TE)
1073                 iommu_disable_translation(iommu);
1074
1075         if (iommu->irq) {
1076                 set_irq_data(iommu->irq, NULL);
1077                 /* This will mask the irq */
1078                 free_irq(iommu->irq, iommu);
1079                 destroy_irq(iommu->irq);
1080         }
1081
1082         kfree(iommu->domains);
1083         kfree(iommu->domain_ids);
1084
1085         g_iommus[iommu->seq_id] = NULL;
1086
1087         /* if all iommus are freed, free g_iommus */
1088         for (i = 0; i < g_num_of_iommus; i++) {
1089                 if (g_iommus[i])
1090                         break;
1091         }
1092
1093         if (i == g_num_of_iommus)
1094                 kfree(g_iommus);
1095
1096         /* free context mapping */
1097         free_context_table(iommu);
1098 }
1099
1100 static struct dmar_domain * iommu_alloc_domain(struct intel_iommu *iommu)
1101 {
1102         unsigned long num;
1103         unsigned long ndomains;
1104         struct dmar_domain *domain;
1105         unsigned long flags;
1106
1107         domain = alloc_domain_mem();
1108         if (!domain)
1109                 return NULL;
1110
1111         ndomains = cap_ndoms(iommu->cap);
1112
1113         spin_lock_irqsave(&iommu->lock, flags);
1114         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1115         if (num >= ndomains) {
1116                 spin_unlock_irqrestore(&iommu->lock, flags);
1117                 free_domain_mem(domain);
1118                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1119                 return NULL;
1120         }
1121
1122         set_bit(num, iommu->domain_ids);
1123         domain->id = num;
1124         memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
1125         set_bit(iommu->seq_id, &domain->iommu_bmp);
1126         domain->flags = 0;
1127         iommu->domains[num] = domain;
1128         spin_unlock_irqrestore(&iommu->lock, flags);
1129
1130         return domain;
1131 }
1132
1133 static void iommu_free_domain(struct dmar_domain *domain)
1134 {
1135         unsigned long flags;
1136         struct intel_iommu *iommu;
1137
1138         iommu = domain_get_iommu(domain);
1139
1140         spin_lock_irqsave(&iommu->lock, flags);
1141         clear_bit(domain->id, iommu->domain_ids);
1142         spin_unlock_irqrestore(&iommu->lock, flags);
1143 }
1144
1145 static struct iova_domain reserved_iova_list;
1146 static struct lock_class_key reserved_alloc_key;
1147 static struct lock_class_key reserved_rbtree_key;
1148
1149 static void dmar_init_reserved_ranges(void)
1150 {
1151         struct pci_dev *pdev = NULL;
1152         struct iova *iova;
1153         int i;
1154         u64 addr, size;
1155
1156         init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1157
1158         lockdep_set_class(&reserved_iova_list.iova_alloc_lock,
1159                 &reserved_alloc_key);
1160         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1161                 &reserved_rbtree_key);
1162
1163         /* IOAPIC ranges shouldn't be accessed by DMA */
1164         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1165                 IOVA_PFN(IOAPIC_RANGE_END));
1166         if (!iova)
1167                 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1168
1169         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1170         for_each_pci_dev(pdev) {
1171                 struct resource *r;
1172
1173                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1174                         r = &pdev->resource[i];
1175                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1176                                 continue;
1177                         addr = r->start;
1178                         addr &= PAGE_MASK;
1179                         size = r->end - addr;
1180                         size = PAGE_ALIGN(size);
1181                         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(addr),
1182                                 IOVA_PFN(size + addr) - 1);
1183                         if (!iova)
1184                                 printk(KERN_ERR "Reserve iova failed\n");
1185                 }
1186         }
1187
1188 }
1189
1190 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1191 {
1192         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1193 }
1194
1195 static inline int guestwidth_to_adjustwidth(int gaw)
1196 {
1197         int agaw;
1198         int r = (gaw - 12) % 9;
1199
1200         if (r == 0)
1201                 agaw = gaw;
1202         else
1203                 agaw = gaw + 9 - r;
1204         if (agaw > 64)
1205                 agaw = 64;
1206         return agaw;
1207 }
1208
1209 static int domain_init(struct dmar_domain *domain, int guest_width)
1210 {
1211         struct intel_iommu *iommu;
1212         int adjust_width, agaw;
1213         unsigned long sagaw;
1214
1215         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1216         spin_lock_init(&domain->mapping_lock);
1217         spin_lock_init(&domain->iommu_lock);
1218
1219         domain_reserve_special_ranges(domain);
1220
1221         /* calculate AGAW */
1222         iommu = domain_get_iommu(domain);
1223         if (guest_width > cap_mgaw(iommu->cap))
1224                 guest_width = cap_mgaw(iommu->cap);
1225         domain->gaw = guest_width;
1226         adjust_width = guestwidth_to_adjustwidth(guest_width);
1227         agaw = width_to_agaw(adjust_width);
1228         sagaw = cap_sagaw(iommu->cap);
1229         if (!test_bit(agaw, &sagaw)) {
1230                 /* hardware doesn't support it, choose a bigger one */
1231                 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1232                 agaw = find_next_bit(&sagaw, 5, agaw);
1233                 if (agaw >= 5)
1234                         return -ENODEV;
1235         }
1236         domain->agaw = agaw;
1237         INIT_LIST_HEAD(&domain->devices);
1238
1239         if (ecap_coherent(iommu->ecap))
1240                 domain->iommu_coherency = 1;
1241         else
1242                 domain->iommu_coherency = 0;
1243
1244         domain->iommu_count = 1;
1245
1246         /* always allocate the top pgd */
1247         domain->pgd = (struct dma_pte *)alloc_pgtable_page();
1248         if (!domain->pgd)
1249                 return -ENOMEM;
1250         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE);
1251         return 0;
1252 }
1253
1254 static void domain_exit(struct dmar_domain *domain)
1255 {
1256         u64 end;
1257
1258         /* Domain 0 is reserved, so dont process it */
1259         if (!domain)
1260                 return;
1261
1262         domain_remove_dev_info(domain);
1263         /* destroy iovas */
1264         put_iova_domain(&domain->iovad);
1265         end = DOMAIN_MAX_ADDR(domain->gaw);
1266         end = end & (~PAGE_MASK);
1267
1268         /* clear ptes */
1269         dma_pte_clear_range(domain, 0, end);
1270
1271         /* free page tables */
1272         dma_pte_free_pagetable(domain, 0, end);
1273
1274         iommu_free_domain(domain);
1275         free_domain_mem(domain);
1276 }
1277
1278 static int domain_context_mapping_one(struct dmar_domain *domain,
1279                 u8 bus, u8 devfn)
1280 {
1281         struct context_entry *context;
1282         unsigned long flags;
1283         struct intel_iommu *iommu;
1284         struct dma_pte *pgd;
1285         unsigned long num;
1286         unsigned long ndomains;
1287         int id;
1288         int agaw;
1289
1290         pr_debug("Set context mapping for %02x:%02x.%d\n",
1291                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1292         BUG_ON(!domain->pgd);
1293
1294         iommu = device_to_iommu(bus, devfn);
1295         if (!iommu)
1296                 return -ENODEV;
1297
1298         context = device_to_context_entry(iommu, bus, devfn);
1299         if (!context)
1300                 return -ENOMEM;
1301         spin_lock_irqsave(&iommu->lock, flags);
1302         if (context_present(context)) {
1303                 spin_unlock_irqrestore(&iommu->lock, flags);
1304                 return 0;
1305         }
1306
1307         id = domain->id;
1308         pgd = domain->pgd;
1309
1310         if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) {
1311                 int found = 0;
1312
1313                 /* find an available domain id for this device in iommu */
1314                 ndomains = cap_ndoms(iommu->cap);
1315                 num = find_first_bit(iommu->domain_ids, ndomains);
1316                 for (; num < ndomains; ) {
1317                         if (iommu->domains[num] == domain) {
1318                                 id = num;
1319                                 found = 1;
1320                                 break;
1321                         }
1322                         num = find_next_bit(iommu->domain_ids,
1323                                             cap_ndoms(iommu->cap), num+1);
1324                 }
1325
1326                 if (found == 0) {
1327                         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1328                         if (num >= ndomains) {
1329                                 spin_unlock_irqrestore(&iommu->lock, flags);
1330                                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1331                                 return -EFAULT;
1332                         }
1333
1334                         set_bit(num, iommu->domain_ids);
1335                         iommu->domains[num] = domain;
1336                         id = num;
1337                 }
1338
1339                 /* Skip top levels of page tables for
1340                  * iommu which has less agaw than default.
1341                  */
1342                 for (agaw = domain->agaw; agaw != iommu->agaw; agaw--) {
1343                         pgd = phys_to_virt(dma_pte_addr(pgd));
1344                         if (!dma_pte_present(pgd)) {
1345                                 spin_unlock_irqrestore(&iommu->lock, flags);
1346                                 return -ENOMEM;
1347                         }
1348                 }
1349         }
1350
1351         context_set_domain_id(context, id);
1352         context_set_address_width(context, iommu->agaw);
1353         context_set_address_root(context, virt_to_phys(pgd));
1354         context_set_translation_type(context, CONTEXT_TT_MULTI_LEVEL);
1355         context_set_fault_enable(context);
1356         context_set_present(context);
1357         domain_flush_cache(domain, context, sizeof(*context));
1358
1359         /* it's a non-present to present mapping */
1360         if (iommu->flush.flush_context(iommu, domain->id,
1361                 (((u16)bus) << 8) | devfn, DMA_CCMD_MASK_NOBIT,
1362                 DMA_CCMD_DEVICE_INVL, 1))
1363                 iommu_flush_write_buffer(iommu);
1364         else
1365                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_DSI_FLUSH, 0);
1366
1367         spin_unlock_irqrestore(&iommu->lock, flags);
1368
1369         spin_lock_irqsave(&domain->iommu_lock, flags);
1370         if (!test_and_set_bit(iommu->seq_id, &domain->iommu_bmp)) {
1371                 domain->iommu_count++;
1372                 domain_update_iommu_coherency(domain);
1373         }
1374         spin_unlock_irqrestore(&domain->iommu_lock, flags);
1375         return 0;
1376 }
1377
1378 static int
1379 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev)
1380 {
1381         int ret;
1382         struct pci_dev *tmp, *parent;
1383
1384         ret = domain_context_mapping_one(domain, pdev->bus->number,
1385                 pdev->devfn);
1386         if (ret)
1387                 return ret;
1388
1389         /* dependent device mapping */
1390         tmp = pci_find_upstream_pcie_bridge(pdev);
1391         if (!tmp)
1392                 return 0;
1393         /* Secondary interface's bus number and devfn 0 */
1394         parent = pdev->bus->self;
1395         while (parent != tmp) {
1396                 ret = domain_context_mapping_one(domain, parent->bus->number,
1397                         parent->devfn);
1398                 if (ret)
1399                         return ret;
1400                 parent = parent->bus->self;
1401         }
1402         if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
1403                 return domain_context_mapping_one(domain,
1404                         tmp->subordinate->number, 0);
1405         else /* this is a legacy PCI bridge */
1406                 return domain_context_mapping_one(domain,
1407                         tmp->bus->number, tmp->devfn);
1408 }
1409
1410 static int domain_context_mapped(struct pci_dev *pdev)
1411 {
1412         int ret;
1413         struct pci_dev *tmp, *parent;
1414         struct intel_iommu *iommu;
1415
1416         iommu = device_to_iommu(pdev->bus->number, pdev->devfn);
1417         if (!iommu)
1418                 return -ENODEV;
1419
1420         ret = device_context_mapped(iommu,
1421                 pdev->bus->number, pdev->devfn);
1422         if (!ret)
1423                 return ret;
1424         /* dependent device mapping */
1425         tmp = pci_find_upstream_pcie_bridge(pdev);
1426         if (!tmp)
1427                 return ret;
1428         /* Secondary interface's bus number and devfn 0 */
1429         parent = pdev->bus->self;
1430         while (parent != tmp) {
1431                 ret = device_context_mapped(iommu, parent->bus->number,
1432                         parent->devfn);
1433                 if (!ret)
1434                         return ret;
1435                 parent = parent->bus->self;
1436         }
1437         if (tmp->is_pcie)
1438                 return device_context_mapped(iommu,
1439                         tmp->subordinate->number, 0);
1440         else
1441                 return device_context_mapped(iommu,
1442                         tmp->bus->number, tmp->devfn);
1443 }
1444
1445 static int
1446 domain_page_mapping(struct dmar_domain *domain, dma_addr_t iova,
1447                         u64 hpa, size_t size, int prot)
1448 {
1449         u64 start_pfn, end_pfn;
1450         struct dma_pte *pte;
1451         int index;
1452         int addr_width = agaw_to_width(domain->agaw);
1453
1454         hpa &= (((u64)1) << addr_width) - 1;
1455
1456         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1457                 return -EINVAL;
1458         iova &= PAGE_MASK;
1459         start_pfn = ((u64)hpa) >> VTD_PAGE_SHIFT;
1460         end_pfn = (VTD_PAGE_ALIGN(((u64)hpa) + size)) >> VTD_PAGE_SHIFT;
1461         index = 0;
1462         while (start_pfn < end_pfn) {
1463                 pte = addr_to_dma_pte(domain, iova + VTD_PAGE_SIZE * index);
1464                 if (!pte)
1465                         return -ENOMEM;
1466                 /* We don't need lock here, nobody else
1467                  * touches the iova range
1468                  */
1469                 BUG_ON(dma_pte_addr(pte));
1470                 dma_set_pte_addr(pte, start_pfn << VTD_PAGE_SHIFT);
1471                 dma_set_pte_prot(pte, prot);
1472                 domain_flush_cache(domain, pte, sizeof(*pte));
1473                 start_pfn++;
1474                 index++;
1475         }
1476         return 0;
1477 }
1478
1479 static void iommu_detach_dev(struct intel_iommu *iommu, u8 bus, u8 devfn)
1480 {
1481         if (!iommu)
1482                 return;
1483
1484         clear_context_table(iommu, bus, devfn);
1485         iommu->flush.flush_context(iommu, 0, 0, 0,
1486                                            DMA_CCMD_GLOBAL_INVL, 0);
1487         iommu->flush.flush_iotlb(iommu, 0, 0, 0,
1488                                          DMA_TLB_GLOBAL_FLUSH, 0);
1489 }
1490
1491 static void domain_remove_dev_info(struct dmar_domain *domain)
1492 {
1493         struct device_domain_info *info;
1494         unsigned long flags;
1495         struct intel_iommu *iommu;
1496
1497         spin_lock_irqsave(&device_domain_lock, flags);
1498         while (!list_empty(&domain->devices)) {
1499                 info = list_entry(domain->devices.next,
1500                         struct device_domain_info, link);
1501                 list_del(&info->link);
1502                 list_del(&info->global);
1503                 if (info->dev)
1504                         info->dev->dev.archdata.iommu = NULL;
1505                 spin_unlock_irqrestore(&device_domain_lock, flags);
1506
1507                 iommu = device_to_iommu(info->bus, info->devfn);
1508                 iommu_detach_dev(iommu, info->bus, info->devfn);
1509                 free_devinfo_mem(info);
1510
1511                 spin_lock_irqsave(&device_domain_lock, flags);
1512         }
1513         spin_unlock_irqrestore(&device_domain_lock, flags);
1514 }
1515
1516 /*
1517  * find_domain
1518  * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1519  */
1520 static struct dmar_domain *
1521 find_domain(struct pci_dev *pdev)
1522 {
1523         struct device_domain_info *info;
1524
1525         /* No lock here, assumes no domain exit in normal case */
1526         info = pdev->dev.archdata.iommu;
1527         if (info)
1528                 return info->domain;
1529         return NULL;
1530 }
1531
1532 /* domain is initialized */
1533 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1534 {
1535         struct dmar_domain *domain, *found = NULL;
1536         struct intel_iommu *iommu;
1537         struct dmar_drhd_unit *drhd;
1538         struct device_domain_info *info, *tmp;
1539         struct pci_dev *dev_tmp;
1540         unsigned long flags;
1541         int bus = 0, devfn = 0;
1542
1543         domain = find_domain(pdev);
1544         if (domain)
1545                 return domain;
1546
1547         dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1548         if (dev_tmp) {
1549                 if (dev_tmp->is_pcie) {
1550                         bus = dev_tmp->subordinate->number;
1551                         devfn = 0;
1552                 } else {
1553                         bus = dev_tmp->bus->number;
1554                         devfn = dev_tmp->devfn;
1555                 }
1556                 spin_lock_irqsave(&device_domain_lock, flags);
1557                 list_for_each_entry(info, &device_domain_list, global) {
1558                         if (info->bus == bus && info->devfn == devfn) {
1559                                 found = info->domain;
1560                                 break;
1561                         }
1562                 }
1563                 spin_unlock_irqrestore(&device_domain_lock, flags);
1564                 /* pcie-pci bridge already has a domain, uses it */
1565                 if (found) {
1566                         domain = found;
1567                         goto found_domain;
1568                 }
1569         }
1570
1571         /* Allocate new domain for the device */
1572         drhd = dmar_find_matched_drhd_unit(pdev);
1573         if (!drhd) {
1574                 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1575                         pci_name(pdev));
1576                 return NULL;
1577         }
1578         iommu = drhd->iommu;
1579
1580         domain = iommu_alloc_domain(iommu);
1581         if (!domain)
1582                 goto error;
1583
1584         if (domain_init(domain, gaw)) {
1585                 domain_exit(domain);
1586                 goto error;
1587         }
1588
1589         /* register pcie-to-pci device */
1590         if (dev_tmp) {
1591                 info = alloc_devinfo_mem();
1592                 if (!info) {
1593                         domain_exit(domain);
1594                         goto error;
1595                 }
1596                 info->bus = bus;
1597                 info->devfn = devfn;
1598                 info->dev = NULL;
1599                 info->domain = domain;
1600                 /* This domain is shared by devices under p2p bridge */
1601                 domain->flags |= DOMAIN_FLAG_P2P_MULTIPLE_DEVICES;
1602
1603                 /* pcie-to-pci bridge already has a domain, uses it */
1604                 found = NULL;
1605                 spin_lock_irqsave(&device_domain_lock, flags);
1606                 list_for_each_entry(tmp, &device_domain_list, global) {
1607                         if (tmp->bus == bus && tmp->devfn == devfn) {
1608                                 found = tmp->domain;
1609                                 break;
1610                         }
1611                 }
1612                 if (found) {
1613                         free_devinfo_mem(info);
1614                         domain_exit(domain);
1615                         domain = found;
1616                 } else {
1617                         list_add(&info->link, &domain->devices);
1618                         list_add(&info->global, &device_domain_list);
1619                 }
1620                 spin_unlock_irqrestore(&device_domain_lock, flags);
1621         }
1622
1623 found_domain:
1624         info = alloc_devinfo_mem();
1625         if (!info)
1626                 goto error;
1627         info->bus = pdev->bus->number;
1628         info->devfn = pdev->devfn;
1629         info->dev = pdev;
1630         info->domain = domain;
1631         spin_lock_irqsave(&device_domain_lock, flags);
1632         /* somebody is fast */
1633         found = find_domain(pdev);
1634         if (found != NULL) {
1635                 spin_unlock_irqrestore(&device_domain_lock, flags);
1636                 if (found != domain) {
1637                         domain_exit(domain);
1638                         domain = found;
1639                 }
1640                 free_devinfo_mem(info);
1641                 return domain;
1642         }
1643         list_add(&info->link, &domain->devices);
1644         list_add(&info->global, &device_domain_list);
1645         pdev->dev.archdata.iommu = info;
1646         spin_unlock_irqrestore(&device_domain_lock, flags);
1647         return domain;
1648 error:
1649         /* recheck it here, maybe others set it */
1650         return find_domain(pdev);
1651 }
1652
1653 static int iommu_prepare_identity_map(struct pci_dev *pdev,
1654                                       unsigned long long start,
1655                                       unsigned long long end)
1656 {
1657         struct dmar_domain *domain;
1658         unsigned long size;
1659         unsigned long long base;
1660         int ret;
1661
1662         printk(KERN_INFO
1663                 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1664                 pci_name(pdev), start, end);
1665         /* page table init */
1666         domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1667         if (!domain)
1668                 return -ENOMEM;
1669
1670         /* The address might not be aligned */
1671         base = start & PAGE_MASK;
1672         size = end - base;
1673         size = PAGE_ALIGN(size);
1674         if (!reserve_iova(&domain->iovad, IOVA_PFN(base),
1675                         IOVA_PFN(base + size) - 1)) {
1676                 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1677                 ret = -ENOMEM;
1678                 goto error;
1679         }
1680
1681         pr_debug("Mapping reserved region %lx@%llx for %s\n",
1682                 size, base, pci_name(pdev));
1683         /*
1684          * RMRR range might have overlap with physical memory range,
1685          * clear it first
1686          */
1687         dma_pte_clear_range(domain, base, base + size);
1688
1689         ret = domain_page_mapping(domain, base, base, size,
1690                 DMA_PTE_READ|DMA_PTE_WRITE);
1691         if (ret)
1692                 goto error;
1693
1694         /* context entry init */
1695         ret = domain_context_mapping(domain, pdev);
1696         if (!ret)
1697                 return 0;
1698 error:
1699         domain_exit(domain);
1700         return ret;
1701
1702 }
1703
1704 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
1705         struct pci_dev *pdev)
1706 {
1707         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1708                 return 0;
1709         return iommu_prepare_identity_map(pdev, rmrr->base_address,
1710                 rmrr->end_address + 1);
1711 }
1712
1713 #ifdef CONFIG_DMAR_GFX_WA
1714 struct iommu_prepare_data {
1715         struct pci_dev *pdev;
1716         int ret;
1717 };
1718
1719 static int __init iommu_prepare_work_fn(unsigned long start_pfn,
1720                                          unsigned long end_pfn, void *datax)
1721 {
1722         struct iommu_prepare_data *data;
1723
1724         data = (struct iommu_prepare_data *)datax;
1725
1726         data->ret = iommu_prepare_identity_map(data->pdev,
1727                                 start_pfn<<PAGE_SHIFT, end_pfn<<PAGE_SHIFT);
1728         return data->ret;
1729
1730 }
1731
1732 static int __init iommu_prepare_with_active_regions(struct pci_dev *pdev)
1733 {
1734         int nid;
1735         struct iommu_prepare_data data;
1736
1737         data.pdev = pdev;
1738         data.ret = 0;
1739
1740         for_each_online_node(nid) {
1741                 work_with_active_regions(nid, iommu_prepare_work_fn, &data);
1742                 if (data.ret)
1743                         return data.ret;
1744         }
1745         return data.ret;
1746 }
1747
1748 static void __init iommu_prepare_gfx_mapping(void)
1749 {
1750         struct pci_dev *pdev = NULL;
1751         int ret;
1752
1753         for_each_pci_dev(pdev) {
1754                 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO ||
1755                                 !IS_GFX_DEVICE(pdev))
1756                         continue;
1757                 printk(KERN_INFO "IOMMU: gfx device %s 1-1 mapping\n",
1758                         pci_name(pdev));
1759                 ret = iommu_prepare_with_active_regions(pdev);
1760                 if (ret)
1761                         printk(KERN_ERR "IOMMU: mapping reserved region failed\n");
1762         }
1763 }
1764 #else /* !CONFIG_DMAR_GFX_WA */
1765 static inline void iommu_prepare_gfx_mapping(void)
1766 {
1767         return;
1768 }
1769 #endif
1770
1771 #ifdef CONFIG_DMAR_FLOPPY_WA
1772 static inline void iommu_prepare_isa(void)
1773 {
1774         struct pci_dev *pdev;
1775         int ret;
1776
1777         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
1778         if (!pdev)
1779                 return;
1780
1781         printk(KERN_INFO "IOMMU: Prepare 0-16M unity mapping for LPC\n");
1782         ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
1783
1784         if (ret)
1785                 printk("IOMMU: Failed to create 0-64M identity map, "
1786                         "floppy might not work\n");
1787
1788 }
1789 #else
1790 static inline void iommu_prepare_isa(void)
1791 {
1792         return;
1793 }
1794 #endif /* !CONFIG_DMAR_FLPY_WA */
1795
1796 static int __init init_dmars(void)
1797 {
1798         struct dmar_drhd_unit *drhd;
1799         struct dmar_rmrr_unit *rmrr;
1800         struct pci_dev *pdev;
1801         struct intel_iommu *iommu;
1802         int i, ret;
1803
1804         /*
1805          * for each drhd
1806          *    allocate root
1807          *    initialize and program root entry to not present
1808          * endfor
1809          */
1810         for_each_drhd_unit(drhd) {
1811                 g_num_of_iommus++;
1812                 /*
1813                  * lock not needed as this is only incremented in the single
1814                  * threaded kernel __init code path all other access are read
1815                  * only
1816                  */
1817         }
1818
1819         g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
1820                         GFP_KERNEL);
1821         if (!g_iommus) {
1822                 printk(KERN_ERR "Allocating global iommu array failed\n");
1823                 ret = -ENOMEM;
1824                 goto error;
1825         }
1826
1827         deferred_flush = kzalloc(g_num_of_iommus *
1828                 sizeof(struct deferred_flush_tables), GFP_KERNEL);
1829         if (!deferred_flush) {
1830                 kfree(g_iommus);
1831                 ret = -ENOMEM;
1832                 goto error;
1833         }
1834
1835         for_each_drhd_unit(drhd) {
1836                 if (drhd->ignored)
1837                         continue;
1838
1839                 iommu = drhd->iommu;
1840                 g_iommus[iommu->seq_id] = iommu;
1841
1842                 ret = iommu_init_domains(iommu);
1843                 if (ret)
1844                         goto error;
1845
1846                 /*
1847                  * TBD:
1848                  * we could share the same root & context tables
1849                  * amoung all IOMMU's. Need to Split it later.
1850                  */
1851                 ret = iommu_alloc_root_entry(iommu);
1852                 if (ret) {
1853                         printk(KERN_ERR "IOMMU: allocate root entry failed\n");
1854                         goto error;
1855                 }
1856         }
1857
1858         /*
1859          * Start from the sane iommu hardware state.
1860          */
1861         for_each_drhd_unit(drhd) {
1862                 if (drhd->ignored)
1863                         continue;
1864
1865                 iommu = drhd->iommu;
1866
1867                 /*
1868                  * If the queued invalidation is already initialized by us
1869                  * (for example, while enabling interrupt-remapping) then
1870                  * we got the things already rolling from a sane state.
1871                  */
1872                 if (iommu->qi)
1873                         continue;
1874
1875                 /*
1876                  * Clear any previous faults.
1877                  */
1878                 dmar_fault(-1, iommu);
1879                 /*
1880                  * Disable queued invalidation if supported and already enabled
1881                  * before OS handover.
1882                  */
1883                 dmar_disable_qi(iommu);
1884         }
1885
1886         for_each_drhd_unit(drhd) {
1887                 if (drhd->ignored)
1888                         continue;
1889
1890                 iommu = drhd->iommu;
1891
1892                 if (dmar_enable_qi(iommu)) {
1893                         /*
1894                          * Queued Invalidate not enabled, use Register Based
1895                          * Invalidate
1896                          */
1897                         iommu->flush.flush_context = __iommu_flush_context;
1898                         iommu->flush.flush_iotlb = __iommu_flush_iotlb;
1899                         printk(KERN_INFO "IOMMU 0x%Lx: using Register based "
1900                                "invalidation\n",
1901                                (unsigned long long)drhd->reg_base_addr);
1902                 } else {
1903                         iommu->flush.flush_context = qi_flush_context;
1904                         iommu->flush.flush_iotlb = qi_flush_iotlb;
1905                         printk(KERN_INFO "IOMMU 0x%Lx: using Queued "
1906                                "invalidation\n",
1907                                (unsigned long long)drhd->reg_base_addr);
1908                 }
1909         }
1910
1911         /*
1912          * For each rmrr
1913          *   for each dev attached to rmrr
1914          *   do
1915          *     locate drhd for dev, alloc domain for dev
1916          *     allocate free domain
1917          *     allocate page table entries for rmrr
1918          *     if context not allocated for bus
1919          *           allocate and init context
1920          *           set present in root table for this bus
1921          *     init context with domain, translation etc
1922          *    endfor
1923          * endfor
1924          */
1925         for_each_rmrr_units(rmrr) {
1926                 for (i = 0; i < rmrr->devices_cnt; i++) {
1927                         pdev = rmrr->devices[i];
1928                         /* some BIOS lists non-exist devices in DMAR table */
1929                         if (!pdev)
1930                                 continue;
1931                         ret = iommu_prepare_rmrr_dev(rmrr, pdev);
1932                         if (ret)
1933                                 printk(KERN_ERR
1934                                  "IOMMU: mapping reserved region failed\n");
1935                 }
1936         }
1937
1938         iommu_prepare_gfx_mapping();
1939
1940         iommu_prepare_isa();
1941
1942         /*
1943          * for each drhd
1944          *   enable fault log
1945          *   global invalidate context cache
1946          *   global invalidate iotlb
1947          *   enable translation
1948          */
1949         for_each_drhd_unit(drhd) {
1950                 if (drhd->ignored)
1951                         continue;
1952                 iommu = drhd->iommu;
1953
1954                 iommu_flush_write_buffer(iommu);
1955
1956                 ret = dmar_set_interrupt(iommu);
1957                 if (ret)
1958                         goto error;
1959
1960                 iommu_set_root_entry(iommu);
1961
1962                 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
1963                                            0);
1964                 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
1965                                          0);
1966                 iommu_disable_protect_mem_regions(iommu);
1967
1968                 ret = iommu_enable_translation(iommu);
1969                 if (ret)
1970                         goto error;
1971         }
1972
1973         return 0;
1974 error:
1975         for_each_drhd_unit(drhd) {
1976                 if (drhd->ignored)
1977                         continue;
1978                 iommu = drhd->iommu;
1979                 free_iommu(iommu);
1980         }
1981         kfree(g_iommus);
1982         return ret;
1983 }
1984
1985 static inline u64 aligned_size(u64 host_addr, size_t size)
1986 {
1987         u64 addr;
1988         addr = (host_addr & (~PAGE_MASK)) + size;
1989         return PAGE_ALIGN(addr);
1990 }
1991
1992 struct iova *
1993 iommu_alloc_iova(struct dmar_domain *domain, size_t size, u64 end)
1994 {
1995         struct iova *piova;
1996
1997         /* Make sure it's in range */
1998         end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end);
1999         if (!size || (IOVA_START_ADDR + size > end))
2000                 return NULL;
2001
2002         piova = alloc_iova(&domain->iovad,
2003                         size >> PAGE_SHIFT, IOVA_PFN(end), 1);
2004         return piova;
2005 }
2006
2007 static struct iova *
2008 __intel_alloc_iova(struct device *dev, struct dmar_domain *domain,
2009                    size_t size, u64 dma_mask)
2010 {
2011         struct pci_dev *pdev = to_pci_dev(dev);
2012         struct iova *iova = NULL;
2013
2014         if (dma_mask <= DMA_32BIT_MASK || dmar_forcedac)
2015                 iova = iommu_alloc_iova(domain, size, dma_mask);
2016         else {
2017                 /*
2018                  * First try to allocate an io virtual address in
2019                  * DMA_32BIT_MASK and if that fails then try allocating
2020                  * from higher range
2021                  */
2022                 iova = iommu_alloc_iova(domain, size, DMA_32BIT_MASK);
2023                 if (!iova)
2024                         iova = iommu_alloc_iova(domain, size, dma_mask);
2025         }
2026
2027         if (!iova) {
2028                 printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev));
2029                 return NULL;
2030         }
2031
2032         return iova;
2033 }
2034
2035 static struct dmar_domain *
2036 get_valid_domain_for_dev(struct pci_dev *pdev)
2037 {
2038         struct dmar_domain *domain;
2039         int ret;
2040
2041         domain = get_domain_for_dev(pdev,
2042                         DEFAULT_DOMAIN_ADDRESS_WIDTH);
2043         if (!domain) {
2044                 printk(KERN_ERR
2045                         "Allocating domain for %s failed", pci_name(pdev));
2046                 return NULL;
2047         }
2048
2049         /* make sure context mapping is ok */
2050         if (unlikely(!domain_context_mapped(pdev))) {
2051                 ret = domain_context_mapping(domain, pdev);
2052                 if (ret) {
2053                         printk(KERN_ERR
2054                                 "Domain context map for %s failed",
2055                                 pci_name(pdev));
2056                         return NULL;
2057                 }
2058         }
2059
2060         return domain;
2061 }
2062
2063 static dma_addr_t __intel_map_single(struct device *hwdev, phys_addr_t paddr,
2064                                      size_t size, int dir, u64 dma_mask)
2065 {
2066         struct pci_dev *pdev = to_pci_dev(hwdev);
2067         struct dmar_domain *domain;
2068         phys_addr_t start_paddr;
2069         struct iova *iova;
2070         int prot = 0;
2071         int ret;
2072         struct intel_iommu *iommu;
2073
2074         BUG_ON(dir == DMA_NONE);
2075         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2076                 return paddr;
2077
2078         domain = get_valid_domain_for_dev(pdev);
2079         if (!domain)
2080                 return 0;
2081
2082         iommu = domain_get_iommu(domain);
2083         size = aligned_size((u64)paddr, size);
2084
2085         iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
2086         if (!iova)
2087                 goto error;
2088
2089         start_paddr = (phys_addr_t)iova->pfn_lo << PAGE_SHIFT;
2090
2091         /*
2092          * Check if DMAR supports zero-length reads on write only
2093          * mappings..
2094          */
2095         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2096                         !cap_zlr(iommu->cap))
2097                 prot |= DMA_PTE_READ;
2098         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2099                 prot |= DMA_PTE_WRITE;
2100         /*
2101          * paddr - (paddr + size) might be partial page, we should map the whole
2102          * page.  Note: if two part of one page are separately mapped, we
2103          * might have two guest_addr mapping to the same host paddr, but this
2104          * is not a big problem
2105          */
2106         ret = domain_page_mapping(domain, start_paddr,
2107                 ((u64)paddr) & PAGE_MASK, size, prot);
2108         if (ret)
2109                 goto error;
2110
2111         /* it's a non-present to present mapping */
2112         ret = iommu_flush_iotlb_psi(iommu, domain->id,
2113                         start_paddr, size >> VTD_PAGE_SHIFT, 1);
2114         if (ret)
2115                 iommu_flush_write_buffer(iommu);
2116
2117         return start_paddr + ((u64)paddr & (~PAGE_MASK));
2118
2119 error:
2120         if (iova)
2121                 __free_iova(&domain->iovad, iova);
2122         printk(KERN_ERR"Device %s request: %lx@%llx dir %d --- failed\n",
2123                 pci_name(pdev), size, (unsigned long long)paddr, dir);
2124         return 0;
2125 }
2126
2127 dma_addr_t intel_map_single(struct device *hwdev, phys_addr_t paddr,
2128                             size_t size, int dir)
2129 {
2130         return __intel_map_single(hwdev, paddr, size, dir,
2131                                   to_pci_dev(hwdev)->dma_mask);
2132 }
2133
2134 static void flush_unmaps(void)
2135 {
2136         int i, j;
2137
2138         timer_on = 0;
2139
2140         /* just flush them all */
2141         for (i = 0; i < g_num_of_iommus; i++) {
2142                 struct intel_iommu *iommu = g_iommus[i];
2143                 if (!iommu)
2144                         continue;
2145
2146                 if (deferred_flush[i].next) {
2147                         iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2148                                                  DMA_TLB_GLOBAL_FLUSH, 0);
2149                         for (j = 0; j < deferred_flush[i].next; j++) {
2150                                 __free_iova(&deferred_flush[i].domain[j]->iovad,
2151                                                 deferred_flush[i].iova[j]);
2152                         }
2153                         deferred_flush[i].next = 0;
2154                 }
2155         }
2156
2157         list_size = 0;
2158 }
2159
2160 static void flush_unmaps_timeout(unsigned long data)
2161 {
2162         unsigned long flags;
2163
2164         spin_lock_irqsave(&async_umap_flush_lock, flags);
2165         flush_unmaps();
2166         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2167 }
2168
2169 static void add_unmap(struct dmar_domain *dom, struct iova *iova)
2170 {
2171         unsigned long flags;
2172         int next, iommu_id;
2173         struct intel_iommu *iommu;
2174
2175         spin_lock_irqsave(&async_umap_flush_lock, flags);
2176         if (list_size == HIGH_WATER_MARK)
2177                 flush_unmaps();
2178
2179         iommu = domain_get_iommu(dom);
2180         iommu_id = iommu->seq_id;
2181
2182         next = deferred_flush[iommu_id].next;
2183         deferred_flush[iommu_id].domain[next] = dom;
2184         deferred_flush[iommu_id].iova[next] = iova;
2185         deferred_flush[iommu_id].next++;
2186
2187         if (!timer_on) {
2188                 mod_timer(&unmap_timer, jiffies + msecs_to_jiffies(10));
2189                 timer_on = 1;
2190         }
2191         list_size++;
2192         spin_unlock_irqrestore(&async_umap_flush_lock, flags);
2193 }
2194
2195 void intel_unmap_single(struct device *dev, dma_addr_t dev_addr, size_t size,
2196                         int dir)
2197 {
2198         struct pci_dev *pdev = to_pci_dev(dev);
2199         struct dmar_domain *domain;
2200         unsigned long start_addr;
2201         struct iova *iova;
2202         struct intel_iommu *iommu;
2203
2204         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2205                 return;
2206         domain = find_domain(pdev);
2207         BUG_ON(!domain);
2208
2209         iommu = domain_get_iommu(domain);
2210
2211         iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
2212         if (!iova)
2213                 return;
2214
2215         start_addr = iova->pfn_lo << PAGE_SHIFT;
2216         size = aligned_size((u64)dev_addr, size);
2217
2218         pr_debug("Device %s unmapping: %lx@%llx\n",
2219                 pci_name(pdev), size, (unsigned long long)start_addr);
2220
2221         /*  clear the whole page */
2222         dma_pte_clear_range(domain, start_addr, start_addr + size);
2223         /* free page tables */
2224         dma_pte_free_pagetable(domain, start_addr, start_addr + size);
2225         if (intel_iommu_strict) {
2226                 if (iommu_flush_iotlb_psi(iommu,
2227                         domain->id, start_addr, size >> VTD_PAGE_SHIFT, 0))
2228                         iommu_flush_write_buffer(iommu);
2229                 /* free iova */
2230                 __free_iova(&domain->iovad, iova);
2231         } else {
2232                 add_unmap(domain, iova);
2233                 /*
2234                  * queue up the release of the unmap to save the 1/6th of the
2235                  * cpu used up by the iotlb flush operation...
2236                  */
2237         }
2238 }
2239
2240 void *intel_alloc_coherent(struct device *hwdev, size_t size,
2241                            dma_addr_t *dma_handle, gfp_t flags)
2242 {
2243         void *vaddr;
2244         int order;
2245
2246         size = PAGE_ALIGN(size);
2247         order = get_order(size);
2248         flags &= ~(GFP_DMA | GFP_DMA32);
2249
2250         vaddr = (void *)__get_free_pages(flags, order);
2251         if (!vaddr)
2252                 return NULL;
2253         memset(vaddr, 0, size);
2254
2255         *dma_handle = __intel_map_single(hwdev, virt_to_bus(vaddr), size,
2256                                          DMA_BIDIRECTIONAL,
2257                                          hwdev->coherent_dma_mask);
2258         if (*dma_handle)
2259                 return vaddr;
2260         free_pages((unsigned long)vaddr, order);
2261         return NULL;
2262 }
2263
2264 void intel_free_coherent(struct device *hwdev, size_t size, void *vaddr,
2265                          dma_addr_t dma_handle)
2266 {
2267         int order;
2268
2269         size = PAGE_ALIGN(size);
2270         order = get_order(size);
2271
2272         intel_unmap_single(hwdev, dma_handle, size, DMA_BIDIRECTIONAL);
2273         free_pages((unsigned long)vaddr, order);
2274 }
2275
2276 #define SG_ENT_VIRT_ADDRESS(sg) (sg_virt((sg)))
2277
2278 void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
2279                     int nelems, int dir)
2280 {
2281         int i;
2282         struct pci_dev *pdev = to_pci_dev(hwdev);
2283         struct dmar_domain *domain;
2284         unsigned long start_addr;
2285         struct iova *iova;
2286         size_t size = 0;
2287         void *addr;
2288         struct scatterlist *sg;
2289         struct intel_iommu *iommu;
2290
2291         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2292                 return;
2293
2294         domain = find_domain(pdev);
2295         BUG_ON(!domain);
2296
2297         iommu = domain_get_iommu(domain);
2298
2299         iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2300         if (!iova)
2301                 return;
2302         for_each_sg(sglist, sg, nelems, i) {
2303                 addr = SG_ENT_VIRT_ADDRESS(sg);
2304                 size += aligned_size((u64)addr, sg->length);
2305         }
2306
2307         start_addr = iova->pfn_lo << PAGE_SHIFT;
2308
2309         /*  clear the whole page */
2310         dma_pte_clear_range(domain, start_addr, start_addr + size);
2311         /* free page tables */
2312         dma_pte_free_pagetable(domain, start_addr, start_addr + size);
2313
2314         if (iommu_flush_iotlb_psi(iommu, domain->id, start_addr,
2315                         size >> VTD_PAGE_SHIFT, 0))
2316                 iommu_flush_write_buffer(iommu);
2317
2318         /* free iova */
2319         __free_iova(&domain->iovad, iova);
2320 }
2321
2322 static int intel_nontranslate_map_sg(struct device *hddev,
2323         struct scatterlist *sglist, int nelems, int dir)
2324 {
2325         int i;
2326         struct scatterlist *sg;
2327
2328         for_each_sg(sglist, sg, nelems, i) {
2329                 BUG_ON(!sg_page(sg));
2330                 sg->dma_address = virt_to_bus(SG_ENT_VIRT_ADDRESS(sg));
2331                 sg->dma_length = sg->length;
2332         }
2333         return nelems;
2334 }
2335
2336 int intel_map_sg(struct device *hwdev, struct scatterlist *sglist, int nelems,
2337                  int dir)
2338 {
2339         void *addr;
2340         int i;
2341         struct pci_dev *pdev = to_pci_dev(hwdev);
2342         struct dmar_domain *domain;
2343         size_t size = 0;
2344         int prot = 0;
2345         size_t offset = 0;
2346         struct iova *iova = NULL;
2347         int ret;
2348         struct scatterlist *sg;
2349         unsigned long start_addr;
2350         struct intel_iommu *iommu;
2351
2352         BUG_ON(dir == DMA_NONE);
2353         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2354                 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
2355
2356         domain = get_valid_domain_for_dev(pdev);
2357         if (!domain)
2358                 return 0;
2359
2360         iommu = domain_get_iommu(domain);
2361
2362         for_each_sg(sglist, sg, nelems, i) {
2363                 addr = SG_ENT_VIRT_ADDRESS(sg);
2364                 addr = (void *)virt_to_phys(addr);
2365                 size += aligned_size((u64)addr, sg->length);
2366         }
2367
2368         iova = __intel_alloc_iova(hwdev, domain, size, pdev->dma_mask);
2369         if (!iova) {
2370                 sglist->dma_length = 0;
2371                 return 0;
2372         }
2373
2374         /*
2375          * Check if DMAR supports zero-length reads on write only
2376          * mappings..
2377          */
2378         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2379                         !cap_zlr(iommu->cap))
2380                 prot |= DMA_PTE_READ;
2381         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2382                 prot |= DMA_PTE_WRITE;
2383
2384         start_addr = iova->pfn_lo << PAGE_SHIFT;
2385         offset = 0;
2386         for_each_sg(sglist, sg, nelems, i) {
2387                 addr = SG_ENT_VIRT_ADDRESS(sg);
2388                 addr = (void *)virt_to_phys(addr);
2389                 size = aligned_size((u64)addr, sg->length);
2390                 ret = domain_page_mapping(domain, start_addr + offset,
2391                         ((u64)addr) & PAGE_MASK,
2392                         size, prot);
2393                 if (ret) {
2394                         /*  clear the page */
2395                         dma_pte_clear_range(domain, start_addr,
2396                                   start_addr + offset);
2397                         /* free page tables */
2398                         dma_pte_free_pagetable(domain, start_addr,
2399                                   start_addr + offset);
2400                         /* free iova */
2401                         __free_iova(&domain->iovad, iova);
2402                         return 0;
2403                 }
2404                 sg->dma_address = start_addr + offset +
2405                                 ((u64)addr & (~PAGE_MASK));
2406                 sg->dma_length = sg->length;
2407                 offset += size;
2408         }
2409
2410         /* it's a non-present to present mapping */
2411         if (iommu_flush_iotlb_psi(iommu, domain->id,
2412                         start_addr, offset >> VTD_PAGE_SHIFT, 1))
2413                 iommu_flush_write_buffer(iommu);
2414         return nelems;
2415 }
2416
2417 static struct dma_mapping_ops intel_dma_ops = {
2418         .alloc_coherent = intel_alloc_coherent,
2419         .free_coherent = intel_free_coherent,
2420         .map_single = intel_map_single,
2421         .unmap_single = intel_unmap_single,
2422         .map_sg = intel_map_sg,
2423         .unmap_sg = intel_unmap_sg,
2424 };
2425
2426 static inline int iommu_domain_cache_init(void)
2427 {
2428         int ret = 0;
2429
2430         iommu_domain_cache = kmem_cache_create("iommu_domain",
2431                                          sizeof(struct dmar_domain),
2432                                          0,
2433                                          SLAB_HWCACHE_ALIGN,
2434
2435                                          NULL);
2436         if (!iommu_domain_cache) {
2437                 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2438                 ret = -ENOMEM;
2439         }
2440
2441         return ret;
2442 }
2443
2444 static inline int iommu_devinfo_cache_init(void)
2445 {
2446         int ret = 0;
2447
2448         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2449                                          sizeof(struct device_domain_info),
2450                                          0,
2451                                          SLAB_HWCACHE_ALIGN,
2452                                          NULL);
2453         if (!iommu_devinfo_cache) {
2454                 printk(KERN_ERR "Couldn't create devinfo cache\n");
2455                 ret = -ENOMEM;
2456         }
2457
2458         return ret;
2459 }
2460
2461 static inline int iommu_iova_cache_init(void)
2462 {
2463         int ret = 0;
2464
2465         iommu_iova_cache = kmem_cache_create("iommu_iova",
2466                                          sizeof(struct iova),
2467                                          0,
2468                                          SLAB_HWCACHE_ALIGN,
2469                                          NULL);
2470         if (!iommu_iova_cache) {
2471                 printk(KERN_ERR "Couldn't create iova cache\n");
2472                 ret = -ENOMEM;
2473         }
2474
2475         return ret;
2476 }
2477
2478 static int __init iommu_init_mempool(void)
2479 {
2480         int ret;
2481         ret = iommu_iova_cache_init();
2482         if (ret)
2483                 return ret;
2484
2485         ret = iommu_domain_cache_init();
2486         if (ret)
2487                 goto domain_error;
2488
2489         ret = iommu_devinfo_cache_init();
2490         if (!ret)
2491                 return ret;
2492
2493         kmem_cache_destroy(iommu_domain_cache);
2494 domain_error:
2495         kmem_cache_destroy(iommu_iova_cache);
2496
2497         return -ENOMEM;
2498 }
2499
2500 static void __init iommu_exit_mempool(void)
2501 {
2502         kmem_cache_destroy(iommu_devinfo_cache);
2503         kmem_cache_destroy(iommu_domain_cache);
2504         kmem_cache_destroy(iommu_iova_cache);
2505
2506 }
2507
2508 static void __init init_no_remapping_devices(void)
2509 {
2510         struct dmar_drhd_unit *drhd;
2511
2512         for_each_drhd_unit(drhd) {
2513                 if (!drhd->include_all) {
2514                         int i;
2515                         for (i = 0; i < drhd->devices_cnt; i++)
2516                                 if (drhd->devices[i] != NULL)
2517                                         break;
2518                         /* ignore DMAR unit if no pci devices exist */
2519                         if (i == drhd->devices_cnt)
2520                                 drhd->ignored = 1;
2521                 }
2522         }
2523
2524         if (dmar_map_gfx)
2525                 return;
2526
2527         for_each_drhd_unit(drhd) {
2528                 int i;
2529                 if (drhd->ignored || drhd->include_all)
2530                         continue;
2531
2532                 for (i = 0; i < drhd->devices_cnt; i++)
2533                         if (drhd->devices[i] &&
2534                                 !IS_GFX_DEVICE(drhd->devices[i]))
2535                                 break;
2536
2537                 if (i < drhd->devices_cnt)
2538                         continue;
2539
2540                 /* bypass IOMMU if it is just for gfx devices */
2541                 drhd->ignored = 1;
2542                 for (i = 0; i < drhd->devices_cnt; i++) {
2543                         if (!drhd->devices[i])
2544                                 continue;
2545                         drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
2546                 }
2547         }
2548 }
2549
2550 int __init intel_iommu_init(void)
2551 {
2552         int ret = 0;
2553
2554         if (dmar_table_init())
2555                 return  -ENODEV;
2556
2557         if (dmar_dev_scope_init())
2558                 return  -ENODEV;
2559
2560         /*
2561          * Check the need for DMA-remapping initialization now.
2562          * Above initialization will also be used by Interrupt-remapping.
2563          */
2564         if (no_iommu || swiotlb || dmar_disabled)
2565                 return -ENODEV;
2566
2567         iommu_init_mempool();
2568         dmar_init_reserved_ranges();
2569
2570         init_no_remapping_devices();
2571
2572         ret = init_dmars();
2573         if (ret) {
2574                 printk(KERN_ERR "IOMMU: dmar init failed\n");
2575                 put_iova_domain(&reserved_iova_list);
2576                 iommu_exit_mempool();
2577                 return ret;
2578         }
2579         printk(KERN_INFO
2580         "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
2581
2582         init_timer(&unmap_timer);
2583         force_iommu = 1;
2584         dma_ops = &intel_dma_ops;
2585
2586         register_iommu(&intel_iommu_ops);
2587
2588         return 0;
2589 }
2590
2591 static int vm_domain_add_dev_info(struct dmar_domain *domain,
2592                                   struct pci_dev *pdev)
2593 {
2594         struct device_domain_info *info;
2595         unsigned long flags;
2596
2597         info = alloc_devinfo_mem();
2598         if (!info)
2599                 return -ENOMEM;
2600
2601         info->bus = pdev->bus->number;
2602         info->devfn = pdev->devfn;
2603         info->dev = pdev;
2604         info->domain = domain;
2605
2606         spin_lock_irqsave(&device_domain_lock, flags);
2607         list_add(&info->link, &domain->devices);
2608         list_add(&info->global, &device_domain_list);
2609         pdev->dev.archdata.iommu = info;
2610         spin_unlock_irqrestore(&device_domain_lock, flags);
2611
2612         return 0;
2613 }
2614
2615 static void vm_domain_remove_one_dev_info(struct dmar_domain *domain,
2616                                           struct pci_dev *pdev)
2617 {
2618         struct device_domain_info *info;
2619         struct intel_iommu *iommu;
2620         unsigned long flags;
2621         int found = 0;
2622         struct list_head *entry, *tmp;
2623
2624         iommu = device_to_iommu(pdev->bus->number, pdev->devfn);
2625         if (!iommu)
2626                 return;
2627
2628         spin_lock_irqsave(&device_domain_lock, flags);
2629         list_for_each_safe(entry, tmp, &domain->devices) {
2630                 info = list_entry(entry, struct device_domain_info, link);
2631                 if (info->bus == pdev->bus->number &&
2632                     info->devfn == pdev->devfn) {
2633                         list_del(&info->link);
2634                         list_del(&info->global);
2635                         if (info->dev)
2636                                 info->dev->dev.archdata.iommu = NULL;
2637                         spin_unlock_irqrestore(&device_domain_lock, flags);
2638
2639                         iommu_detach_dev(iommu, info->bus, info->devfn);
2640                         free_devinfo_mem(info);
2641
2642                         spin_lock_irqsave(&device_domain_lock, flags);
2643
2644                         if (found)
2645                                 break;
2646                         else
2647                                 continue;
2648                 }
2649
2650                 /* if there is no other devices under the same iommu
2651                  * owned by this domain, clear this iommu in iommu_bmp
2652                  * update iommu count and coherency
2653                  */
2654                 if (device_to_iommu(info->bus, info->devfn) == iommu)
2655                         found = 1;
2656         }
2657
2658         if (found == 0) {
2659                 unsigned long tmp_flags;
2660                 spin_lock_irqsave(&domain->iommu_lock, tmp_flags);
2661                 clear_bit(iommu->seq_id, &domain->iommu_bmp);
2662                 domain->iommu_count--;
2663                 domain_update_iommu_coherency(domain);
2664                 spin_unlock_irqrestore(&domain->iommu_lock, tmp_flags);
2665         }
2666
2667         spin_unlock_irqrestore(&device_domain_lock, flags);
2668 }
2669
2670 static void vm_domain_remove_all_dev_info(struct dmar_domain *domain)
2671 {
2672         struct device_domain_info *info;
2673         struct intel_iommu *iommu;
2674         unsigned long flags1, flags2;
2675
2676         spin_lock_irqsave(&device_domain_lock, flags1);
2677         while (!list_empty(&domain->devices)) {
2678                 info = list_entry(domain->devices.next,
2679                         struct device_domain_info, link);
2680                 list_del(&info->link);
2681                 list_del(&info->global);
2682                 if (info->dev)
2683                         info->dev->dev.archdata.iommu = NULL;
2684
2685                 spin_unlock_irqrestore(&device_domain_lock, flags1);
2686
2687                 iommu = device_to_iommu(info->bus, info->devfn);
2688                 iommu_detach_dev(iommu, info->bus, info->devfn);
2689
2690                 /* clear this iommu in iommu_bmp, update iommu count
2691                  * and coherency
2692                  */
2693                 spin_lock_irqsave(&domain->iommu_lock, flags2);
2694                 if (test_and_clear_bit(iommu->seq_id,
2695                                        &domain->iommu_bmp)) {
2696                         domain->iommu_count--;
2697                         domain_update_iommu_coherency(domain);
2698                 }
2699                 spin_unlock_irqrestore(&domain->iommu_lock, flags2);
2700
2701                 free_devinfo_mem(info);
2702                 spin_lock_irqsave(&device_domain_lock, flags1);
2703         }
2704         spin_unlock_irqrestore(&device_domain_lock, flags1);
2705 }
2706
2707 /* domain id for virtual machine, it won't be set in context */
2708 static unsigned long vm_domid;
2709
2710 static int vm_domain_min_agaw(struct dmar_domain *domain)
2711 {
2712         int i;
2713         int min_agaw = domain->agaw;
2714
2715         i = find_first_bit(&domain->iommu_bmp, g_num_of_iommus);
2716         for (; i < g_num_of_iommus; ) {
2717                 if (min_agaw > g_iommus[i]->agaw)
2718                         min_agaw = g_iommus[i]->agaw;
2719
2720                 i = find_next_bit(&domain->iommu_bmp, g_num_of_iommus, i+1);
2721         }
2722
2723         return min_agaw;
2724 }
2725
2726 static struct dmar_domain *iommu_alloc_vm_domain(void)
2727 {
2728         struct dmar_domain *domain;
2729
2730         domain = alloc_domain_mem();
2731         if (!domain)
2732                 return NULL;
2733
2734         domain->id = vm_domid++;
2735         memset(&domain->iommu_bmp, 0, sizeof(unsigned long));
2736         domain->flags = DOMAIN_FLAG_VIRTUAL_MACHINE;
2737
2738         return domain;
2739 }
2740
2741 static int vm_domain_init(struct dmar_domain *domain, int guest_width)
2742 {
2743         int adjust_width;
2744
2745         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
2746         spin_lock_init(&domain->mapping_lock);
2747         spin_lock_init(&domain->iommu_lock);
2748
2749         domain_reserve_special_ranges(domain);
2750
2751         /* calculate AGAW */
2752         domain->gaw = guest_width;
2753         adjust_width = guestwidth_to_adjustwidth(guest_width);
2754         domain->agaw = width_to_agaw(adjust_width);
2755
2756         INIT_LIST_HEAD(&domain->devices);
2757
2758         domain->iommu_count = 0;
2759         domain->iommu_coherency = 0;
2760         domain->max_addr = 0;
2761
2762         /* always allocate the top pgd */
2763         domain->pgd = (struct dma_pte *)alloc_pgtable_page();
2764         if (!domain->pgd)
2765                 return -ENOMEM;
2766         domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
2767         return 0;
2768 }
2769
2770 static void iommu_free_vm_domain(struct dmar_domain *domain)
2771 {
2772         unsigned long flags;
2773         struct dmar_drhd_unit *drhd;
2774         struct intel_iommu *iommu;
2775         unsigned long i;
2776         unsigned long ndomains;
2777
2778         for_each_drhd_unit(drhd) {
2779                 if (drhd->ignored)
2780                         continue;
2781                 iommu = drhd->iommu;
2782
2783                 ndomains = cap_ndoms(iommu->cap);
2784                 i = find_first_bit(iommu->domain_ids, ndomains);
2785                 for (; i < ndomains; ) {
2786                         if (iommu->domains[i] == domain) {
2787                                 spin_lock_irqsave(&iommu->lock, flags);
2788                                 clear_bit(i, iommu->domain_ids);
2789                                 iommu->domains[i] = NULL;
2790                                 spin_unlock_irqrestore(&iommu->lock, flags);
2791                                 break;
2792                         }
2793                         i = find_next_bit(iommu->domain_ids, ndomains, i+1);
2794                 }
2795         }
2796 }
2797
2798 static void vm_domain_exit(struct dmar_domain *domain)
2799 {
2800         u64 end;
2801
2802         /* Domain 0 is reserved, so dont process it */
2803         if (!domain)
2804                 return;
2805
2806         vm_domain_remove_all_dev_info(domain);
2807         /* destroy iovas */
2808         put_iova_domain(&domain->iovad);
2809         end = DOMAIN_MAX_ADDR(domain->gaw);
2810         end = end & (~VTD_PAGE_MASK);
2811
2812         /* clear ptes */
2813         dma_pte_clear_range(domain, 0, end);
2814
2815         /* free page tables */
2816         dma_pte_free_pagetable(domain, 0, end);
2817
2818         iommu_free_vm_domain(domain);
2819         free_domain_mem(domain);
2820 }
2821
2822 static int intel_iommu_domain_init(struct iommu_domain *domain)
2823 {
2824         struct dmar_domain *dmar_domain;
2825
2826         dmar_domain = iommu_alloc_vm_domain();
2827         if (!dmar_domain) {
2828                 printk(KERN_ERR
2829                         "intel_iommu_domain_init: dmar_domain == NULL\n");
2830                 return -ENOMEM;
2831         }
2832         if (vm_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2833                 printk(KERN_ERR
2834                         "intel_iommu_domain_init() failed\n");
2835                 vm_domain_exit(dmar_domain);
2836                 return -ENOMEM;
2837         }
2838         domain->priv = dmar_domain;
2839
2840         return 0;
2841 }
2842
2843 static void intel_iommu_domain_destroy(struct iommu_domain *domain)
2844 {
2845         struct dmar_domain *dmar_domain = domain->priv;
2846
2847         domain->priv = NULL;
2848         vm_domain_exit(dmar_domain);
2849 }
2850
2851 static int intel_iommu_attach_device(struct iommu_domain *domain,
2852                                      struct device *dev)
2853 {
2854         struct dmar_domain *dmar_domain = domain->priv;
2855         struct pci_dev *pdev = to_pci_dev(dev);
2856         struct intel_iommu *iommu;
2857         int addr_width;
2858         u64 end;
2859         int ret;
2860
2861         /* normally pdev is not mapped */
2862         if (unlikely(domain_context_mapped(pdev))) {
2863                 struct dmar_domain *old_domain;
2864
2865                 old_domain = find_domain(pdev);
2866                 if (old_domain) {
2867                         if (dmar_domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE)
2868                                 vm_domain_remove_one_dev_info(old_domain, pdev);
2869                         else
2870                                 domain_remove_dev_info(old_domain);
2871                 }
2872         }
2873
2874         iommu = device_to_iommu(pdev->bus->number, pdev->devfn);
2875         if (!iommu)
2876                 return -ENODEV;
2877
2878         /* check if this iommu agaw is sufficient for max mapped address */
2879         addr_width = agaw_to_width(iommu->agaw);
2880         end = DOMAIN_MAX_ADDR(addr_width);
2881         end = end & VTD_PAGE_MASK;
2882         if (end < dmar_domain->max_addr) {
2883                 printk(KERN_ERR "%s: iommu agaw (%d) is not "
2884                        "sufficient for the mapped address (%llx)\n",
2885                        __func__, iommu->agaw, dmar_domain->max_addr);
2886                 return -EFAULT;
2887         }
2888
2889         ret = domain_context_mapping(dmar_domain, pdev);
2890         if (ret)
2891                 return ret;
2892
2893         ret = vm_domain_add_dev_info(dmar_domain, pdev);
2894         return ret;
2895 }
2896
2897 static void intel_iommu_detach_device(struct iommu_domain *domain,
2898                                       struct device *dev)
2899 {
2900         struct dmar_domain *dmar_domain = domain->priv;
2901         struct pci_dev *pdev = to_pci_dev(dev);
2902
2903         vm_domain_remove_one_dev_info(dmar_domain, pdev);
2904 }
2905
2906 static int intel_iommu_map_range(struct iommu_domain *domain,
2907                                  unsigned long iova, phys_addr_t hpa,
2908                                  size_t size, int iommu_prot)
2909 {
2910         struct dmar_domain *dmar_domain = domain->priv;
2911         u64 max_addr;
2912         int addr_width;
2913         int prot = 0;
2914         int ret;
2915
2916         if (iommu_prot & IOMMU_READ)
2917                 prot |= DMA_PTE_READ;
2918         if (iommu_prot & IOMMU_WRITE)
2919                 prot |= DMA_PTE_WRITE;
2920
2921         max_addr = (iova & VTD_PAGE_MASK) + VTD_PAGE_ALIGN(size);
2922         if (dmar_domain->max_addr < max_addr) {
2923                 int min_agaw;
2924                 u64 end;
2925
2926                 /* check if minimum agaw is sufficient for mapped address */
2927                 min_agaw = vm_domain_min_agaw(dmar_domain);
2928                 addr_width = agaw_to_width(min_agaw);
2929                 end = DOMAIN_MAX_ADDR(addr_width);
2930                 end = end & VTD_PAGE_MASK;
2931                 if (end < max_addr) {
2932                         printk(KERN_ERR "%s: iommu agaw (%d) is not "
2933                                "sufficient for the mapped address (%llx)\n",
2934                                __func__, min_agaw, max_addr);
2935                         return -EFAULT;
2936                 }
2937                 dmar_domain->max_addr = max_addr;
2938         }
2939
2940         ret = domain_page_mapping(dmar_domain, iova, hpa, size, prot);
2941         return ret;
2942 }
2943
2944 static void intel_iommu_unmap_range(struct iommu_domain *domain,
2945                                     unsigned long iova, size_t size)
2946 {
2947         struct dmar_domain *dmar_domain = domain->priv;
2948         dma_addr_t base;
2949
2950         /* The address might not be aligned */
2951         base = iova & VTD_PAGE_MASK;
2952         size = VTD_PAGE_ALIGN(size);
2953         dma_pte_clear_range(dmar_domain, base, base + size);
2954
2955         if (dmar_domain->max_addr == base + size)
2956                 dmar_domain->max_addr = base;
2957 }
2958
2959 static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
2960                                             unsigned long iova)
2961 {
2962         struct dmar_domain *dmar_domain = domain->priv;
2963         struct dma_pte *pte;
2964         u64 phys = 0;
2965
2966         pte = addr_to_dma_pte(dmar_domain, iova);
2967         if (pte)
2968                 phys = dma_pte_addr(pte);
2969
2970         return phys;
2971 }
2972
2973 static struct iommu_ops intel_iommu_ops = {
2974         .domain_init    = intel_iommu_domain_init,
2975         .domain_destroy = intel_iommu_domain_destroy,
2976         .attach_dev     = intel_iommu_attach_device,
2977         .detach_dev     = intel_iommu_detach_device,
2978         .map            = intel_iommu_map_range,
2979         .unmap          = intel_iommu_unmap_range,
2980         .iova_to_phys   = intel_iommu_iova_to_phys,
2981 };
2982
2983 static void __devinit quirk_iommu_rwbf(struct pci_dev *dev)
2984 {
2985         /*
2986          * Mobile 4 Series Chipset neglects to set RWBF capability,
2987          * but needs it:
2988          */
2989         printk(KERN_INFO "DMAR: Forcing write-buffer flush capability\n");
2990         rwbf_quirk = 1;
2991 }
2992
2993 DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);