drivers/pci/intel-iommu.c

   1 /*
   2  * Copyright (c) 2006, Intel Corporation.
   3  *
   4  * This program is free software; you can redistribute it and/or modify it
   5  * under the terms and conditions of the GNU General Public License,
   6  * version 2, as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope it will be useful, but WITHOUT
   9  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11  * more details.
  12  *
  13  * You should have received a copy of the GNU General Public License along with
  14  * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15  * Place - Suite 330, Boston, MA 02111-1307 USA.
  16  *
  17  * Copyright (C) 2006-2008 Intel Corporation
  18  * Author: Ashok Raj <ashok.raj@intel.com>
  19  * Author: Shaohua Li <shaohua.li@intel.com>
  20  * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21  */
  22
  23 #include <linux/init.h>
  24 #include <linux/bitmap.h>
  25 #include <linux/slab.h>
  26 #include <linux/irq.h>
  27 #include <linux/interrupt.h>
  28 #include <linux/sysdev.h>
  29 #include <linux/spinlock.h>
  30 #include <linux/pci.h>
  31 #include <linux/dmar.h>
  32 #include <linux/dma-mapping.h>
  33 #include <linux/mempool.h>
  34 #include "iova.h"
  35 #include "intel-iommu.h"
  36 #include <asm/proto.h> /* force_iommu in this header in x86-64*/
  37 #include <asm/cacheflush.h>
  38 #include <asm/gart.h>
  39 #include "pci.h"
  40
  41 #define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
  42 #define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
  43
  44 #define IOAPIC_RANGE_START      (0xfee00000)
  45 #define IOAPIC_RANGE_END        (0xfeefffff)
  46 #define IOVA_START_ADDR         (0x1000)
  47
  48 #define DEFAULT_DOMAIN_ADDRESS_WIDTH 48
  49
  50 #define DMAR_OPERATION_TIMEOUT (HZ*60) /* 1m */
  51
  52 #define DOMAIN_MAX_ADDR(gaw) ((((u64)1) << gaw) - 1)
  53
  54 static void domain_remove_dev_info(struct dmar_domain *domain);
  55
  56 static int dmar_disabled;
  57 static int __initdata dmar_map_gfx = 1;
  58 static int dmar_forcedac;
  59
  60 #define DUMMY_DEVICE_DOMAIN_INFO ((struct device_domain_info *)(-1))
  61 static DEFINE_SPINLOCK(device_domain_lock);
  62 static LIST_HEAD(device_domain_list);
  63
  64 static int __init intel_iommu_setup(char *str)
  65 {
  66         if (!str)
  67                 return -EINVAL;
  68         while (*str) {
  69                 if (!strncmp(str, "off", 3)) {
  70                         dmar_disabled = 1;
  71                         printk(KERN_INFO"Intel-IOMMU: disabled\n");
  72                 } else if (!strncmp(str, "igfx_off", 8)) {
  73                         dmar_map_gfx = 0;
  74                         printk(KERN_INFO
  75                                 "Intel-IOMMU: disable GFX device mapping\n");
  76                 } else if (!strncmp(str, "forcedac", 8)) {
  77                         printk (KERN_INFO
  78                                 "Intel-IOMMU: Forcing DAC for PCI devices\n");
  79                         dmar_forcedac = 1;
  80                 }
  81
  82                 str += strcspn(str, ",");
  83                 while (*str == ',')
  84                         str++;
  85         }
  86         return 0;
  87 }
  88 __setup("intel_iommu=", intel_iommu_setup);
  89
  90 static struct kmem_cache *iommu_domain_cache;
  91 static struct kmem_cache *iommu_devinfo_cache;
  92 static struct kmem_cache *iommu_iova_cache;
  93
  94 static inline void *iommu_kmem_cache_alloc(struct kmem_cache *cachep)
  95 {
  96         unsigned int flags;
  97         void *vaddr;
  98
  99         /* trying to avoid low memory issues */
 100         flags = current->flags & PF_MEMALLOC;
 101         current->flags |= PF_MEMALLOC;
 102         vaddr = kmem_cache_alloc(cachep, GFP_ATOMIC);
 103         current->flags &= (~PF_MEMALLOC | flags);
 104         return vaddr;
 105 }
 106
 107
 108 static inline void *alloc_pgtable_page(void)
 109 {
 110         unsigned int flags;
 111         void *vaddr;
 112
 113         /* trying to avoid low memory issues */
 114         flags = current->flags & PF_MEMALLOC;
 115         current->flags |= PF_MEMALLOC;
 116         vaddr = (void *)get_zeroed_page(GFP_ATOMIC);
 117         current->flags &= (~PF_MEMALLOC | flags);
 118         return vaddr;
 119 }
 120
 121 static inline void free_pgtable_page(void *vaddr)
 122 {
 123         free_page((unsigned long)vaddr);
 124 }
 125
 126 static inline void *alloc_domain_mem(void)
 127 {
 128         return iommu_kmem_cache_alloc(iommu_domain_cache);
 129 }
 130
 131 static inline void free_domain_mem(void *vaddr)
 132 {
 133         kmem_cache_free(iommu_domain_cache, vaddr);
 134 }
 135
 136 static inline void * alloc_devinfo_mem(void)
 137 {
 138         return iommu_kmem_cache_alloc(iommu_devinfo_cache);
 139 }
 140
 141 static inline void free_devinfo_mem(void *vaddr)
 142 {
 143         kmem_cache_free(iommu_devinfo_cache, vaddr);
 144 }
 145
 146 struct iova *alloc_iova_mem(void)
 147 {
 148         return iommu_kmem_cache_alloc(iommu_iova_cache);
 149 }
 150
 151 void free_iova_mem(struct iova *iova)
 152 {
 153         kmem_cache_free(iommu_iova_cache, iova);
 154 }
 155
 156 static inline void __iommu_flush_cache(
 157         struct intel_iommu *iommu, void *addr, int size)
 158 {
 159         if (!ecap_coherent(iommu->ecap))
 160                 clflush_cache_range(addr, size);
 161 }
 162
 163 /* Gets context entry for a given bus and devfn */
 164 static struct context_entry * device_to_context_entry(struct intel_iommu *iommu,
 165                 u8 bus, u8 devfn)
 166 {
 167         struct root_entry *root;
 168         struct context_entry *context;
 169         unsigned long phy_addr;
 170         unsigned long flags;
 171
 172         spin_lock_irqsave(&iommu->lock, flags);
 173         root = &iommu->root_entry[bus];
 174         context = get_context_addr_from_root(root);
 175         if (!context) {
 176                 context = (struct context_entry *)alloc_pgtable_page();
 177                 if (!context) {
 178                         spin_unlock_irqrestore(&iommu->lock, flags);
 179                         return NULL;
 180                 }
 181                 __iommu_flush_cache(iommu, (void *)context, PAGE_SIZE_4K);
 182                 phy_addr = virt_to_phys((void *)context);
 183                 set_root_value(root, phy_addr);
 184                 set_root_present(root);
 185                 __iommu_flush_cache(iommu, root, sizeof(*root));
 186         }
 187         spin_unlock_irqrestore(&iommu->lock, flags);
 188         return &context[devfn];
 189 }
 190
 191 static int device_context_mapped(struct intel_iommu *iommu, u8 bus, u8 devfn)
 192 {
 193         struct root_entry *root;
 194         struct context_entry *context;
 195         int ret;
 196         unsigned long flags;
 197
 198         spin_lock_irqsave(&iommu->lock, flags);
 199         root = &iommu->root_entry[bus];
 200         context = get_context_addr_from_root(root);
 201         if (!context) {
 202                 ret = 0;
 203                 goto out;
 204         }
 205         ret = context_present(context[devfn]);
 206 out:
 207         spin_unlock_irqrestore(&iommu->lock, flags);
 208         return ret;
 209 }
 210
 211 static void clear_context_table(struct intel_iommu *iommu, u8 bus, u8 devfn)
 212 {
 213         struct root_entry *root;
 214         struct context_entry *context;
 215         unsigned long flags;
 216
 217         spin_lock_irqsave(&iommu->lock, flags);
 218         root = &iommu->root_entry[bus];
 219         context = get_context_addr_from_root(root);
 220         if (context) {
 221                 context_clear_entry(context[devfn]);
 222                 __iommu_flush_cache(iommu, &context[devfn], \
 223                         sizeof(*context));
 224         }
 225         spin_unlock_irqrestore(&iommu->lock, flags);
 226 }
 227
 228 static void free_context_table(struct intel_iommu *iommu)
 229 {
 230         struct root_entry *root;
 231         int i;
 232         unsigned long flags;
 233         struct context_entry *context;
 234
 235         spin_lock_irqsave(&iommu->lock, flags);
 236         if (!iommu->root_entry) {
 237                 goto out;
 238         }
 239         for (i = 0; i < ROOT_ENTRY_NR; i++) {
 240                 root = &iommu->root_entry[i];
 241                 context = get_context_addr_from_root(root);
 242                 if (context)
 243                         free_pgtable_page(context);
 244         }
 245         free_pgtable_page(iommu->root_entry);
 246         iommu->root_entry = NULL;
 247 out:
 248         spin_unlock_irqrestore(&iommu->lock, flags);
 249 }
 250
 251 /* page table handling */
 252 #define LEVEL_STRIDE            (9)
 253 #define LEVEL_MASK              (((u64)1 << LEVEL_STRIDE) - 1)
 254
 255 static inline int agaw_to_level(int agaw)
 256 {
 257         return agaw + 2;
 258 }
 259
 260 static inline int agaw_to_width(int agaw)
 261 {
 262         return 30 + agaw * LEVEL_STRIDE;
 263
 264 }
 265
 266 static inline int width_to_agaw(int width)
 267 {
 268         return (width - 30) / LEVEL_STRIDE;
 269 }
 270
 271 static inline unsigned int level_to_offset_bits(int level)
 272 {
 273         return (12 + (level - 1) * LEVEL_STRIDE);
 274 }
 275
 276 static inline int address_level_offset(u64 addr, int level)
 277 {
 278         return ((addr >> level_to_offset_bits(level)) & LEVEL_MASK);
 279 }
 280
 281 static inline u64 level_mask(int level)
 282 {
 283         return ((u64)-1 << level_to_offset_bits(level));
 284 }
 285
 286 static inline u64 level_size(int level)
 287 {
 288         return ((u64)1 << level_to_offset_bits(level));
 289 }
 290
 291 static inline u64 align_to_level(u64 addr, int level)
 292 {
 293         return ((addr + level_size(level) - 1) & level_mask(level));
 294 }
 295
 296 static struct dma_pte * addr_to_dma_pte(struct dmar_domain *domain, u64 addr)
 297 {
 298         int addr_width = agaw_to_width(domain->agaw);
 299         struct dma_pte *parent, *pte = NULL;
 300         int level = agaw_to_level(domain->agaw);
 301         int offset;
 302         unsigned long flags;
 303
 304         BUG_ON(!domain->pgd);
 305
 306         addr &= (((u64)1) << addr_width) - 1;
 307         parent = domain->pgd;
 308
 309         spin_lock_irqsave(&domain->mapping_lock, flags);
 310         while (level > 0) {
 311                 void *tmp_page;
 312
 313                 offset = address_level_offset(addr, level);
 314                 pte = &parent[offset];
 315                 if (level == 1)
 316                         break;
 317
 318                 if (!dma_pte_present(*pte)) {
 319                         tmp_page = alloc_pgtable_page();
 320
 321                         if (!tmp_page) {
 322                                 spin_unlock_irqrestore(&domain->mapping_lock,
 323                                         flags);
 324                                 return NULL;
 325                         }
 326                         __iommu_flush_cache(domain->iommu, tmp_page,
 327                                         PAGE_SIZE_4K);
 328                         dma_set_pte_addr(*pte, virt_to_phys(tmp_page));
 329                         /*
 330                          * high level table always sets r/w, last level page
 331                          * table control read/write
 332                          */
 333                         dma_set_pte_readable(*pte);
 334                         dma_set_pte_writable(*pte);
 335                         __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
 336                 }
 337                 parent = phys_to_virt(dma_pte_addr(*pte));
 338                 level--;
 339         }
 340
 341         spin_unlock_irqrestore(&domain->mapping_lock, flags);
 342         return pte;
 343 }
 344
 345 /* return address's pte at specific level */
 346 static struct dma_pte *dma_addr_level_pte(struct dmar_domain *domain, u64 addr,
 347                 int level)
 348 {
 349         struct dma_pte *parent, *pte = NULL;
 350         int total = agaw_to_level(domain->agaw);
 351         int offset;
 352
 353         parent = domain->pgd;
 354         while (level <= total) {
 355                 offset = address_level_offset(addr, total);
 356                 pte = &parent[offset];
 357                 if (level == total)
 358                         return pte;
 359
 360                 if (!dma_pte_present(*pte))
 361                         break;
 362                 parent = phys_to_virt(dma_pte_addr(*pte));
 363                 total--;
 364         }
 365         return NULL;
 366 }
 367
 368 /* clear one page's page table */
 369 static void dma_pte_clear_one(struct dmar_domain *domain, u64 addr)
 370 {
 371         struct dma_pte *pte = NULL;
 372
 373         /* get last level pte */
 374         pte = dma_addr_level_pte(domain, addr, 1);
 375
 376         if (pte) {
 377                 dma_clear_pte(*pte);
 378                 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
 379         }
 380 }
 381
 382 /* clear last level pte, a tlb flush should be followed */
 383 static void dma_pte_clear_range(struct dmar_domain *domain, u64 start, u64 end)
 384 {
 385         int addr_width = agaw_to_width(domain->agaw);
 386
 387         start &= (((u64)1) << addr_width) - 1;
 388         end &= (((u64)1) << addr_width) - 1;
 389         /* in case it's partial page */
 390         start = PAGE_ALIGN_4K(start);
 391         end &= PAGE_MASK_4K;
 392
 393         /* we don't need lock here, nobody else touches the iova range */
 394         while (start < end) {
 395                 dma_pte_clear_one(domain, start);
 396                 start += PAGE_SIZE_4K;
 397         }
 398 }
 399
 400 /* free page table pages. last level pte should already be cleared */
 401 static void dma_pte_free_pagetable(struct dmar_domain *domain,
 402         u64 start, u64 end)
 403 {
 404         int addr_width = agaw_to_width(domain->agaw);
 405         struct dma_pte *pte;
 406         int total = agaw_to_level(domain->agaw);
 407         int level;
 408         u64 tmp;
 409
 410         start &= (((u64)1) << addr_width) - 1;
 411         end &= (((u64)1) << addr_width) - 1;
 412
 413         /* we don't need lock here, nobody else touches the iova range */
 414         level = 2;
 415         while (level <= total) {
 416                 tmp = align_to_level(start, level);
 417                 if (tmp >= end || (tmp + level_size(level) > end))
 418                         return;
 419
 420                 while (tmp < end) {
 421                         pte = dma_addr_level_pte(domain, tmp, level);
 422                         if (pte) {
 423                                 free_pgtable_page(
 424                                         phys_to_virt(dma_pte_addr(*pte)));
 425                                 dma_clear_pte(*pte);
 426                                 __iommu_flush_cache(domain->iommu,
 427                                                 pte, sizeof(*pte));
 428                         }
 429                         tmp += level_size(level);
 430                 }
 431                 level++;
 432         }
 433         /* free pgd */
 434         if (start == 0 && end >= ((((u64)1) << addr_width) - 1)) {
 435                 free_pgtable_page(domain->pgd);
 436                 domain->pgd = NULL;
 437         }
 438 }
 439
 440 /* iommu handling */
 441 static int iommu_alloc_root_entry(struct intel_iommu *iommu)
 442 {
 443         struct root_entry *root;
 444         unsigned long flags;
 445
 446         root = (struct root_entry *)alloc_pgtable_page();
 447         if (!root)
 448                 return -ENOMEM;
 449
 450         __iommu_flush_cache(iommu, root, PAGE_SIZE_4K);
 451
 452         spin_lock_irqsave(&iommu->lock, flags);
 453         iommu->root_entry = root;
 454         spin_unlock_irqrestore(&iommu->lock, flags);
 455
 456         return 0;
 457 }
 458
 459 #define IOMMU_WAIT_OP(iommu, offset, op, cond, sts) \
 460 {\
 461         unsigned long start_time = jiffies;\
 462         while (1) {\
 463                 sts = op (iommu->reg + offset);\
 464                 if (cond)\
 465                         break;\
 466                 if (time_after(jiffies, start_time + DMAR_OPERATION_TIMEOUT))\
 467                         panic("DMAR hardware is malfunctioning\n");\
 468                 cpu_relax();\
 469         }\
 470 }
 471
 472 static void iommu_set_root_entry(struct intel_iommu *iommu)
 473 {
 474         void *addr;
 475         u32 cmd, sts;
 476         unsigned long flag;
 477
 478         addr = iommu->root_entry;
 479
 480         spin_lock_irqsave(&iommu->register_lock, flag);
 481         dmar_writeq(iommu->reg + DMAR_RTADDR_REG, virt_to_phys(addr));
 482
 483         cmd = iommu->gcmd | DMA_GCMD_SRTP;
 484         writel(cmd, iommu->reg + DMAR_GCMD_REG);
 485
 486         /* Make sure hardware complete it */
 487         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 488                 readl, (sts & DMA_GSTS_RTPS), sts);
 489
 490         spin_unlock_irqrestore(&iommu->register_lock, flag);
 491 }
 492
 493 static void iommu_flush_write_buffer(struct intel_iommu *iommu)
 494 {
 495         u32 val;
 496         unsigned long flag;
 497
 498         if (!cap_rwbf(iommu->cap))
 499                 return;
 500         val = iommu->gcmd | DMA_GCMD_WBF;
 501
 502         spin_lock_irqsave(&iommu->register_lock, flag);
 503         writel(val, iommu->reg + DMAR_GCMD_REG);
 504
 505         /* Make sure hardware complete it */
 506         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 507                         readl, (!(val & DMA_GSTS_WBFS)), val);
 508
 509         spin_unlock_irqrestore(&iommu->register_lock, flag);
 510 }
 511
 512 /* return value determine if we need a write buffer flush */
 513 static int __iommu_flush_context(struct intel_iommu *iommu,
 514         u16 did, u16 source_id, u8 function_mask, u64 type,
 515         int non_present_entry_flush)
 516 {
 517         u64 val = 0;
 518         unsigned long flag;
 519
 520         /*
 521          * In the non-present entry flush case, if hardware doesn't cache
 522          * non-present entry we do nothing and if hardware cache non-present
 523          * entry, we flush entries of domain 0 (the domain id is used to cache
 524          * any non-present entries)
 525          */
 526         if (non_present_entry_flush) {
 527                 if (!cap_caching_mode(iommu->cap))
 528                         return 1;
 529                 else
 530                         did = 0;
 531         }
 532
 533         switch (type) {
 534         case DMA_CCMD_GLOBAL_INVL:
 535                 val = DMA_CCMD_GLOBAL_INVL;
 536                 break;
 537         case DMA_CCMD_DOMAIN_INVL:
 538                 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
 539                 break;
 540         case DMA_CCMD_DEVICE_INVL:
 541                 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
 542                         | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
 543                 break;
 544         default:
 545                 BUG();
 546         }
 547         val |= DMA_CCMD_ICC;
 548
 549         spin_lock_irqsave(&iommu->register_lock, flag);
 550         dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
 551
 552         /* Make sure hardware complete it */
 553         IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
 554                 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
 555
 556         spin_unlock_irqrestore(&iommu->register_lock, flag);
 557
 558         /* flush context entry will implictly flush write buffer */
 559         return 0;
 560 }
 561
 562 static int inline iommu_flush_context_global(struct intel_iommu *iommu,
 563         int non_present_entry_flush)
 564 {
 565         return __iommu_flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL,
 566                 non_present_entry_flush);
 567 }
 568
 569 static int inline iommu_flush_context_domain(struct intel_iommu *iommu, u16 did,
 570         int non_present_entry_flush)
 571 {
 572         return __iommu_flush_context(iommu, did, 0, 0, DMA_CCMD_DOMAIN_INVL,
 573                 non_present_entry_flush);
 574 }
 575
 576 static int inline iommu_flush_context_device(struct intel_iommu *iommu,
 577         u16 did, u16 source_id, u8 function_mask, int non_present_entry_flush)
 578 {
 579         return __iommu_flush_context(iommu, did, source_id, function_mask,
 580                 DMA_CCMD_DEVICE_INVL, non_present_entry_flush);
 581 }
 582
 583 /* return value determine if we need a write buffer flush */
 584 static int __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
 585         u64 addr, unsigned int size_order, u64 type,
 586         int non_present_entry_flush)
 587 {
 588         int tlb_offset = ecap_iotlb_offset(iommu->ecap);
 589         u64 val = 0, val_iva = 0;
 590         unsigned long flag;
 591
 592         /*
 593          * In the non-present entry flush case, if hardware doesn't cache
 594          * non-present entry we do nothing and if hardware cache non-present
 595          * entry, we flush entries of domain 0 (the domain id is used to cache
 596          * any non-present entries)
 597          */
 598         if (non_present_entry_flush) {
 599                 if (!cap_caching_mode(iommu->cap))
 600                         return 1;
 601                 else
 602                         did = 0;
 603         }
 604
 605         switch (type) {
 606         case DMA_TLB_GLOBAL_FLUSH:
 607                 /* global flush doesn't need set IVA_REG */
 608                 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
 609                 break;
 610         case DMA_TLB_DSI_FLUSH:
 611                 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
 612                 break;
 613         case DMA_TLB_PSI_FLUSH:
 614                 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
 615                 /* Note: always flush non-leaf currently */
 616                 val_iva = size_order | addr;
 617                 break;
 618         default:
 619                 BUG();
 620         }
 621         /* Note: set drain read/write */
 622 #if 0
 623         /*
 624          * This is probably to be super secure.. Looks like we can
 625          * ignore it without any impact.
 626          */
 627         if (cap_read_drain(iommu->cap))
 628                 val |= DMA_TLB_READ_DRAIN;
 629 #endif
 630         if (cap_write_drain(iommu->cap))
 631                 val |= DMA_TLB_WRITE_DRAIN;
 632
 633         spin_lock_irqsave(&iommu->register_lock, flag);
 634         /* Note: Only uses first TLB reg currently */
 635         if (val_iva)
 636                 dmar_writeq(iommu->reg + tlb_offset, val_iva);
 637         dmar_writeq(iommu->reg + tlb_offset + 8, val);
 638
 639         /* Make sure hardware complete it */
 640         IOMMU_WAIT_OP(iommu, tlb_offset + 8,
 641                 dmar_readq, (!(val & DMA_TLB_IVT)), val);
 642
 643         spin_unlock_irqrestore(&iommu->register_lock, flag);
 644
 645         /* check IOTLB invalidation granularity */
 646         if (DMA_TLB_IAIG(val) == 0)
 647                 printk(KERN_ERR"IOMMU: flush IOTLB failed\n");
 648         if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
 649                 pr_debug("IOMMU: tlb flush request %Lx, actual %Lx\n",
 650                         DMA_TLB_IIRG(type), DMA_TLB_IAIG(val));
 651         /* flush context entry will implictly flush write buffer */
 652         return 0;
 653 }
 654
 655 static int inline iommu_flush_iotlb_global(struct intel_iommu *iommu,
 656         int non_present_entry_flush)
 657 {
 658         return __iommu_flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH,
 659                 non_present_entry_flush);
 660 }
 661
 662 static int inline iommu_flush_iotlb_dsi(struct intel_iommu *iommu, u16 did,
 663         int non_present_entry_flush)
 664 {
 665         return __iommu_flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH,
 666                 non_present_entry_flush);
 667 }
 668
 669 static int iommu_flush_iotlb_psi(struct intel_iommu *iommu, u16 did,
 670         u64 addr, unsigned int pages, int non_present_entry_flush)
 671 {
 672         unsigned int mask;
 673
 674         BUG_ON(addr & (~PAGE_MASK_4K));
 675         BUG_ON(pages == 0);
 676
 677         /* Fallback to domain selective flush if no PSI support */
 678         if (!cap_pgsel_inv(iommu->cap))
 679                 return iommu_flush_iotlb_dsi(iommu, did,
 680                         non_present_entry_flush);
 681
 682         /*
 683          * PSI requires page size to be 2 ^ x, and the base address is naturally
 684          * aligned to the size
 685          */
 686         mask = ilog2(__roundup_pow_of_two(pages));
 687         /* Fallback to domain selective flush if size is too big */
 688         if (mask > cap_max_amask_val(iommu->cap))
 689                 return iommu_flush_iotlb_dsi(iommu, did,
 690                         non_present_entry_flush);
 691
 692         return __iommu_flush_iotlb(iommu, did, addr, mask,
 693                 DMA_TLB_PSI_FLUSH, non_present_entry_flush);
 694 }
 695
 696 static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
 697 {
 698         u32 pmen;
 699         unsigned long flags;
 700
 701         spin_lock_irqsave(&iommu->register_lock, flags);
 702         pmen = readl(iommu->reg + DMAR_PMEN_REG);
 703         pmen &= ~DMA_PMEN_EPM;
 704         writel(pmen, iommu->reg + DMAR_PMEN_REG);
 705
 706         /* wait for the protected region status bit to clear */
 707         IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
 708                 readl, !(pmen & DMA_PMEN_PRS), pmen);
 709
 710         spin_unlock_irqrestore(&iommu->register_lock, flags);
 711 }
 712
 713 static int iommu_enable_translation(struct intel_iommu *iommu)
 714 {
 715         u32 sts;
 716         unsigned long flags;
 717
 718         spin_lock_irqsave(&iommu->register_lock, flags);
 719         writel(iommu->gcmd|DMA_GCMD_TE, iommu->reg + DMAR_GCMD_REG);
 720
 721         /* Make sure hardware complete it */
 722         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 723                 readl, (sts & DMA_GSTS_TES), sts);
 724
 725         iommu->gcmd |= DMA_GCMD_TE;
 726         spin_unlock_irqrestore(&iommu->register_lock, flags);
 727         return 0;
 728 }
 729
 730 static int iommu_disable_translation(struct intel_iommu *iommu)
 731 {
 732         u32 sts;
 733         unsigned long flag;
 734
 735         spin_lock_irqsave(&iommu->register_lock, flag);
 736         iommu->gcmd &= ~DMA_GCMD_TE;
 737         writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
 738
 739         /* Make sure hardware complete it */
 740         IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
 741                 readl, (!(sts & DMA_GSTS_TES)), sts);
 742
 743         spin_unlock_irqrestore(&iommu->register_lock, flag);
 744         return 0;
 745 }
 746
 747 /* iommu interrupt handling. Most stuff are MSI-like. */
 748
 749 static const char *fault_reason_strings[] =
 750 {
 751         "Software",
 752         "Present bit in root entry is clear",
 753         "Present bit in context entry is clear",
 754         "Invalid context entry",
 755         "Access beyond MGAW",
 756         "PTE Write access is not set",
 757         "PTE Read access is not set",
 758         "Next page table ptr is invalid",
 759         "Root table address invalid",
 760         "Context table ptr is invalid",
 761         "non-zero reserved fields in RTP",
 762         "non-zero reserved fields in CTP",
 763         "non-zero reserved fields in PTE",
 764 };
 765 #define MAX_FAULT_REASON_IDX    (ARRAY_SIZE(fault_reason_strings) - 1)
 766
 767 const char *dmar_get_fault_reason(u8 fault_reason)
 768 {
 769         if (fault_reason > MAX_FAULT_REASON_IDX)
 770                 return "Unknown";
 771         else
 772                 return fault_reason_strings[fault_reason];
 773 }
 774
 775 void dmar_msi_unmask(unsigned int irq)
 776 {
 777         struct intel_iommu *iommu = get_irq_data(irq);
 778         unsigned long flag;
 779
 780         /* unmask it */
 781         spin_lock_irqsave(&iommu->register_lock, flag);
 782         writel(0, iommu->reg + DMAR_FECTL_REG);
 783         /* Read a reg to force flush the post write */
 784         readl(iommu->reg + DMAR_FECTL_REG);
 785         spin_unlock_irqrestore(&iommu->register_lock, flag);
 786 }
 787
 788 void dmar_msi_mask(unsigned int irq)
 789 {
 790         unsigned long flag;
 791         struct intel_iommu *iommu = get_irq_data(irq);
 792
 793         /* mask it */
 794         spin_lock_irqsave(&iommu->register_lock, flag);
 795         writel(DMA_FECTL_IM, iommu->reg + DMAR_FECTL_REG);
 796         /* Read a reg to force flush the post write */
 797         readl(iommu->reg + DMAR_FECTL_REG);
 798         spin_unlock_irqrestore(&iommu->register_lock, flag);
 799 }
 800
 801 void dmar_msi_write(int irq, struct msi_msg *msg)
 802 {
 803         struct intel_iommu *iommu = get_irq_data(irq);
 804         unsigned long flag;
 805
 806         spin_lock_irqsave(&iommu->register_lock, flag);
 807         writel(msg->data, iommu->reg + DMAR_FEDATA_REG);
 808         writel(msg->address_lo, iommu->reg + DMAR_FEADDR_REG);
 809         writel(msg->address_hi, iommu->reg + DMAR_FEUADDR_REG);
 810         spin_unlock_irqrestore(&iommu->register_lock, flag);
 811 }
 812
 813 void dmar_msi_read(int irq, struct msi_msg *msg)
 814 {
 815         struct intel_iommu *iommu = get_irq_data(irq);
 816         unsigned long flag;
 817
 818         spin_lock_irqsave(&iommu->register_lock, flag);
 819         msg->data = readl(iommu->reg + DMAR_FEDATA_REG);
 820         msg->address_lo = readl(iommu->reg + DMAR_FEADDR_REG);
 821         msg->address_hi = readl(iommu->reg + DMAR_FEUADDR_REG);
 822         spin_unlock_irqrestore(&iommu->register_lock, flag);
 823 }
 824
 825 static int iommu_page_fault_do_one(struct intel_iommu *iommu, int type,
 826                 u8 fault_reason, u16 source_id, u64 addr)
 827 {
 828         const char *reason;
 829
 830         reason = dmar_get_fault_reason(fault_reason);
 831
 832         printk(KERN_ERR
 833                 "DMAR:[%s] Request device [%02x:%02x.%d] "
 834                 "fault addr %llx \n"
 835                 "DMAR:[fault reason %02d] %s\n",
 836                 (type ? "DMA Read" : "DMA Write"),
 837                 (source_id >> 8), PCI_SLOT(source_id & 0xFF),
 838                 PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
 839         return 0;
 840 }
 841
 842 #define PRIMARY_FAULT_REG_LEN (16)
 843 static irqreturn_t iommu_page_fault(int irq, void *dev_id)
 844 {
 845         struct intel_iommu *iommu = dev_id;
 846         int reg, fault_index;
 847         u32 fault_status;
 848         unsigned long flag;
 849
 850         spin_lock_irqsave(&iommu->register_lock, flag);
 851         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
 852
 853         /* TBD: ignore advanced fault log currently */
 854         if (!(fault_status & DMA_FSTS_PPF))
 855                 goto clear_overflow;
 856
 857         fault_index = dma_fsts_fault_record_index(fault_status);
 858         reg = cap_fault_reg_offset(iommu->cap);
 859         while (1) {
 860                 u8 fault_reason;
 861                 u16 source_id;
 862                 u64 guest_addr;
 863                 int type;
 864                 u32 data;
 865
 866                 /* highest 32 bits */
 867                 data = readl(iommu->reg + reg +
 868                                 fault_index * PRIMARY_FAULT_REG_LEN + 12);
 869                 if (!(data & DMA_FRCD_F))
 870                         break;
 871
 872                 fault_reason = dma_frcd_fault_reason(data);
 873                 type = dma_frcd_type(data);
 874
 875                 data = readl(iommu->reg + reg +
 876                                 fault_index * PRIMARY_FAULT_REG_LEN + 8);
 877                 source_id = dma_frcd_source_id(data);
 878
 879                 guest_addr = dmar_readq(iommu->reg + reg +
 880                                 fault_index * PRIMARY_FAULT_REG_LEN);
 881                 guest_addr = dma_frcd_page_addr(guest_addr);
 882                 /* clear the fault */
 883                 writel(DMA_FRCD_F, iommu->reg + reg +
 884                         fault_index * PRIMARY_FAULT_REG_LEN + 12);
 885
 886                 spin_unlock_irqrestore(&iommu->register_lock, flag);
 887
 888                 iommu_page_fault_do_one(iommu, type, fault_reason,
 889                                 source_id, guest_addr);
 890
 891                 fault_index++;
 892                 if (fault_index > cap_num_fault_regs(iommu->cap))
 893                         fault_index = 0;
 894                 spin_lock_irqsave(&iommu->register_lock, flag);
 895         }
 896 clear_overflow:
 897         /* clear primary fault overflow */
 898         fault_status = readl(iommu->reg + DMAR_FSTS_REG);
 899         if (fault_status & DMA_FSTS_PFO)
 900                 writel(DMA_FSTS_PFO, iommu->reg + DMAR_FSTS_REG);
 901
 902         spin_unlock_irqrestore(&iommu->register_lock, flag);
 903         return IRQ_HANDLED;
 904 }
 905
 906 int dmar_set_interrupt(struct intel_iommu *iommu)
 907 {
 908         int irq, ret;
 909
 910         irq = create_irq();
 911         if (!irq) {
 912                 printk(KERN_ERR "IOMMU: no free vectors\n");
 913                 return -EINVAL;
 914         }
 915
 916         set_irq_data(irq, iommu);
 917         iommu->irq = irq;
 918
 919         ret = arch_setup_dmar_msi(irq);
 920         if (ret) {
 921                 set_irq_data(irq, NULL);
 922                 iommu->irq = 0;
 923                 destroy_irq(irq);
 924                 return 0;
 925         }
 926
 927         /* Force fault register is cleared */
 928         iommu_page_fault(irq, iommu);
 929
 930         ret = request_irq(irq, iommu_page_fault, 0, iommu->name, iommu);
 931         if (ret)
 932                 printk(KERN_ERR "IOMMU: can't request irq\n");
 933         return ret;
 934 }
 935
 936 static int iommu_init_domains(struct intel_iommu *iommu)
 937 {
 938         unsigned long ndomains;
 939         unsigned long nlongs;
 940
 941         ndomains = cap_ndoms(iommu->cap);
 942         pr_debug("Number of Domains supportd <%ld>\n", ndomains);
 943         nlongs = BITS_TO_LONGS(ndomains);
 944
 945         /* TBD: there might be 64K domains,
 946          * consider other allocation for future chip
 947          */
 948         iommu->domain_ids = kcalloc(nlongs, sizeof(unsigned long), GFP_KERNEL);
 949         if (!iommu->domain_ids) {
 950                 printk(KERN_ERR "Allocating domain id array failed\n");
 951                 return -ENOMEM;
 952         }
 953         iommu->domains = kcalloc(ndomains, sizeof(struct dmar_domain *),
 954                         GFP_KERNEL);
 955         if (!iommu->domains) {
 956                 printk(KERN_ERR "Allocating domain array failed\n");
 957                 kfree(iommu->domain_ids);
 958                 return -ENOMEM;
 959         }
 960
 961         /*
 962          * if Caching mode is set, then invalid translations are tagged
 963          * with domainid 0. Hence we need to pre-allocate it.
 964          */
 965         if (cap_caching_mode(iommu->cap))
 966                 set_bit(0, iommu->domain_ids);
 967         return 0;
 968 }
 969
 970 static struct intel_iommu *alloc_iommu(struct dmar_drhd_unit *drhd)
 971 {
 972         struct intel_iommu *iommu;
 973         int ret;
 974         int map_size;
 975         u32 ver;
 976
 977         iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
 978         if (!iommu)
 979                 return NULL;
 980         iommu->reg = ioremap(drhd->reg_base_addr, PAGE_SIZE_4K);
 981         if (!iommu->reg) {
 982                 printk(KERN_ERR "IOMMU: can't map the region\n");
 983                 goto error;
 984         }
 985         iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 986         iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 987
 988         /* the registers might be more than one page */
 989         map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 990                 cap_max_fault_reg_offset(iommu->cap));
 991         map_size = PAGE_ALIGN_4K(map_size);
 992         if (map_size > PAGE_SIZE_4K) {
 993                 iounmap(iommu->reg);
 994                 iommu->reg = ioremap(drhd->reg_base_addr, map_size);
 995                 if (!iommu->reg) {
 996                         printk(KERN_ERR "IOMMU: can't map the region\n");
 997                         goto error;
 998                 }
 999         }
1000
1001         ver = readl(iommu->reg + DMAR_VER_REG);
1002         pr_debug("IOMMU %llx: ver %d:%d cap %llx ecap %llx\n",
1003                 drhd->reg_base_addr, DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
1004                 iommu->cap, iommu->ecap);
1005         ret = iommu_init_domains(iommu);
1006         if (ret)
1007                 goto error_unmap;
1008         spin_lock_init(&iommu->lock);
1009         spin_lock_init(&iommu->register_lock);
1010
1011         drhd->iommu = iommu;
1012         return iommu;
1013 error_unmap:
1014         iounmap(iommu->reg);
1015 error:
1016         kfree(iommu);
1017         return NULL;
1018 }
1019
1020 static void domain_exit(struct dmar_domain *domain);
1021 static void free_iommu(struct intel_iommu *iommu)
1022 {
1023         struct dmar_domain *domain;
1024         int i;
1025
1026         if (!iommu)
1027                 return;
1028
1029         i = find_first_bit(iommu->domain_ids, cap_ndoms(iommu->cap));
1030         for (; i < cap_ndoms(iommu->cap); ) {
1031                 domain = iommu->domains[i];
1032                 clear_bit(i, iommu->domain_ids);
1033                 domain_exit(domain);
1034                 i = find_next_bit(iommu->domain_ids,
1035                         cap_ndoms(iommu->cap), i+1);
1036         }
1037
1038         if (iommu->gcmd & DMA_GCMD_TE)
1039                 iommu_disable_translation(iommu);
1040
1041         if (iommu->irq) {
1042                 set_irq_data(iommu->irq, NULL);
1043                 /* This will mask the irq */
1044                 free_irq(iommu->irq, iommu);
1045                 destroy_irq(iommu->irq);
1046         }
1047
1048         kfree(iommu->domains);
1049         kfree(iommu->domain_ids);
1050
1051         /* free context mapping */
1052         free_context_table(iommu);
1053
1054         if (iommu->reg)
1055                 iounmap(iommu->reg);
1056         kfree(iommu);
1057 }
1058
1059 static struct dmar_domain * iommu_alloc_domain(struct intel_iommu *iommu)
1060 {
1061         unsigned long num;
1062         unsigned long ndomains;
1063         struct dmar_domain *domain;
1064         unsigned long flags;
1065
1066         domain = alloc_domain_mem();
1067         if (!domain)
1068                 return NULL;
1069
1070         ndomains = cap_ndoms(iommu->cap);
1071
1072         spin_lock_irqsave(&iommu->lock, flags);
1073         num = find_first_zero_bit(iommu->domain_ids, ndomains);
1074         if (num >= ndomains) {
1075                 spin_unlock_irqrestore(&iommu->lock, flags);
1076                 free_domain_mem(domain);
1077                 printk(KERN_ERR "IOMMU: no free domain ids\n");
1078                 return NULL;
1079         }
1080
1081         set_bit(num, iommu->domain_ids);
1082         domain->id = num;
1083         domain->iommu = iommu;
1084         iommu->domains[num] = domain;
1085         spin_unlock_irqrestore(&iommu->lock, flags);
1086
1087         return domain;
1088 }
1089
1090 static void iommu_free_domain(struct dmar_domain *domain)
1091 {
1092         unsigned long flags;
1093
1094         spin_lock_irqsave(&domain->iommu->lock, flags);
1095         clear_bit(domain->id, domain->iommu->domain_ids);
1096         spin_unlock_irqrestore(&domain->iommu->lock, flags);
1097 }
1098
1099 static struct iova_domain reserved_iova_list;
1100 static struct lock_class_key reserved_alloc_key;
1101 static struct lock_class_key reserved_rbtree_key;
1102
1103 static void dmar_init_reserved_ranges(void)
1104 {
1105         struct pci_dev *pdev = NULL;
1106         struct iova *iova;
1107         int i;
1108         u64 addr, size;
1109
1110         init_iova_domain(&reserved_iova_list, DMA_32BIT_PFN);
1111
1112         lockdep_set_class(&reserved_iova_list.iova_alloc_lock,
1113                 &reserved_alloc_key);
1114         lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
1115                 &reserved_rbtree_key);
1116
1117         /* IOAPIC ranges shouldn't be accessed by DMA */
1118         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
1119                 IOVA_PFN(IOAPIC_RANGE_END));
1120         if (!iova)
1121                 printk(KERN_ERR "Reserve IOAPIC range failed\n");
1122
1123         /* Reserve all PCI MMIO to avoid peer-to-peer access */
1124         for_each_pci_dev(pdev) {
1125                 struct resource *r;
1126
1127                 for (i = 0; i < PCI_NUM_RESOURCES; i++) {
1128                         r = &pdev->resource[i];
1129                         if (!r->flags || !(r->flags & IORESOURCE_MEM))
1130                                 continue;
1131                         addr = r->start;
1132                         addr &= PAGE_MASK_4K;
1133                         size = r->end - addr;
1134                         size = PAGE_ALIGN_4K(size);
1135                         iova = reserve_iova(&reserved_iova_list, IOVA_PFN(addr),
1136                                 IOVA_PFN(size + addr) - 1);
1137                         if (!iova)
1138                                 printk(KERN_ERR "Reserve iova failed\n");
1139                 }
1140         }
1141
1142 }
1143
1144 static void domain_reserve_special_ranges(struct dmar_domain *domain)
1145 {
1146         copy_reserved_iova(&reserved_iova_list, &domain->iovad);
1147 }
1148
1149 static inline int guestwidth_to_adjustwidth(int gaw)
1150 {
1151         int agaw;
1152         int r = (gaw - 12) % 9;
1153
1154         if (r == 0)
1155                 agaw = gaw;
1156         else
1157                 agaw = gaw + 9 - r;
1158         if (agaw > 64)
1159                 agaw = 64;
1160         return agaw;
1161 }
1162
1163 static int domain_init(struct dmar_domain *domain, int guest_width)
1164 {
1165         struct intel_iommu *iommu;
1166         int adjust_width, agaw;
1167         unsigned long sagaw;
1168
1169         init_iova_domain(&domain->iovad, DMA_32BIT_PFN);
1170         spin_lock_init(&domain->mapping_lock);
1171
1172         domain_reserve_special_ranges(domain);
1173
1174         /* calculate AGAW */
1175         iommu = domain->iommu;
1176         if (guest_width > cap_mgaw(iommu->cap))
1177                 guest_width = cap_mgaw(iommu->cap);
1178         domain->gaw = guest_width;
1179         adjust_width = guestwidth_to_adjustwidth(guest_width);
1180         agaw = width_to_agaw(adjust_width);
1181         sagaw = cap_sagaw(iommu->cap);
1182         if (!test_bit(agaw, &sagaw)) {
1183                 /* hardware doesn't support it, choose a bigger one */
1184                 pr_debug("IOMMU: hardware doesn't support agaw %d\n", agaw);
1185                 agaw = find_next_bit(&sagaw, 5, agaw);
1186                 if (agaw >= 5)
1187                         return -ENODEV;
1188         }
1189         domain->agaw = agaw;
1190         INIT_LIST_HEAD(&domain->devices);
1191
1192         /* always allocate the top pgd */
1193         domain->pgd = (struct dma_pte *)alloc_pgtable_page();
1194         if (!domain->pgd)
1195                 return -ENOMEM;
1196         __iommu_flush_cache(iommu, domain->pgd, PAGE_SIZE_4K);
1197         return 0;
1198 }
1199
1200 static void domain_exit(struct dmar_domain *domain)
1201 {
1202         u64 end;
1203
1204         /* Domain 0 is reserved, so dont process it */
1205         if (!domain)
1206                 return;
1207
1208         domain_remove_dev_info(domain);
1209         /* destroy iovas */
1210         put_iova_domain(&domain->iovad);
1211         end = DOMAIN_MAX_ADDR(domain->gaw);
1212         end = end & (~PAGE_MASK_4K);
1213
1214         /* clear ptes */
1215         dma_pte_clear_range(domain, 0, end);
1216
1217         /* free page tables */
1218         dma_pte_free_pagetable(domain, 0, end);
1219
1220         iommu_free_domain(domain);
1221         free_domain_mem(domain);
1222 }
1223
1224 static int domain_context_mapping_one(struct dmar_domain *domain,
1225                 u8 bus, u8 devfn)
1226 {
1227         struct context_entry *context;
1228         struct intel_iommu *iommu = domain->iommu;
1229         unsigned long flags;
1230
1231         pr_debug("Set context mapping for %02x:%02x.%d\n",
1232                 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1233         BUG_ON(!domain->pgd);
1234         context = device_to_context_entry(iommu, bus, devfn);
1235         if (!context)
1236                 return -ENOMEM;
1237         spin_lock_irqsave(&iommu->lock, flags);
1238         if (context_present(*context)) {
1239                 spin_unlock_irqrestore(&iommu->lock, flags);
1240                 return 0;
1241         }
1242
1243         context_set_domain_id(*context, domain->id);
1244         context_set_address_width(*context, domain->agaw);
1245         context_set_address_root(*context, virt_to_phys(domain->pgd));
1246         context_set_translation_type(*context, CONTEXT_TT_MULTI_LEVEL);
1247         context_set_fault_enable(*context);
1248         context_set_present(*context);
1249         __iommu_flush_cache(iommu, context, sizeof(*context));
1250
1251         /* it's a non-present to present mapping */
1252         if (iommu_flush_context_device(iommu, domain->id,
1253                         (((u16)bus) << 8) | devfn, DMA_CCMD_MASK_NOBIT, 1))
1254                 iommu_flush_write_buffer(iommu);
1255         else
1256                 iommu_flush_iotlb_dsi(iommu, 0, 0);
1257         spin_unlock_irqrestore(&iommu->lock, flags);
1258         return 0;
1259 }
1260
1261 static int
1262 domain_context_mapping(struct dmar_domain *domain, struct pci_dev *pdev)
1263 {
1264         int ret;
1265         struct pci_dev *tmp, *parent;
1266
1267         ret = domain_context_mapping_one(domain, pdev->bus->number,
1268                 pdev->devfn);
1269         if (ret)
1270                 return ret;
1271
1272         /* dependent device mapping */
1273         tmp = pci_find_upstream_pcie_bridge(pdev);
1274         if (!tmp)
1275                 return 0;
1276         /* Secondary interface's bus number and devfn 0 */
1277         parent = pdev->bus->self;
1278         while (parent != tmp) {
1279                 ret = domain_context_mapping_one(domain, parent->bus->number,
1280                         parent->devfn);
1281                 if (ret)
1282                         return ret;
1283                 parent = parent->bus->self;
1284         }
1285         if (tmp->is_pcie) /* this is a PCIE-to-PCI bridge */
1286                 return domain_context_mapping_one(domain,
1287                         tmp->subordinate->number, 0);
1288         else /* this is a legacy PCI bridge */
1289                 return domain_context_mapping_one(domain,
1290                         tmp->bus->number, tmp->devfn);
1291 }
1292
1293 static int domain_context_mapped(struct dmar_domain *domain,
1294         struct pci_dev *pdev)
1295 {
1296         int ret;
1297         struct pci_dev *tmp, *parent;
1298
1299         ret = device_context_mapped(domain->iommu,
1300                 pdev->bus->number, pdev->devfn);
1301         if (!ret)
1302                 return ret;
1303         /* dependent device mapping */
1304         tmp = pci_find_upstream_pcie_bridge(pdev);
1305         if (!tmp)
1306                 return ret;
1307         /* Secondary interface's bus number and devfn 0 */
1308         parent = pdev->bus->self;
1309         while (parent != tmp) {
1310                 ret = device_context_mapped(domain->iommu, parent->bus->number,
1311                         parent->devfn);
1312                 if (!ret)
1313                         return ret;
1314                 parent = parent->bus->self;
1315         }
1316         if (tmp->is_pcie)
1317                 return device_context_mapped(domain->iommu,
1318                         tmp->subordinate->number, 0);
1319         else
1320                 return device_context_mapped(domain->iommu,
1321                         tmp->bus->number, tmp->devfn);
1322 }
1323
1324 static int
1325 domain_page_mapping(struct dmar_domain *domain, dma_addr_t iova,
1326                         u64 hpa, size_t size, int prot)
1327 {
1328         u64 start_pfn, end_pfn;
1329         struct dma_pte *pte;
1330         int index;
1331
1332         if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1333                 return -EINVAL;
1334         iova &= PAGE_MASK_4K;
1335         start_pfn = ((u64)hpa) >> PAGE_SHIFT_4K;
1336         end_pfn = (PAGE_ALIGN_4K(((u64)hpa) + size)) >> PAGE_SHIFT_4K;
1337         index = 0;
1338         while (start_pfn < end_pfn) {
1339                 pte = addr_to_dma_pte(domain, iova + PAGE_SIZE_4K * index);
1340                 if (!pte)
1341                         return -ENOMEM;
1342                 /* We don't need lock here, nobody else
1343                  * touches the iova range
1344                  */
1345                 BUG_ON(dma_pte_addr(*pte));
1346                 dma_set_pte_addr(*pte, start_pfn << PAGE_SHIFT_4K);
1347                 dma_set_pte_prot(*pte, prot);
1348                 __iommu_flush_cache(domain->iommu, pte, sizeof(*pte));
1349                 start_pfn++;
1350                 index++;
1351         }
1352         return 0;
1353 }
1354
1355 static void detach_domain_for_dev(struct dmar_domain *domain, u8 bus, u8 devfn)
1356 {
1357         clear_context_table(domain->iommu, bus, devfn);
1358         iommu_flush_context_global(domain->iommu, 0);
1359         iommu_flush_iotlb_global(domain->iommu, 0);
1360 }
1361
1362 static void domain_remove_dev_info(struct dmar_domain *domain)
1363 {
1364         struct device_domain_info *info;
1365         unsigned long flags;
1366
1367         spin_lock_irqsave(&device_domain_lock, flags);
1368         while (!list_empty(&domain->devices)) {
1369                 info = list_entry(domain->devices.next,
1370                         struct device_domain_info, link);
1371                 list_del(&info->link);
1372                 list_del(&info->global);
1373                 if (info->dev)
1374                         info->dev->dev.archdata.iommu = NULL;
1375                 spin_unlock_irqrestore(&device_domain_lock, flags);
1376
1377                 detach_domain_for_dev(info->domain, info->bus, info->devfn);
1378                 free_devinfo_mem(info);
1379
1380                 spin_lock_irqsave(&device_domain_lock, flags);
1381         }
1382         spin_unlock_irqrestore(&device_domain_lock, flags);
1383 }
1384
1385 /*
1386  * find_domain
1387  * Note: we use struct pci_dev->dev.archdata.iommu stores the info
1388  */
1389 struct dmar_domain *
1390 find_domain(struct pci_dev *pdev)
1391 {
1392         struct device_domain_info *info;
1393
1394         /* No lock here, assumes no domain exit in normal case */
1395         info = pdev->dev.archdata.iommu;
1396         if (info)
1397                 return info->domain;
1398         return NULL;
1399 }
1400
1401 static int dmar_pci_device_match(struct pci_dev *devices[], int cnt,
1402      struct pci_dev *dev)
1403 {
1404         int index;
1405
1406         while (dev) {
1407                 for (index = 0; index < cnt; index ++)
1408                         if (dev == devices[index])
1409                                 return 1;
1410
1411                 /* Check our parent */
1412                 dev = dev->bus->self;
1413         }
1414
1415         return 0;
1416 }
1417
1418 static struct dmar_drhd_unit *
1419 dmar_find_matched_drhd_unit(struct pci_dev *dev)
1420 {
1421         struct dmar_drhd_unit *drhd = NULL;
1422
1423         list_for_each_entry(drhd, &dmar_drhd_units, list) {
1424                 if (drhd->include_all || dmar_pci_device_match(drhd->devices,
1425                                                 drhd->devices_cnt, dev))
1426                         return drhd;
1427         }
1428
1429         return NULL;
1430 }
1431
1432 /* domain is initialized */
1433 static struct dmar_domain *get_domain_for_dev(struct pci_dev *pdev, int gaw)
1434 {
1435         struct dmar_domain *domain, *found = NULL;
1436         struct intel_iommu *iommu;
1437         struct dmar_drhd_unit *drhd;
1438         struct device_domain_info *info, *tmp;
1439         struct pci_dev *dev_tmp;
1440         unsigned long flags;
1441         int bus = 0, devfn = 0;
1442
1443         domain = find_domain(pdev);
1444         if (domain)
1445                 return domain;
1446
1447         dev_tmp = pci_find_upstream_pcie_bridge(pdev);
1448         if (dev_tmp) {
1449                 if (dev_tmp->is_pcie) {
1450                         bus = dev_tmp->subordinate->number;
1451                         devfn = 0;
1452                 } else {
1453                         bus = dev_tmp->bus->number;
1454                         devfn = dev_tmp->devfn;
1455                 }
1456                 spin_lock_irqsave(&device_domain_lock, flags);
1457                 list_for_each_entry(info, &device_domain_list, global) {
1458                         if (info->bus == bus && info->devfn == devfn) {
1459                                 found = info->domain;
1460                                 break;
1461                         }
1462                 }
1463                 spin_unlock_irqrestore(&device_domain_lock, flags);
1464                 /* pcie-pci bridge already has a domain, uses it */
1465                 if (found) {
1466                         domain = found;
1467                         goto found_domain;
1468                 }
1469         }
1470
1471         /* Allocate new domain for the device */
1472         drhd = dmar_find_matched_drhd_unit(pdev);
1473         if (!drhd) {
1474                 printk(KERN_ERR "IOMMU: can't find DMAR for device %s\n",
1475                         pci_name(pdev));
1476                 return NULL;
1477         }
1478         iommu = drhd->iommu;
1479
1480         domain = iommu_alloc_domain(iommu);
1481         if (!domain)
1482                 goto error;
1483
1484         if (domain_init(domain, gaw)) {
1485                 domain_exit(domain);
1486                 goto error;
1487         }
1488
1489         /* register pcie-to-pci device */
1490         if (dev_tmp) {
1491                 info = alloc_devinfo_mem();
1492                 if (!info) {
1493                         domain_exit(domain);
1494                         goto error;
1495                 }
1496                 info->bus = bus;
1497                 info->devfn = devfn;
1498                 info->dev = NULL;
1499                 info->domain = domain;
1500                 /* This domain is shared by devices under p2p bridge */
1501                 domain->flags |= DOMAIN_FLAG_MULTIPLE_DEVICES;
1502
1503                 /* pcie-to-pci bridge already has a domain, uses it */
1504                 found = NULL;
1505                 spin_lock_irqsave(&device_domain_lock, flags);
1506                 list_for_each_entry(tmp, &device_domain_list, global) {
1507                         if (tmp->bus == bus && tmp->devfn == devfn) {
1508                                 found = tmp->domain;
1509                                 break;
1510                         }
1511                 }
1512                 if (found) {
1513                         free_devinfo_mem(info);
1514                         domain_exit(domain);
1515                         domain = found;
1516                 } else {
1517                         list_add(&info->link, &domain->devices);
1518                         list_add(&info->global, &device_domain_list);
1519                 }
1520                 spin_unlock_irqrestore(&device_domain_lock, flags);
1521         }
1522
1523 found_domain:
1524         info = alloc_devinfo_mem();
1525         if (!info)
1526                 goto error;
1527         info->bus = pdev->bus->number;
1528         info->devfn = pdev->devfn;
1529         info->dev = pdev;
1530         info->domain = domain;
1531         spin_lock_irqsave(&device_domain_lock, flags);
1532         /* somebody is fast */
1533         found = find_domain(pdev);
1534         if (found != NULL) {
1535                 spin_unlock_irqrestore(&device_domain_lock, flags);
1536                 if (found != domain) {
1537                         domain_exit(domain);
1538                         domain = found;
1539                 }
1540                 free_devinfo_mem(info);
1541                 return domain;
1542         }
1543         list_add(&info->link, &domain->devices);
1544         list_add(&info->global, &device_domain_list);
1545         pdev->dev.archdata.iommu = info;
1546         spin_unlock_irqrestore(&device_domain_lock, flags);
1547         return domain;
1548 error:
1549         /* recheck it here, maybe others set it */
1550         return find_domain(pdev);
1551 }
1552
1553 static int iommu_prepare_identity_map(struct pci_dev *pdev, u64 start, u64 end)
1554 {
1555         struct dmar_domain *domain;
1556         unsigned long size;
1557         u64 base;
1558         int ret;
1559
1560         printk(KERN_INFO
1561                 "IOMMU: Setting identity map for device %s [0x%Lx - 0x%Lx]\n",
1562                 pci_name(pdev), start, end);
1563         /* page table init */
1564         domain = get_domain_for_dev(pdev, DEFAULT_DOMAIN_ADDRESS_WIDTH);
1565         if (!domain)
1566                 return -ENOMEM;
1567
1568         /* The address might not be aligned */
1569         base = start & PAGE_MASK_4K;
1570         size = end - base;
1571         size = PAGE_ALIGN_4K(size);
1572         if (!reserve_iova(&domain->iovad, IOVA_PFN(base),
1573                         IOVA_PFN(base + size) - 1)) {
1574                 printk(KERN_ERR "IOMMU: reserve iova failed\n");
1575                 ret = -ENOMEM;
1576                 goto error;
1577         }
1578
1579         pr_debug("Mapping reserved region %lx@%llx for %s\n",
1580                 size, base, pci_name(pdev));
1581         /*
1582          * RMRR range might have overlap with physical memory range,
1583          * clear it first
1584          */
1585         dma_pte_clear_range(domain, base, base + size);
1586
1587         ret = domain_page_mapping(domain, base, base, size,
1588                 DMA_PTE_READ|DMA_PTE_WRITE);
1589         if (ret)
1590                 goto error;
1591
1592         /* context entry init */
1593         ret = domain_context_mapping(domain, pdev);
1594         if (!ret)
1595                 return 0;
1596 error:
1597         domain_exit(domain);
1598         return ret;
1599
1600 }
1601
1602 static inline int iommu_prepare_rmrr_dev(struct dmar_rmrr_unit *rmrr,
1603         struct pci_dev *pdev)
1604 {
1605         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1606                 return 0;
1607         return iommu_prepare_identity_map(pdev, rmrr->base_address,
1608                 rmrr->end_address + 1);
1609 }
1610
1611 #ifdef CONFIG_DMAR_GFX_WA
1612 extern int arch_get_ram_range(int slot, u64 *addr, u64 *size);
1613 static void __init iommu_prepare_gfx_mapping(void)
1614 {
1615         struct pci_dev *pdev = NULL;
1616         u64 base, size;
1617         int slot;
1618         int ret;
1619
1620         for_each_pci_dev(pdev) {
1621                 if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO ||
1622                                 !IS_GFX_DEVICE(pdev))
1623                         continue;
1624                 printk(KERN_INFO "IOMMU: gfx device %s 1-1 mapping\n",
1625                         pci_name(pdev));
1626                 slot = arch_get_ram_range(0, &base, &size);
1627                 while (slot >= 0) {
1628                         ret = iommu_prepare_identity_map(pdev,
1629                                         base, base + size);
1630                         if (ret)
1631                                 goto error;
1632                         slot = arch_get_ram_range(slot, &base, &size);
1633                 }
1634                 continue;
1635 error:
1636                 printk(KERN_ERR "IOMMU: mapping reserved region failed\n");
1637         }
1638 }
1639 #endif
1640
1641 #ifdef CONFIG_DMAR_FLOPPY_WA
1642 static inline void iommu_prepare_isa(void)
1643 {
1644         struct pci_dev *pdev;
1645         int ret;
1646
1647         pdev = pci_get_class(PCI_CLASS_BRIDGE_ISA << 8, NULL);
1648         if (!pdev)
1649                 return;
1650
1651         printk(KERN_INFO "IOMMU: Prepare 0-16M unity mapping for LPC\n");
1652         ret = iommu_prepare_identity_map(pdev, 0, 16*1024*1024);
1653
1654         if (ret)
1655                 printk("IOMMU: Failed to create 0-64M identity map, "
1656                         "floppy might not work\n");
1657
1658 }
1659 #else
1660 static inline void iommu_prepare_isa(void)
1661 {
1662         return;
1663 }
1664 #endif /* !CONFIG_DMAR_FLPY_WA */
1665
1666 int __init init_dmars(void)
1667 {
1668         struct dmar_drhd_unit *drhd;
1669         struct dmar_rmrr_unit *rmrr;
1670         struct pci_dev *pdev;
1671         struct intel_iommu *iommu;
1672         int ret, unit = 0;
1673
1674         /*
1675          * for each drhd
1676          *    allocate root
1677          *    initialize and program root entry to not present
1678          * endfor
1679          */
1680         for_each_drhd_unit(drhd) {
1681                 if (drhd->ignored)
1682                         continue;
1683                 iommu = alloc_iommu(drhd);
1684                 if (!iommu) {
1685                         ret = -ENOMEM;
1686                         goto error;
1687                 }
1688
1689                 /*
1690                  * TBD:
1691                  * we could share the same root & context tables
1692                  * amoung all IOMMU's. Need to Split it later.
1693                  */
1694                 ret = iommu_alloc_root_entry(iommu);
1695                 if (ret) {
1696                         printk(KERN_ERR "IOMMU: allocate root entry failed\n");
1697                         goto error;
1698                 }
1699         }
1700
1701         /*
1702          * For each rmrr
1703          *   for each dev attached to rmrr
1704          *   do
1705          *     locate drhd for dev, alloc domain for dev
1706          *     allocate free domain
1707          *     allocate page table entries for rmrr
1708          *     if context not allocated for bus
1709          *           allocate and init context
1710          *           set present in root table for this bus
1711          *     init context with domain, translation etc
1712          *    endfor
1713          * endfor
1714          */
1715         for_each_rmrr_units(rmrr) {
1716                 int i;
1717                 for (i = 0; i < rmrr->devices_cnt; i++) {
1718                         pdev = rmrr->devices[i];
1719                         /* some BIOS lists non-exist devices in DMAR table */
1720                         if (!pdev)
1721                                 continue;
1722                         ret = iommu_prepare_rmrr_dev(rmrr, pdev);
1723                         if (ret)
1724                                 printk(KERN_ERR
1725                                  "IOMMU: mapping reserved region failed\n");
1726                 }
1727         }
1728
1729         iommu_prepare_gfx_mapping();
1730
1731         iommu_prepare_isa();
1732
1733         /*
1734          * for each drhd
1735          *   enable fault log
1736          *   global invalidate context cache
1737          *   global invalidate iotlb
1738          *   enable translation
1739          */
1740         for_each_drhd_unit(drhd) {
1741                 if (drhd->ignored)
1742                         continue;
1743                 iommu = drhd->iommu;
1744                 sprintf (iommu->name, "dmar%d", unit++);
1745
1746                 iommu_flush_write_buffer(iommu);
1747
1748                 ret = dmar_set_interrupt(iommu);
1749                 if (ret)
1750                         goto error;
1751
1752                 iommu_set_root_entry(iommu);
1753
1754                 iommu_flush_context_global(iommu, 0);
1755                 iommu_flush_iotlb_global(iommu, 0);
1756
1757                 iommu_disable_protect_mem_regions(iommu);
1758
1759                 ret = iommu_enable_translation(iommu);
1760                 if (ret)
1761                         goto error;
1762         }
1763
1764         return 0;
1765 error:
1766         for_each_drhd_unit(drhd) {
1767                 if (drhd->ignored)
1768                         continue;
1769                 iommu = drhd->iommu;
1770                 free_iommu(iommu);
1771         }
1772         return ret;
1773 }
1774
1775 static inline u64 aligned_size(u64 host_addr, size_t size)
1776 {
1777         u64 addr;
1778         addr = (host_addr & (~PAGE_MASK_4K)) + size;
1779         return PAGE_ALIGN_4K(addr);
1780 }
1781
1782 struct iova *
1783 iommu_alloc_iova(struct dmar_domain *domain, size_t size, u64 end)
1784 {
1785         struct iova *piova;
1786
1787         /* Make sure it's in range */
1788         end = min_t(u64, DOMAIN_MAX_ADDR(domain->gaw), end);
1789         if (!size || (IOVA_START_ADDR + size > end))
1790                 return NULL;
1791
1792         piova = alloc_iova(&domain->iovad,
1793                         size >> PAGE_SHIFT_4K, IOVA_PFN(end), 1);
1794         return piova;
1795 }
1796
1797 static struct iova *
1798 __intel_alloc_iova(struct device *dev, struct dmar_domain *domain,
1799                 size_t size)
1800 {
1801         struct pci_dev *pdev = to_pci_dev(dev);
1802         struct iova *iova = NULL;
1803
1804         if ((pdev->dma_mask <= DMA_32BIT_MASK) || (dmar_forcedac)) {
1805                 iova = iommu_alloc_iova(domain, size, pdev->dma_mask);
1806         } else  {
1807                 /*
1808                  * First try to allocate an io virtual address in
1809                  * DMA_32BIT_MASK and if that fails then try allocating
1810                  * from higher range
1811                  */
1812                 iova = iommu_alloc_iova(domain, size, DMA_32BIT_MASK);
1813                 if (!iova)
1814                         iova = iommu_alloc_iova(domain, size, pdev->dma_mask);
1815         }
1816
1817         if (!iova) {
1818                 printk(KERN_ERR"Allocating iova for %s failed", pci_name(pdev));
1819                 return NULL;
1820         }
1821
1822         return iova;
1823 }
1824
1825 static struct dmar_domain *
1826 get_valid_domain_for_dev(struct pci_dev *pdev)
1827 {
1828         struct dmar_domain *domain;
1829         int ret;
1830
1831         domain = get_domain_for_dev(pdev,
1832                         DEFAULT_DOMAIN_ADDRESS_WIDTH);
1833         if (!domain) {
1834                 printk(KERN_ERR
1835                         "Allocating domain for %s failed", pci_name(pdev));
1836                 return NULL;
1837         }
1838
1839         /* make sure context mapping is ok */
1840         if (unlikely(!domain_context_mapped(domain, pdev))) {
1841                 ret = domain_context_mapping(domain, pdev);
1842                 if (ret) {
1843                         printk(KERN_ERR
1844                                 "Domain context map for %s failed",
1845                                 pci_name(pdev));
1846                         return NULL;
1847                 }
1848         }
1849
1850         return domain;
1851 }
1852
1853 static dma_addr_t intel_map_single(struct device *hwdev, void *addr,
1854         size_t size, int dir)
1855 {
1856         struct pci_dev *pdev = to_pci_dev(hwdev);
1857         int ret;
1858         struct dmar_domain *domain;
1859         unsigned long start_addr;
1860         struct iova *iova;
1861         int prot = 0;
1862
1863         BUG_ON(dir == DMA_NONE);
1864         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1865                 return virt_to_bus(addr);
1866
1867         domain = get_valid_domain_for_dev(pdev);
1868         if (!domain)
1869                 return 0;
1870
1871         addr = (void *)virt_to_phys(addr);
1872         size = aligned_size((u64)addr, size);
1873
1874         iova = __intel_alloc_iova(hwdev, domain, size);
1875         if (!iova)
1876                 goto error;
1877
1878         start_addr = iova->pfn_lo << PAGE_SHIFT_4K;
1879
1880         /*
1881          * Check if DMAR supports zero-length reads on write only
1882          * mappings..
1883          */
1884         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
1885                         !cap_zlr(domain->iommu->cap))
1886                 prot |= DMA_PTE_READ;
1887         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
1888                 prot |= DMA_PTE_WRITE;
1889         /*
1890          * addr - (addr + size) might be partial page, we should map the whole
1891          * page.  Note: if two part of one page are separately mapped, we
1892          * might have two guest_addr mapping to the same host addr, but this
1893          * is not a big problem
1894          */
1895         ret = domain_page_mapping(domain, start_addr,
1896                 ((u64)addr) & PAGE_MASK_4K, size, prot);
1897         if (ret)
1898                 goto error;
1899
1900         pr_debug("Device %s request: %lx@%llx mapping: %lx@%llx, dir %d\n",
1901                 pci_name(pdev), size, (u64)addr,
1902                 size, (u64)start_addr, dir);
1903
1904         /* it's a non-present to present mapping */
1905         ret = iommu_flush_iotlb_psi(domain->iommu, domain->id,
1906                         start_addr, size >> PAGE_SHIFT_4K, 1);
1907         if (ret)
1908                 iommu_flush_write_buffer(domain->iommu);
1909
1910         return (start_addr + ((u64)addr & (~PAGE_MASK_4K)));
1911
1912 error:
1913         if (iova)
1914                 __free_iova(&domain->iovad, iova);
1915         printk(KERN_ERR"Device %s request: %lx@%llx dir %d --- failed\n",
1916                 pci_name(pdev), size, (u64)addr, dir);
1917         return 0;
1918 }
1919
1920 static void intel_unmap_single(struct device *dev, dma_addr_t dev_addr,
1921         size_t size, int dir)
1922 {
1923         struct pci_dev *pdev = to_pci_dev(dev);
1924         struct dmar_domain *domain;
1925         unsigned long start_addr;
1926         struct iova *iova;
1927
1928         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
1929                 return;
1930         domain = find_domain(pdev);
1931         BUG_ON(!domain);
1932
1933         iova = find_iova(&domain->iovad, IOVA_PFN(dev_addr));
1934         if (!iova)
1935                 return;
1936
1937         start_addr = iova->pfn_lo << PAGE_SHIFT_4K;
1938         size = aligned_size((u64)dev_addr, size);
1939
1940         pr_debug("Device %s unmapping: %lx@%llx\n",
1941                 pci_name(pdev), size, (u64)start_addr);
1942
1943         /*  clear the whole page */
1944         dma_pte_clear_range(domain, start_addr, start_addr + size);
1945         /* free page tables */
1946         dma_pte_free_pagetable(domain, start_addr, start_addr + size);
1947
1948         if (iommu_flush_iotlb_psi(domain->iommu, domain->id, start_addr,
1949                         size >> PAGE_SHIFT_4K, 0))
1950                 iommu_flush_write_buffer(domain->iommu);
1951
1952         /* free iova */
1953         __free_iova(&domain->iovad, iova);
1954 }
1955
1956 static void * intel_alloc_coherent(struct device *hwdev, size_t size,
1957                        dma_addr_t *dma_handle, gfp_t flags)
1958 {
1959         void *vaddr;
1960         int order;
1961
1962         size = PAGE_ALIGN_4K(size);
1963         order = get_order(size);
1964         flags &= ~(GFP_DMA | GFP_DMA32);
1965
1966         vaddr = (void *)__get_free_pages(flags, order);
1967         if (!vaddr)
1968                 return NULL;
1969         memset(vaddr, 0, size);
1970
1971         *dma_handle = intel_map_single(hwdev, vaddr, size, DMA_BIDIRECTIONAL);
1972         if (*dma_handle)
1973                 return vaddr;
1974         free_pages((unsigned long)vaddr, order);
1975         return NULL;
1976 }
1977
1978 static void intel_free_coherent(struct device *hwdev, size_t size,
1979         void *vaddr, dma_addr_t dma_handle)
1980 {
1981         int order;
1982
1983         size = PAGE_ALIGN_4K(size);
1984         order = get_order(size);
1985
1986         intel_unmap_single(hwdev, dma_handle, size, DMA_BIDIRECTIONAL);
1987         free_pages((unsigned long)vaddr, order);
1988 }
1989
1990 #define SG_ENT_VIRT_ADDRESS(sg) (sg_virt((sg)))
1991 static void intel_unmap_sg(struct device *hwdev, struct scatterlist *sglist,
1992         int nelems, int dir)
1993 {
1994         int i;
1995         struct pci_dev *pdev = to_pci_dev(hwdev);
1996         struct dmar_domain *domain;
1997         unsigned long start_addr;
1998         struct iova *iova;
1999         size_t size = 0;
2000         void *addr;
2001         struct scatterlist *sg;
2002
2003         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2004                 return;
2005
2006         domain = find_domain(pdev);
2007
2008         iova = find_iova(&domain->iovad, IOVA_PFN(sglist[0].dma_address));
2009         if (!iova)
2010                 return;
2011         for_each_sg(sglist, sg, nelems, i) {
2012                 addr = SG_ENT_VIRT_ADDRESS(sg);
2013                 size += aligned_size((u64)addr, sg->length);
2014         }
2015
2016         start_addr = iova->pfn_lo << PAGE_SHIFT_4K;
2017
2018         /*  clear the whole page */
2019         dma_pte_clear_range(domain, start_addr, start_addr + size);
2020         /* free page tables */
2021         dma_pte_free_pagetable(domain, start_addr, start_addr + size);
2022
2023         if (iommu_flush_iotlb_psi(domain->iommu, domain->id, start_addr,
2024                         size >> PAGE_SHIFT_4K, 0))
2025                 iommu_flush_write_buffer(domain->iommu);
2026
2027         /* free iova */
2028         __free_iova(&domain->iovad, iova);
2029 }
2030
2031 static int intel_nontranslate_map_sg(struct device *hddev,
2032         struct scatterlist *sglist, int nelems, int dir)
2033 {
2034         int i;
2035         struct scatterlist *sg;
2036
2037         for_each_sg(sglist, sg, nelems, i) {
2038                 BUG_ON(!sg_page(sg));
2039                 sg->dma_address = virt_to_bus(SG_ENT_VIRT_ADDRESS(sg));
2040                 sg->dma_length = sg->length;
2041         }
2042         return nelems;
2043 }
2044
2045 static int intel_map_sg(struct device *hwdev, struct scatterlist *sglist,
2046                                 int nelems, int dir)
2047 {
2048         void *addr;
2049         int i;
2050         struct pci_dev *pdev = to_pci_dev(hwdev);
2051         struct dmar_domain *domain;
2052         size_t size = 0;
2053         int prot = 0;
2054         size_t offset = 0;
2055         struct iova *iova = NULL;
2056         int ret;
2057         struct scatterlist *sg;
2058         unsigned long start_addr;
2059
2060         BUG_ON(dir == DMA_NONE);
2061         if (pdev->dev.archdata.iommu == DUMMY_DEVICE_DOMAIN_INFO)
2062                 return intel_nontranslate_map_sg(hwdev, sglist, nelems, dir);
2063
2064         domain = get_valid_domain_for_dev(pdev);
2065         if (!domain)
2066                 return 0;
2067
2068         for_each_sg(sglist, sg, nelems, i) {
2069                 addr = SG_ENT_VIRT_ADDRESS(sg);
2070                 addr = (void *)virt_to_phys(addr);
2071                 size += aligned_size((u64)addr, sg->length);
2072         }
2073
2074         iova = __intel_alloc_iova(hwdev, domain, size);
2075         if (!iova) {
2076                 sglist->dma_length = 0;
2077                 return 0;
2078         }
2079
2080         /*
2081          * Check if DMAR supports zero-length reads on write only
2082          * mappings..
2083          */
2084         if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
2085                         !cap_zlr(domain->iommu->cap))
2086                 prot |= DMA_PTE_READ;
2087         if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
2088                 prot |= DMA_PTE_WRITE;
2089
2090         start_addr = iova->pfn_lo << PAGE_SHIFT_4K;
2091         offset = 0;
2092         for_each_sg(sglist, sg, nelems, i) {
2093                 addr = SG_ENT_VIRT_ADDRESS(sg);
2094                 addr = (void *)virt_to_phys(addr);
2095                 size = aligned_size((u64)addr, sg->length);
2096                 ret = domain_page_mapping(domain, start_addr + offset,
2097                         ((u64)addr) & PAGE_MASK_4K,
2098                         size, prot);
2099                 if (ret) {
2100                         /*  clear the page */
2101                         dma_pte_clear_range(domain, start_addr,
2102                                   start_addr + offset);
2103                         /* free page tables */
2104                         dma_pte_free_pagetable(domain, start_addr,
2105                                   start_addr + offset);
2106                         /* free iova */
2107                         __free_iova(&domain->iovad, iova);
2108                         return 0;
2109                 }
2110                 sg->dma_address = start_addr + offset +
2111                                 ((u64)addr & (~PAGE_MASK_4K));
2112                 sg->dma_length = sg->length;
2113                 offset += size;
2114         }
2115
2116         /* it's a non-present to present mapping */
2117         if (iommu_flush_iotlb_psi(domain->iommu, domain->id,
2118                         start_addr, offset >> PAGE_SHIFT_4K, 1))
2119                 iommu_flush_write_buffer(domain->iommu);
2120         return nelems;
2121 }
2122
2123 static struct dma_mapping_ops intel_dma_ops = {
2124         .alloc_coherent = intel_alloc_coherent,
2125         .free_coherent = intel_free_coherent,
2126         .map_single = intel_map_single,
2127         .unmap_single = intel_unmap_single,
2128         .map_sg = intel_map_sg,
2129         .unmap_sg = intel_unmap_sg,
2130 };
2131
2132 static inline int iommu_domain_cache_init(void)
2133 {
2134         int ret = 0;
2135
2136         iommu_domain_cache = kmem_cache_create("iommu_domain",
2137                                          sizeof(struct dmar_domain),
2138                                          0,
2139                                          SLAB_HWCACHE_ALIGN,
2140
2141                                          NULL);
2142         if (!iommu_domain_cache) {
2143                 printk(KERN_ERR "Couldn't create iommu_domain cache\n");
2144                 ret = -ENOMEM;
2145         }
2146
2147         return ret;
2148 }
2149
2150 static inline int iommu_devinfo_cache_init(void)
2151 {
2152         int ret = 0;
2153
2154         iommu_devinfo_cache = kmem_cache_create("iommu_devinfo",
2155                                          sizeof(struct device_domain_info),
2156                                          0,
2157                                          SLAB_HWCACHE_ALIGN,
2158
2159                                          NULL);
2160         if (!iommu_devinfo_cache) {
2161                 printk(KERN_ERR "Couldn't create devinfo cache\n");
2162                 ret = -ENOMEM;
2163         }
2164
2165         return ret;
2166 }
2167
2168 static inline int iommu_iova_cache_init(void)
2169 {
2170         int ret = 0;
2171
2172         iommu_iova_cache = kmem_cache_create("iommu_iova",
2173                                          sizeof(struct iova),
2174                                          0,
2175                                          SLAB_HWCACHE_ALIGN,
2176
2177                                          NULL);
2178         if (!iommu_iova_cache) {
2179                 printk(KERN_ERR "Couldn't create iova cache\n");
2180                 ret = -ENOMEM;
2181         }
2182
2183         return ret;
2184 }
2185
2186 static int __init iommu_init_mempool(void)
2187 {
2188         int ret;
2189         ret = iommu_iova_cache_init();
2190         if (ret)
2191                 return ret;
2192
2193         ret = iommu_domain_cache_init();
2194         if (ret)
2195                 goto domain_error;
2196
2197         ret = iommu_devinfo_cache_init();
2198         if (!ret)
2199                 return ret;
2200
2201         kmem_cache_destroy(iommu_domain_cache);
2202 domain_error:
2203         kmem_cache_destroy(iommu_iova_cache);
2204
2205         return -ENOMEM;
2206 }
2207
2208 static void __init iommu_exit_mempool(void)
2209 {
2210         kmem_cache_destroy(iommu_devinfo_cache);
2211         kmem_cache_destroy(iommu_domain_cache);
2212         kmem_cache_destroy(iommu_iova_cache);
2213
2214 }
2215
2216 void __init detect_intel_iommu(void)
2217 {
2218         if (swiotlb || no_iommu || iommu_detected || dmar_disabled)
2219                 return;
2220         if (early_dmar_detect()) {
2221                 iommu_detected = 1;
2222         }
2223 }
2224
2225 static void __init init_no_remapping_devices(void)
2226 {
2227         struct dmar_drhd_unit *drhd;
2228
2229         for_each_drhd_unit(drhd) {
2230                 if (!drhd->include_all) {
2231                         int i;
2232                         for (i = 0; i < drhd->devices_cnt; i++)
2233                                 if (drhd->devices[i] != NULL)
2234                                         break;
2235                         /* ignore DMAR unit if no pci devices exist */
2236                         if (i == drhd->devices_cnt)
2237                                 drhd->ignored = 1;
2238                 }
2239         }
2240
2241         if (dmar_map_gfx)
2242                 return;
2243
2244         for_each_drhd_unit(drhd) {
2245                 int i;
2246                 if (drhd->ignored || drhd->include_all)
2247                         continue;
2248
2249                 for (i = 0; i < drhd->devices_cnt; i++)
2250                         if (drhd->devices[i] &&
2251                                 !IS_GFX_DEVICE(drhd->devices[i]))
2252                                 break;
2253
2254                 if (i < drhd->devices_cnt)
2255                         continue;
2256
2257                 /* bypass IOMMU if it is just for gfx devices */
2258                 drhd->ignored = 1;
2259                 for (i = 0; i < drhd->devices_cnt; i++) {
2260                         if (!drhd->devices[i])
2261                                 continue;
2262                         drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO;
2263                 }
2264         }
2265 }
2266
2267 int __init intel_iommu_init(void)
2268 {
2269         int ret = 0;
2270
2271         if (no_iommu || swiotlb || dmar_disabled)
2272                 return -ENODEV;
2273
2274         if (dmar_table_init())
2275                 return  -ENODEV;
2276
2277         iommu_init_mempool();
2278         dmar_init_reserved_ranges();
2279
2280         init_no_remapping_devices();
2281
2282         ret = init_dmars();
2283         if (ret) {
2284                 printk(KERN_ERR "IOMMU: dmar init failed\n");
2285                 put_iova_domain(&reserved_iova_list);
2286                 iommu_exit_mempool();
2287                 return ret;
2288         }
2289         printk(KERN_INFO
2290         "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n");
2291
2292         force_iommu = 1;
2293         dma_ops = &intel_dma_ops;
2294         return 0;
2295 }
2296